From c41abc24de0eca7728ca375135cd55afa56edded Mon Sep 17 00:00:00 2001 From: Hung-chih Yang Date: Wed, 22 Sep 2021 15:05:06 -0700 Subject: [PATCH 1/4] no None/Unknown consolidation --- packages/orchestratorlib/src/orchestratorhelper.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/orchestratorlib/src/orchestratorhelper.ts b/packages/orchestratorlib/src/orchestratorhelper.ts index 790a142da..3404b62d9 100644 --- a/packages/orchestratorlib/src/orchestratorhelper.ts +++ b/packages/orchestratorlib/src/orchestratorhelper.ts @@ -160,7 +160,6 @@ export class OrchestratorHelper { utteranceEntityLabelDuplicateMap); } } - Utility.processUnknownSpuriousLabelsInUtteranceLabelsMap({utteranceLabelsMap, utteranceLabelDuplicateMap}); return { utteranceLabelsMap, utteranceLabelDuplicateMap, @@ -353,7 +352,7 @@ export class OrchestratorHelper { default: throw new Error(`Unknown file type ${ext}`); } } catch (error: any) { - throw new Error(`${error.message}${os.EOL}Failed to parse ${filePath}`); + throw new Error(`${error.message}${os.EOL}Failed to parse ${filePath}, error=${os.EOL}${UtilityDispatcher.jsonStringify(error)}`); } } From b3f3131ff2a066162b06fb6860db05cd65609dca Mon Sep 17 00:00:00 2001 From: Hung-chih Yang Date: Thu, 23 Sep 2021 09:11:29 -0700 Subject: [PATCH 2/4] Remove silent error processing --- .../orchestratorlib/src/orchestratorhelper.ts | 24 +++---------- .../src/utilitylabelresolver.ts | 36 +++++++++---------- 2 files changed, 22 insertions(+), 38 deletions(-) diff --git a/packages/orchestratorlib/src/orchestratorhelper.ts b/packages/orchestratorlib/src/orchestratorhelper.ts index 3404b62d9..955b1fc1e 100644 --- a/packages/orchestratorlib/src/orchestratorhelper.ts +++ b/packages/orchestratorlib/src/orchestratorhelper.ts @@ -41,24 +41,8 @@ export class OrchestratorHelper { fs.mkdirSync(path, {recursive: true}); } - public static readBluSnapshotFile(filePath: string): string { - return ReadText.readSync(filePath); - // ---- NOTE - // the code below was trying to normalize unknown labels in a BLU file, - // but the unknown labels should have been processed during ingesting - // a raw input file (LU, QnA, TSV, etc.) and before creating a BLU file, - // so there is really no need to process unknown labels in a BLU file - // anymore. The line below is thus deprecated, especially now the BLU - // file can be a JSON, so the statement below does not apply anyway. - // ---- return Utility.processUnknownSpuriousLabelsInTsvBluFileContent(ReadText.readSync(filePath)); - } - public static readFile(filePath: string): string { - try { - return ReadText.readSync(filePath); - } catch { - return ''; - } + return ReadText.readSync(filePath); } public static writeToFile(filePath: string, content: string, options: any = {encoding: 'utf8', flag: 'w'}): string { @@ -124,10 +108,10 @@ export class OrchestratorHelper { } public static getSnapshotFromFile(snapshotPath: string) { - if (Utility.exists(snapshotPath) && !OrchestratorHelper.isDirectory(snapshotPath)) { - return new TextEncoder().encode(OrchestratorHelper.readBluSnapshotFile(snapshotPath)); + if (!Utility.exists(snapshotPath) || OrchestratorHelper.isDirectory(snapshotPath)) { + throw new Error(`Cannot read ${snapshotPath}`); } - return new Uint8Array(); + return new TextEncoder().encode(OrchestratorHelper.readFile(snapshotPath)); } public static async getUtteranceLabelsMap( diff --git a/packages/orchestratorlib/src/utilitylabelresolver.ts b/packages/orchestratorlib/src/utilitylabelresolver.ts index 6a846b866..12ddc11e5 100644 --- a/packages/orchestratorlib/src/utilitylabelresolver.ts +++ b/packages/orchestratorlib/src/utilitylabelresolver.ts @@ -30,14 +30,14 @@ export class UtilityLabelResolver { } = { ignore_same_example: ignoreSameExample, }; - Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), ignoreSameExample=${ignoreSameExample}`); - Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), resetAll=${resetAll}`); - Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), ignoreSameExampleObject=${ignoreSameExampleObject}`); - Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), ignoreSameExampleObject.ignore_same_example=${ignoreSameExampleObject.ignore_same_example}`); + Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), ignoreSameExample=${ignoreSameExample}`); + Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), resetAll=${resetAll}`); + Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), ignoreSameExampleObject=${ignoreSameExampleObject}`); + Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), ignoreSameExampleObject.ignore_same_example=${ignoreSameExampleObject.ignore_same_example}`); const ignoreSameExampleObjectJson: string = Utility.jsonStringify(ignoreSameExampleObject); - Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), ignoreSameExampleObjectJson=${ignoreSameExampleObjectJson}`); + Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), ignoreSameExampleObjectJson=${ignoreSameExampleObjectJson}`); LabelResolver.setRuntimeParams(ignoreSameExampleObjectJson, resetAll); - Utility.debuggingLog(`read to call Utility.getConfigJson(), LabelResolver.LabelResolver=${LabelResolver.LabelResolver}`); + Utility.debuggingLog(`ready to call Utility.getConfigJson(), LabelResolver.LabelResolver=${LabelResolver.LabelResolver}`); return LabelResolver.getConfigJson(); } @@ -49,14 +49,14 @@ export class UtilityLabelResolver { } = { use_compact_embeddings: !fullEmbeddings, }; - Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), fullEmbeddings=${fullEmbeddings}`); - Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), resetAll=${resetAll}`); - Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), useCompactEmbeddingsObject=${useCompactEmbeddingsObject}`); - Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), useCompactEmbeddingsObject.use_compact_embeddings=${useCompactEmbeddingsObject.use_compact_embeddings}`); + Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), fullEmbeddings=${fullEmbeddings}`); + Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), resetAll=${resetAll}`); + Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), useCompactEmbeddingsObject=${useCompactEmbeddingsObject}`); + Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), useCompactEmbeddingsObject.use_compact_embeddings=${useCompactEmbeddingsObject.use_compact_embeddings}`); const useCompactEmbeddingsObjectJson: string = Utility.jsonStringify(useCompactEmbeddingsObject); - Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), useCompactEmbeddingsObjectJson=${useCompactEmbeddingsObjectJson}`); + Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), useCompactEmbeddingsObjectJson=${useCompactEmbeddingsObjectJson}`); LabelResolver.setRuntimeParams(useCompactEmbeddingsObjectJson, resetAll); - Utility.debuggingLog(`read to call Utility.getConfigJson(), LabelResolver.LabelResolver=${LabelResolver.LabelResolver}`); + Utility.debuggingLog(`ready to call Utility.getConfigJson(), LabelResolver.LabelResolver=${LabelResolver.LabelResolver}`); return LabelResolver.getConfigJson(); } @@ -68,14 +68,14 @@ export class UtilityLabelResolver { } = { knn_k: knnK, }; - Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), knnK=${knnK}`); - Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), resetAll=${resetAll}`); - Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), knnKObject=${knnKObject}`); - Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), knnKObject.knn_k=${knnKObject.knn_k}`); + Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), knnK=${knnK}`); + Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), resetAll=${resetAll}`); + Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), knnKObject=${knnKObject}`); + Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), knnKObject.knn_k=${knnKObject.knn_k}`); const knnKObjectJson: string = Utility.jsonStringify(knnKObject); - Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), knnKObjectJson=${knnKObjectJson}`); + Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), knnKObjectJson=${knnKObjectJson}`); LabelResolver.setRuntimeParams(knnKObjectJson, resetAll); - Utility.debuggingLog(`read to call Utility.getConfigJson(), LabelResolver.LabelResolver=${LabelResolver.LabelResolver}`); + Utility.debuggingLog(`ready to call Utility.getConfigJson(), LabelResolver.LabelResolver=${LabelResolver.LabelResolver}`); return LabelResolver.getConfigJson(); } From 34f148f88b89ce0ede067fef9032be85c866ab28 Mon Sep 17 00:00:00 2001 From: Hung-chih Yang Date: Thu, 23 Sep 2021 10:53:02 -0700 Subject: [PATCH 3/4] create can read a default, non-existent snapshot BLU file --- .../orchestratorlib/src/orchestratorhelper.ts | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/packages/orchestratorlib/src/orchestratorhelper.ts b/packages/orchestratorlib/src/orchestratorhelper.ts index 955b1fc1e..f33431375 100644 --- a/packages/orchestratorlib/src/orchestratorhelper.ts +++ b/packages/orchestratorlib/src/orchestratorhelper.ts @@ -42,7 +42,22 @@ export class OrchestratorHelper { } public static readFile(filePath: string): string { - return ReadText.readSync(filePath); + UtilityDispatcher.debuggingLog1( + 'OrchestratorHElper.readFile() calling ReadText.readSync()', + filePath); + try { + const fileStats: fs.Stats = fs.statSync(filePath); + if (fileStats.size === 0) { + return ''; + } + return ReadText.readSync(filePath); + } catch (error) { + UtilityDispatcher.debuggingLog2( + 'EXCEPTION calling ReadText.readSync()', + filePath, + error); + throw error; + } } public static writeToFile(filePath: string, content: string, options: any = {encoding: 'utf8', flag: 'w'}): string { @@ -108,10 +123,13 @@ export class OrchestratorHelper { } public static getSnapshotFromFile(snapshotPath: string) { - if (!Utility.exists(snapshotPath) || OrchestratorHelper.isDirectory(snapshotPath)) { - throw new Error(`Cannot read ${snapshotPath}`); + UtilityDispatcher.debuggingLog1( + 'OrchestratorHelper.getSnapshotFromFile()', + snapshotPath); + if (Utility.exists(snapshotPath) && !OrchestratorHelper.isDirectory(snapshotPath)) { + return new TextEncoder().encode(OrchestratorHelper.readFile(snapshotPath)); } - return new TextEncoder().encode(OrchestratorHelper.readFile(snapshotPath)); + return new Uint8Array(); } public static async getUtteranceLabelsMap( From d533236a8c22002ef58b7a6f6f8ef82a19cf7fbb Mon Sep 17 00:00:00 2001 From: Hung-chih Yang Date: Thu, 23 Sep 2021 12:43:29 -0700 Subject: [PATCH 4/4] LUn parsing without LUIS validation --- packages/orchestratorlib/src/orchestratorhelper.ts | 12 +++++++++--- .../orchestratorlib/test/orchestratorhelper.test.ts | 4 +++- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/packages/orchestratorlib/src/orchestratorhelper.ts b/packages/orchestratorlib/src/orchestratorhelper.ts index f33431375..a4ec8a215 100644 --- a/packages/orchestratorlib/src/orchestratorhelper.ts +++ b/packages/orchestratorlib/src/orchestratorhelper.ts @@ -17,7 +17,7 @@ import {Utility} from './utility'; import {Utility as UtilityDispatcher} from '@microsoft/bf-dispatcher'; const ReadText: any = require('read-text-file'); -const LuisBuilder: any = require('@microsoft/bf-lu').V2.LuisBuilder; +const luisCollateBuildNoValidate: any = require('@microsoft/bf-lu/lib/parser/luis/luisCollate').build; const QnaMakerBuilder: any = require('@microsoft/bf-lu').V2.QnAMakerBuilder; const processedFiles: string[] = []; @@ -233,7 +233,7 @@ export class OrchestratorHelper { } public static async getEntitiesInLu(luObject: any): Promise { - const luisObject: any = await LuisBuilder.fromLUAsync([luObject], OrchestratorHelper.findLuFiles); + const luisObject: any = await luisCollateBuildNoValidate([luObject], false, '', OrchestratorHelper.findLuFiles); return this.transformEntities(luisObject); } @@ -419,6 +419,12 @@ export class OrchestratorHelper { utteranceLabelDuplicateMap: Map>, utteranceEntityLabelsMap: Map, utteranceEntityLabelDuplicateMap: Map): Promise { + UtilityDispatcher.debuggingLog1( + 'OrchestratorHelper.parseLuContent()', + luFile); + UtilityDispatcher.debuggingLog1( + 'OrchestratorHelper.parseLuContent()', + luContent); if (!luContent || luContent.length === 0) { return; } @@ -427,7 +433,7 @@ export class OrchestratorHelper { content: luContent, id: luFile, }; - const luisObject: any = await LuisBuilder.fromLUAsync([luObject], OrchestratorHelper.findLuFiles); + const luisObject: any = await luisCollateBuildNoValidate([luObject], false, '', OrchestratorHelper.findLuFiles); if (Utility.toPrintDetailedDebuggingLogToConsole) { UtilityDispatcher.debuggingNamedLog1('OrchestratorHelper.parseLuContent(): calling getIntentsEntitiesUtterances()', luisObject, 'luisObject'); } diff --git a/packages/orchestratorlib/test/orchestratorhelper.test.ts b/packages/orchestratorlib/test/orchestratorhelper.test.ts index 19b03cbf9..23ef63bb4 100644 --- a/packages/orchestratorlib/test/orchestratorhelper.test.ts +++ b/packages/orchestratorlib/test/orchestratorhelper.test.ts @@ -425,6 +425,7 @@ describe('Test Suite - orchestratorhelper', () => { assert.ok((utteranceEntityLabelsMap.get(utterance) as Label[]).length === 3, `(utteranceEntityLabelsMap.get(utterance) as Label[]).length=${(utteranceEntityLabelsMap.get(utterance) as Label[]).length}`); }); + it('Test.0300 OrchestratorHelper.getJsonIntentEntityScoresUtterances()', function () { Utility.resetFlagToPrintDebuggingLogToConsole(UnitTestHelper.getDefaultUnitTestDebuggingLogFlag()); this.timeout(UnitTestHelper.getDefaultUnitTestTimeout()); @@ -587,6 +588,7 @@ describe('Test Suite - orchestratorhelper', () => { assert.ok(result.has('Add item'), 'Incorrect result from getUtteranceLabelsMap, missing Add item utterance'); assert.ok(result.has('delete to do go shopping'), 'Incorrect result from getUtteranceLabelsMap, missing delete to do go shopping utterance'); }); + it('Test.0600 OrchestratorHelper.parseQnaFile()', async () => { const validFile: string = './test/fixtures/parser/valid.qna'; const inValidFile: string = './test/fixtures/parser/invalid.qna'; @@ -608,7 +610,7 @@ describe('Test Suite - orchestratorhelper', () => { utteranceLabelDuplicateMap); assert.ok(utteranceLabelsMap.size > 0); }); - it('Test.0600 OrchestratorHelper.parseLuFile()', async () => { + it('Test.0601 OrchestratorHelper.parseLuFile()', async () => { const validFile: string = './test/fixtures/parser/valid.lu'; const inValidFile: string = './test/fixtures/parser/invalid.lu'; const utteranceLabelsMap: Map> = new Map>();