diff --git a/packages/orchestratorlib/src/orchestratorhelper.ts b/packages/orchestratorlib/src/orchestratorhelper.ts index 790a142da..a4ec8a215 100644 --- a/packages/orchestratorlib/src/orchestratorhelper.ts +++ b/packages/orchestratorlib/src/orchestratorhelper.ts @@ -17,7 +17,7 @@ import {Utility} from './utility'; import {Utility as UtilityDispatcher} from '@microsoft/bf-dispatcher'; const ReadText: any = require('read-text-file'); -const LuisBuilder: any = require('@microsoft/bf-lu').V2.LuisBuilder; +const luisCollateBuildNoValidate: any = require('@microsoft/bf-lu/lib/parser/luis/luisCollate').build; const QnaMakerBuilder: any = require('@microsoft/bf-lu').V2.QnAMakerBuilder; const processedFiles: string[] = []; @@ -41,23 +41,22 @@ export class OrchestratorHelper { fs.mkdirSync(path, {recursive: true}); } - public static readBluSnapshotFile(filePath: string): string { - return ReadText.readSync(filePath); - // ---- NOTE - // the code below was trying to normalize unknown labels in a BLU file, - // but the unknown labels should have been processed during ingesting - // a raw input file (LU, QnA, TSV, etc.) and before creating a BLU file, - // so there is really no need to process unknown labels in a BLU file - // anymore. The line below is thus deprecated, especially now the BLU - // file can be a JSON, so the statement below does not apply anyway. - // ---- return Utility.processUnknownSpuriousLabelsInTsvBluFileContent(ReadText.readSync(filePath)); - } - public static readFile(filePath: string): string { + UtilityDispatcher.debuggingLog1( + 'OrchestratorHElper.readFile() calling ReadText.readSync()', + filePath); try { + const fileStats: fs.Stats = fs.statSync(filePath); + if (fileStats.size === 0) { + return ''; + } return ReadText.readSync(filePath); - } catch { - return ''; + } catch (error) { + UtilityDispatcher.debuggingLog2( + 'EXCEPTION calling ReadText.readSync()', + filePath, + error); + throw error; } } @@ -124,8 +123,11 @@ export class OrchestratorHelper { } public static getSnapshotFromFile(snapshotPath: string) { + UtilityDispatcher.debuggingLog1( + 'OrchestratorHelper.getSnapshotFromFile()', + snapshotPath); if (Utility.exists(snapshotPath) && !OrchestratorHelper.isDirectory(snapshotPath)) { - return new TextEncoder().encode(OrchestratorHelper.readBluSnapshotFile(snapshotPath)); + return new TextEncoder().encode(OrchestratorHelper.readFile(snapshotPath)); } return new Uint8Array(); } @@ -160,7 +162,6 @@ export class OrchestratorHelper { utteranceEntityLabelDuplicateMap); } } - Utility.processUnknownSpuriousLabelsInUtteranceLabelsMap({utteranceLabelsMap, utteranceLabelDuplicateMap}); return { utteranceLabelsMap, utteranceLabelDuplicateMap, @@ -232,7 +233,7 @@ export class OrchestratorHelper { } public static async getEntitiesInLu(luObject: any): Promise { - const luisObject: any = await LuisBuilder.fromLUAsync([luObject], OrchestratorHelper.findLuFiles); + const luisObject: any = await luisCollateBuildNoValidate([luObject], false, '', OrchestratorHelper.findLuFiles); return this.transformEntities(luisObject); } @@ -353,7 +354,7 @@ export class OrchestratorHelper { default: throw new Error(`Unknown file type ${ext}`); } } catch (error: any) { - throw new Error(`${error.message}${os.EOL}Failed to parse ${filePath}`); + throw new Error(`${error.message}${os.EOL}Failed to parse ${filePath}, error=${os.EOL}${UtilityDispatcher.jsonStringify(error)}`); } } @@ -418,6 +419,12 @@ export class OrchestratorHelper { utteranceLabelDuplicateMap: Map>, utteranceEntityLabelsMap: Map, utteranceEntityLabelDuplicateMap: Map): Promise { + UtilityDispatcher.debuggingLog1( + 'OrchestratorHelper.parseLuContent()', + luFile); + UtilityDispatcher.debuggingLog1( + 'OrchestratorHelper.parseLuContent()', + luContent); if (!luContent || luContent.length === 0) { return; } @@ -426,7 +433,7 @@ export class OrchestratorHelper { content: luContent, id: luFile, }; - const luisObject: any = await LuisBuilder.fromLUAsync([luObject], OrchestratorHelper.findLuFiles); + const luisObject: any = await luisCollateBuildNoValidate([luObject], false, '', OrchestratorHelper.findLuFiles); if (Utility.toPrintDetailedDebuggingLogToConsole) { UtilityDispatcher.debuggingNamedLog1('OrchestratorHelper.parseLuContent(): calling getIntentsEntitiesUtterances()', luisObject, 'luisObject'); } diff --git a/packages/orchestratorlib/src/utilitylabelresolver.ts b/packages/orchestratorlib/src/utilitylabelresolver.ts index 6a846b866..12ddc11e5 100644 --- a/packages/orchestratorlib/src/utilitylabelresolver.ts +++ b/packages/orchestratorlib/src/utilitylabelresolver.ts @@ -30,14 +30,14 @@ export class UtilityLabelResolver { } = { ignore_same_example: ignoreSameExample, }; - Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), ignoreSameExample=${ignoreSameExample}`); - Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), resetAll=${resetAll}`); - Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), ignoreSameExampleObject=${ignoreSameExampleObject}`); - Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), ignoreSameExampleObject.ignore_same_example=${ignoreSameExampleObject.ignore_same_example}`); + Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), ignoreSameExample=${ignoreSameExample}`); + Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), resetAll=${resetAll}`); + Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), ignoreSameExampleObject=${ignoreSameExampleObject}`); + Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), ignoreSameExampleObject.ignore_same_example=${ignoreSameExampleObject.ignore_same_example}`); const ignoreSameExampleObjectJson: string = Utility.jsonStringify(ignoreSameExampleObject); - Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), ignoreSameExampleObjectJson=${ignoreSameExampleObjectJson}`); + Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), ignoreSameExampleObjectJson=${ignoreSameExampleObjectJson}`); LabelResolver.setRuntimeParams(ignoreSameExampleObjectJson, resetAll); - Utility.debuggingLog(`read to call Utility.getConfigJson(), LabelResolver.LabelResolver=${LabelResolver.LabelResolver}`); + Utility.debuggingLog(`ready to call Utility.getConfigJson(), LabelResolver.LabelResolver=${LabelResolver.LabelResolver}`); return LabelResolver.getConfigJson(); } @@ -49,14 +49,14 @@ export class UtilityLabelResolver { } = { use_compact_embeddings: !fullEmbeddings, }; - Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), fullEmbeddings=${fullEmbeddings}`); - Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), resetAll=${resetAll}`); - Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), useCompactEmbeddingsObject=${useCompactEmbeddingsObject}`); - Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), useCompactEmbeddingsObject.use_compact_embeddings=${useCompactEmbeddingsObject.use_compact_embeddings}`); + Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), fullEmbeddings=${fullEmbeddings}`); + Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), resetAll=${resetAll}`); + Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), useCompactEmbeddingsObject=${useCompactEmbeddingsObject}`); + Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), useCompactEmbeddingsObject.use_compact_embeddings=${useCompactEmbeddingsObject.use_compact_embeddings}`); const useCompactEmbeddingsObjectJson: string = Utility.jsonStringify(useCompactEmbeddingsObject); - Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), useCompactEmbeddingsObjectJson=${useCompactEmbeddingsObjectJson}`); + Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), useCompactEmbeddingsObjectJson=${useCompactEmbeddingsObjectJson}`); LabelResolver.setRuntimeParams(useCompactEmbeddingsObjectJson, resetAll); - Utility.debuggingLog(`read to call Utility.getConfigJson(), LabelResolver.LabelResolver=${LabelResolver.LabelResolver}`); + Utility.debuggingLog(`ready to call Utility.getConfigJson(), LabelResolver.LabelResolver=${LabelResolver.LabelResolver}`); return LabelResolver.getConfigJson(); } @@ -68,14 +68,14 @@ export class UtilityLabelResolver { } = { knn_k: knnK, }; - Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), knnK=${knnK}`); - Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), resetAll=${resetAll}`); - Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), knnKObject=${knnKObject}`); - Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), knnKObject.knn_k=${knnKObject.knn_k}`); + Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), knnK=${knnK}`); + Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), resetAll=${resetAll}`); + Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), knnKObject=${knnKObject}`); + Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), knnKObject.knn_k=${knnKObject.knn_k}`); const knnKObjectJson: string = Utility.jsonStringify(knnKObject); - Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), knnKObjectJson=${knnKObjectJson}`); + Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), knnKObjectJson=${knnKObjectJson}`); LabelResolver.setRuntimeParams(knnKObjectJson, resetAll); - Utility.debuggingLog(`read to call Utility.getConfigJson(), LabelResolver.LabelResolver=${LabelResolver.LabelResolver}`); + Utility.debuggingLog(`ready to call Utility.getConfigJson(), LabelResolver.LabelResolver=${LabelResolver.LabelResolver}`); return LabelResolver.getConfigJson(); } diff --git a/packages/orchestratorlib/test/orchestratorhelper.test.ts b/packages/orchestratorlib/test/orchestratorhelper.test.ts index 19b03cbf9..23ef63bb4 100644 --- a/packages/orchestratorlib/test/orchestratorhelper.test.ts +++ b/packages/orchestratorlib/test/orchestratorhelper.test.ts @@ -425,6 +425,7 @@ describe('Test Suite - orchestratorhelper', () => { assert.ok((utteranceEntityLabelsMap.get(utterance) as Label[]).length === 3, `(utteranceEntityLabelsMap.get(utterance) as Label[]).length=${(utteranceEntityLabelsMap.get(utterance) as Label[]).length}`); }); + it('Test.0300 OrchestratorHelper.getJsonIntentEntityScoresUtterances()', function () { Utility.resetFlagToPrintDebuggingLogToConsole(UnitTestHelper.getDefaultUnitTestDebuggingLogFlag()); this.timeout(UnitTestHelper.getDefaultUnitTestTimeout()); @@ -587,6 +588,7 @@ describe('Test Suite - orchestratorhelper', () => { assert.ok(result.has('Add item'), 'Incorrect result from getUtteranceLabelsMap, missing Add item utterance'); assert.ok(result.has('delete to do go shopping'), 'Incorrect result from getUtteranceLabelsMap, missing delete to do go shopping utterance'); }); + it('Test.0600 OrchestratorHelper.parseQnaFile()', async () => { const validFile: string = './test/fixtures/parser/valid.qna'; const inValidFile: string = './test/fixtures/parser/invalid.qna'; @@ -608,7 +610,7 @@ describe('Test Suite - orchestratorhelper', () => { utteranceLabelDuplicateMap); assert.ok(utteranceLabelsMap.size > 0); }); - it('Test.0600 OrchestratorHelper.parseLuFile()', async () => { + it('Test.0601 OrchestratorHelper.parseLuFile()', async () => { const validFile: string = './test/fixtures/parser/valid.lu'; const inValidFile: string = './test/fixtures/parser/invalid.lu'; const utteranceLabelsMap: Map> = new Map>();