Skip to content
This repository was archived by the owner on Jan 15, 2025. It is now read-only.

Commit 20a3f98

Browse files
hcyanghcyang
authored
Remove None/Unknown consolidation (#1299)
* no None/Unknown consolidation * Remove silent error processing * create can read a default, non-existent snapshot BLU file * LUn parsing without LUIS validation
1 parent c1c0528 commit 20a3f98

File tree

3 files changed

+48
-39
lines changed

3 files changed

+48
-39
lines changed

‎packages/orchestratorlib/src/orchestratorhelper.ts

Lines changed: 27 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ import {Utility} from './utility';
1717
import {Utility as UtilityDispatcher} from '@microsoft/bf-dispatcher';
1818

1919
const ReadText: any = require('read-text-file');
20-
const LuisBuilder: any = require('@microsoft/bf-lu').V2.LuisBuilder;
20+
const luisCollateBuildNoValidate: any = require('@microsoft/bf-lu/lib/parser/luis/luisCollate').build;
2121
const QnaMakerBuilder: any = require('@microsoft/bf-lu').V2.QnAMakerBuilder;
2222
const processedFiles: string[] = [];
2323

@@ -41,23 +41,22 @@ export class OrchestratorHelper {
4141
fs.mkdirSync(path, {recursive: true});
4242
}
4343

44-
public static readBluSnapshotFile(filePath: string): string {
45-
return ReadText.readSync(filePath);
46-
// ---- NOTE
47-
// the code below was trying to normalize unknown labels in a BLU file,
48-
// but the unknown labels should have been processed during ingesting
49-
// a raw input file (LU, QnA, TSV, etc.) and before creating a BLU file,
50-
// so there is really no need to process unknown labels in a BLU file
51-
// anymore. The line below is thus deprecated, especially now the BLU
52-
// file can be a JSON, so the statement below does not apply anyway.
53-
// ---- return Utility.processUnknownSpuriousLabelsInTsvBluFileContent(ReadText.readSync(filePath));
54-
}
55-
5644
public static readFile(filePath: string): string {
45+
UtilityDispatcher.debuggingLog1(
46+
'OrchestratorHElper.readFile() calling ReadText.readSync()',
47+
filePath);
5748
try {
49+
const fileStats: fs.Stats = fs.statSync(filePath);
50+
if (fileStats.size === 0) {
51+
return '';
52+
}
5853
return ReadText.readSync(filePath);
59-
} catch {
60-
return '';
54+
} catch (error) {
55+
UtilityDispatcher.debuggingLog2(
56+
'EXCEPTION calling ReadText.readSync()',
57+
filePath,
58+
error);
59+
throw error;
6160
}
6261
}
6362

@@ -124,8 +123,11 @@ export class OrchestratorHelper {
124123
}
125124

126125
public static getSnapshotFromFile(snapshotPath: string) {
126+
UtilityDispatcher.debuggingLog1(
127+
'OrchestratorHelper.getSnapshotFromFile()',
128+
snapshotPath);
127129
if (Utility.exists(snapshotPath) && !OrchestratorHelper.isDirectory(snapshotPath)) {
128-
return new TextEncoder().encode(OrchestratorHelper.readBluSnapshotFile(snapshotPath));
130+
return new TextEncoder().encode(OrchestratorHelper.readFile(snapshotPath));
129131
}
130132
return new Uint8Array();
131133
}
@@ -160,7 +162,6 @@ export class OrchestratorHelper {
160162
utteranceEntityLabelDuplicateMap);
161163
}
162164
}
163-
Utility.processUnknownSpuriousLabelsInUtteranceLabelsMap({utteranceLabelsMap, utteranceLabelDuplicateMap});
164165
return {
165166
utteranceLabelsMap,
166167
utteranceLabelDuplicateMap,
@@ -232,7 +233,7 @@ export class OrchestratorHelper {
232233
}
233234

234235
public static async getEntitiesInLu(luObject: any): Promise<any> {
235-
const luisObject: any = await LuisBuilder.fromLUAsync([luObject], OrchestratorHelper.findLuFiles);
236+
const luisObject: any = await luisCollateBuildNoValidate([luObject], false, '', OrchestratorHelper.findLuFiles);
236237
return this.transformEntities(luisObject);
237238
}
238239

@@ -353,7 +354,7 @@ export class OrchestratorHelper {
353354
default: throw new Error(`Unknown file type ${ext}`);
354355
}
355356
} catch (error: any) {
356-
throw new Error(`${error.message}${os.EOL}Failed to parse ${filePath}`);
357+
throw new Error(`${error.message}${os.EOL}Failed to parse ${filePath}, error=${os.EOL}${UtilityDispatcher.jsonStringify(error)}`);
357358
}
358359
}
359360

@@ -418,6 +419,12 @@ export class OrchestratorHelper {
418419
utteranceLabelDuplicateMap: Map<string, Set<string>>,
419420
utteranceEntityLabelsMap: Map<string, Label[]>,
420421
utteranceEntityLabelDuplicateMap: Map<string, Label[]>): Promise<void> {
422+
UtilityDispatcher.debuggingLog1(
423+
'OrchestratorHelper.parseLuContent()',
424+
luFile);
425+
UtilityDispatcher.debuggingLog1(
426+
'OrchestratorHelper.parseLuContent()',
427+
luContent);
421428
if (!luContent || luContent.length === 0) {
422429
return;
423430
}
@@ -426,7 +433,7 @@ export class OrchestratorHelper {
426433
content: luContent,
427434
id: luFile,
428435
};
429-
const luisObject: any = await LuisBuilder.fromLUAsync([luObject], OrchestratorHelper.findLuFiles);
436+
const luisObject: any = await luisCollateBuildNoValidate([luObject], false, '', OrchestratorHelper.findLuFiles);
430437
if (Utility.toPrintDetailedDebuggingLogToConsole) {
431438
UtilityDispatcher.debuggingNamedLog1('OrchestratorHelper.parseLuContent(): calling getIntentsEntitiesUtterances()', luisObject, 'luisObject');
432439
}

‎packages/orchestratorlib/src/utilitylabelresolver.ts

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -30,14 +30,14 @@ export class UtilityLabelResolver {
3030
} = {
3131
ignore_same_example: ignoreSameExample,
3232
};
33-
Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), ignoreSameExample=${ignoreSameExample}`);
34-
Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), resetAll=${resetAll}`);
35-
Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), ignoreSameExampleObject=${ignoreSameExampleObject}`);
36-
Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), ignoreSameExampleObject.ignore_same_example=${ignoreSameExampleObject.ignore_same_example}`);
33+
Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), ignoreSameExample=${ignoreSameExample}`);
34+
Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), resetAll=${resetAll}`);
35+
Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), ignoreSameExampleObject=${ignoreSameExampleObject}`);
36+
Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), ignoreSameExampleObject.ignore_same_example=${ignoreSameExampleObject.ignore_same_example}`);
3737
const ignoreSameExampleObjectJson: string = Utility.jsonStringify(ignoreSameExampleObject);
38-
Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), ignoreSameExampleObjectJson=${ignoreSameExampleObjectJson}`);
38+
Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), ignoreSameExampleObjectJson=${ignoreSameExampleObjectJson}`);
3939
LabelResolver.setRuntimeParams(ignoreSameExampleObjectJson, resetAll);
40-
Utility.debuggingLog(`read to call Utility.getConfigJson(), LabelResolver.LabelResolver=${LabelResolver.LabelResolver}`);
40+
Utility.debuggingLog(`ready to call Utility.getConfigJson(), LabelResolver.LabelResolver=${LabelResolver.LabelResolver}`);
4141
return LabelResolver.getConfigJson();
4242
}
4343

@@ -49,14 +49,14 @@ export class UtilityLabelResolver {
4949
} = {
5050
use_compact_embeddings: !fullEmbeddings,
5151
};
52-
Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), fullEmbeddings=${fullEmbeddings}`);
53-
Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), resetAll=${resetAll}`);
54-
Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), useCompactEmbeddingsObject=${useCompactEmbeddingsObject}`);
55-
Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), useCompactEmbeddingsObject.use_compact_embeddings=${useCompactEmbeddingsObject.use_compact_embeddings}`);
52+
Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), fullEmbeddings=${fullEmbeddings}`);
53+
Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), resetAll=${resetAll}`);
54+
Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), useCompactEmbeddingsObject=${useCompactEmbeddingsObject}`);
55+
Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), useCompactEmbeddingsObject.use_compact_embeddings=${useCompactEmbeddingsObject.use_compact_embeddings}`);
5656
const useCompactEmbeddingsObjectJson: string = Utility.jsonStringify(useCompactEmbeddingsObject);
57-
Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), useCompactEmbeddingsObjectJson=${useCompactEmbeddingsObjectJson}`);
57+
Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), useCompactEmbeddingsObjectJson=${useCompactEmbeddingsObjectJson}`);
5858
LabelResolver.setRuntimeParams(useCompactEmbeddingsObjectJson, resetAll);
59-
Utility.debuggingLog(`read to call Utility.getConfigJson(), LabelResolver.LabelResolver=${LabelResolver.LabelResolver}`);
59+
Utility.debuggingLog(`ready to call Utility.getConfigJson(), LabelResolver.LabelResolver=${LabelResolver.LabelResolver}`);
6060
return LabelResolver.getConfigJson();
6161
}
6262

@@ -68,14 +68,14 @@ export class UtilityLabelResolver {
6868
} = {
6969
knn_k: knnK,
7070
};
71-
Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), knnK=${knnK}`);
72-
Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), resetAll=${resetAll}`);
73-
Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), knnKObject=${knnKObject}`);
74-
Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), knnKObject.knn_k=${knnKObject.knn_k}`);
71+
Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), knnK=${knnK}`);
72+
Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), resetAll=${resetAll}`);
73+
Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), knnKObject=${knnKObject}`);
74+
Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), knnKObject.knn_k=${knnKObject.knn_k}`);
7575
const knnKObjectJson: string = Utility.jsonStringify(knnKObject);
76-
Utility.debuggingLog(`read to call LabelResolver.setRuntimeParams(), knnKObjectJson=${knnKObjectJson}`);
76+
Utility.debuggingLog(`ready to call LabelResolver.setRuntimeParams(), knnKObjectJson=${knnKObjectJson}`);
7777
LabelResolver.setRuntimeParams(knnKObjectJson, resetAll);
78-
Utility.debuggingLog(`read to call Utility.getConfigJson(), LabelResolver.LabelResolver=${LabelResolver.LabelResolver}`);
78+
Utility.debuggingLog(`ready to call Utility.getConfigJson(), LabelResolver.LabelResolver=${LabelResolver.LabelResolver}`);
7979
return LabelResolver.getConfigJson();
8080
}
8181

‎packages/orchestratorlib/test/orchestratorhelper.test.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -425,6 +425,7 @@ describe('Test Suite - orchestratorhelper', () => {
425425
assert.ok((utteranceEntityLabelsMap.get(utterance) as Label[]).length === 3,
426426
`(utteranceEntityLabelsMap.get(utterance) as Label[]).length=${(utteranceEntityLabelsMap.get(utterance) as Label[]).length}`);
427427
});
428+
428429
it('Test.0300 OrchestratorHelper.getJsonIntentEntityScoresUtterances()', function () {
429430
Utility.resetFlagToPrintDebuggingLogToConsole(UnitTestHelper.getDefaultUnitTestDebuggingLogFlag());
430431
this.timeout(UnitTestHelper.getDefaultUnitTestTimeout());
@@ -587,6 +588,7 @@ describe('Test Suite - orchestratorhelper', () => {
587588
assert.ok(result.has('Add item'), 'Incorrect result from getUtteranceLabelsMap, missing Add item utterance');
588589
assert.ok(result.has('delete to do go shopping'), 'Incorrect result from getUtteranceLabelsMap, missing delete to do go shopping utterance');
589590
});
591+
590592
it('Test.0600 OrchestratorHelper.parseQnaFile()', async () => {
591593
const validFile: string = './test/fixtures/parser/valid.qna';
592594
const inValidFile: string = './test/fixtures/parser/invalid.qna';
@@ -608,7 +610,7 @@ describe('Test Suite - orchestratorhelper', () => {
608610
utteranceLabelDuplicateMap);
609611
assert.ok(utteranceLabelsMap.size > 0);
610612
});
611-
it('Test.0600 OrchestratorHelper.parseLuFile()', async () => {
613+
it('Test.0601 OrchestratorHelper.parseLuFile()', async () => {
612614
const validFile: string = './test/fixtures/parser/valid.lu';
613615
const inValidFile: string = './test/fixtures/parser/invalid.lu';
614616
const utteranceLabelsMap: Map<string, Set<string>> = new Map<string, Set<string>>();

0 commit comments

Comments
 (0)