Update unit and integ tests

q-andy · q-andy · commit 7711e98ccd97 · 2025-05-13T10:38:47.000-07:00
Signed-off-by: Andy Qin &lt;qinandy@amazon.com&gt;
diff --git a/src/main/java/org/opensearch/neuralsearch/processor/TextChunkingProcessor.java b/src/main/java/org/opensearch/neuralsearch/processor/TextChunkingProcessor.java
@@ -195,7 +195,6 @@ public IngestDocument execute(final IngestDocument ingestDocument) {
         runtimeParameters.put(MAX_CHUNK_LIMIT_FIELD, maxChunkLimit);
         runtimeParameters.put(CHUNK_STRING_COUNT_FIELD, chunkStringCount);
         chunkMapType(sourceAndMetadataMap, fieldMap, runtimeParameters);
-
         recordChunkingExecutionStats(chunker.getAlgorithmName());
         return ingestDocument;
     }
diff --git a/src/main/java/org/opensearch/neuralsearch/stats/info/InfoStatsManager.java b/src/main/java/org/opensearch/neuralsearch/stats/info/InfoStatsManager.java
@@ -143,8 +143,8 @@ private void addIngestProcessorStats(Map<InfoStatName, CountableInfoStatSnapshot
 
     /**
      * Counts text chunking processor stats based on processor config
-     * @param stats
-     * @param processorConfig
+     * @param stats map containing the stat to increment
+     * @param processorConfig map of the processor config, parsed to add stats
      */
     private void countTextChunkingProcessorStats(Map<InfoStatName, CountableInfoStatSnapshot> stats, Map<String, Object> processorConfig) {
         increment(stats, InfoStatName.TEXT_CHUNKING_PROCESSORS);
@@ -157,6 +157,8 @@ private void countTextChunkingProcessorStats(Map<InfoStatName, CountableInfoStat
         switch (algorithmKey) {
             case DelimiterChunker.ALGORITHM_NAME -> increment(stats, InfoStatName.TEXT_CHUNKING_DELIMITER_PROCESSORS);
             case FixedTokenLengthChunker.ALGORITHM_NAME -> increment(stats, InfoStatName.TEXT_CHUNKING_FIXED_LENGTH_PROCESSORS);
+            // If no algorithm is specified, the default is fixed length
+            default -> increment(stats, InfoStatName.TEXT_CHUNKING_FIXED_LENGTH_PROCESSORS);
         }
     }
 
diff --git a/src/test/java/org/opensearch/neuralsearch/processor/TextChunkingProcessorIT.java b/src/test/java/org/opensearch/neuralsearch/processor/TextChunkingProcessorIT.java
@@ -186,11 +186,22 @@ public void testTextChunkingProcessor_processorStats() {
         ingestDocument(INDEX_NAME, document);
         ingestDocument(INDEX_NAME, document);
 
+        List<String> expectedPassages = new ArrayList<>();
+        expectedPassages.add("This is an example document to be chunked. The document ");
+        expectedPassages.add("contains a single paragraph, two sentences and 24 tokens by ");
+        expectedPassages.add("standard tokenizer in OpenSearch.");
+        validateIndexIngestResultsWithMultipleDocs(INDEX_NAME, OUTPUT_FIELD, expectedPassages, 2);
+
         ingestDocument(INDEX_NAME2, document);
         ingestDocument(INDEX_NAME2, document);
         ingestDocument(INDEX_NAME2, document);
 
-        // Get stats request
+        expectedPassages = new ArrayList<>();
+        expectedPassages.add("This is an example document to be chunked.");
+        expectedPassages.add(" The document contains a single paragraph, two sentences and 24 tokens by standard tokenizer in OpenSearch.");
+        validateIndexIngestResultsWithMultipleDocs(INDEX_NAME2, OUTPUT_FIELD, expectedPassages, 3);
+
+        // Get stats
         String responseBody = executeNeuralStatRequest(new ArrayList<>(), new ArrayList<>());
         Map<String, Object> stats = parseInfoStatsResponse(responseBody);
         Map<String, Object> allNodesStats = parseAggregatedNodeStatsResponse(responseBody);
@@ -208,8 +219,8 @@ public void testTextChunkingProcessor_processorStats() {
         updateClusterSettings("plugins.neural_search.stats_enabled", false);
     }
 
-    private void validateIndexIngestResults(String indexName, String fieldName, Object expected) {
-        assertEquals(1, getDocCount(indexName));
+    private void validateIndexIngestResultsWithMultipleDocs(String indexName, String fieldName, Object expected, int docCount) {
+        assertEquals(docCount, getDocCount(indexName));
         MatchAllQueryBuilder query = new MatchAllQueryBuilder();
         Map<String, Object> searchResults = search(indexName, query, 10);
         assertNotNull(searchResults);
@@ -224,6 +235,10 @@ private void validateIndexIngestResults(String indexName, String fieldName, Obje
         assertEquals(expected, ingestOutputs);
     }
 
+    private void validateIndexIngestResults(String indexName, String fieldName, Object expected) {
+        validateIndexIngestResultsWithMultipleDocs(indexName, fieldName, expected, 1);
+    }
+
     private void createPipelineProcessor(String pipelineName) throws Exception {
         URL pipelineURLPath = classLoader.getResource(PIPELINE_CONFIGS_BY_NAME.get(pipelineName));
         Objects.requireNonNull(pipelineURLPath);
diff --git a/src/test/java/org/opensearch/neuralsearch/processor/TextChunkingProcessorTests.java b/src/test/java/org/opensearch/neuralsearch/processor/TextChunkingProcessorTests.java
@@ -9,6 +9,7 @@
 import org.apache.lucene.tests.analysis.MockTokenizer;
 import org.junit.Before;
 import java.util.ArrayList;
+import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
@@ -38,7 +39,9 @@
 import org.opensearch.neuralsearch.processor.chunker.FixedTokenLengthChunker;
 import org.opensearch.neuralsearch.processor.factory.TextChunkingProcessorFactory;
 import org.opensearch.neuralsearch.settings.NeuralSearchSettingsAccessor;
+import org.opensearch.neuralsearch.stats.events.EventStatName;
 import org.opensearch.neuralsearch.stats.events.EventStatsManager;
+import org.opensearch.neuralsearch.stats.events.TimestampedEventStatSnapshot;
 import org.opensearch.plugins.AnalysisPlugin;
 import org.opensearch.test.OpenSearchTestCase;
 import static org.opensearch.neuralsearch.processor.TextChunkingProcessor.TYPE;
@@ -95,6 +98,7 @@ public void setup() {
         when(clusterService.state()).thenReturn(clusterState);
         textChunkingProcessorFactory = new TextChunkingProcessorFactory(environment, clusterService, getAnalysisRegistry());
 
+        EventStatsManager.instance().reset();
         NeuralSearchSettingsAccessor settingsAccessor = mock(NeuralSearchSettingsAccessor.class);
         when(settingsAccessor.isStatsEnabled()).thenReturn(true);
         EventStatsManager.instance().initialize(settingsAccessor);
@@ -978,4 +982,70 @@ public void testExecute_withIgnoreMissing_thenSucceed() {
         IngestDocument document = processor.execute(ingestDocument);
         assertFalse(document.getSourceAndMetadata().containsKey(OUTPUT_FIELD));
     }
+
+    @SneakyThrows
+    public void testExecute_statsDisabled_thenSucceed() {
+        NeuralSearchSettingsAccessor settingsAccessor = mock(NeuralSearchSettingsAccessor.class);
+        when(settingsAccessor.isStatsEnabled()).thenReturn(false);
+        EventStatsManager.instance().initialize(settingsAccessor);
+
+        TextChunkingProcessor processor = createFixedTokenLengthInstance(createStringFieldMap());
+        IngestDocument ingestDocument = createIngestDocumentWithSourceData(createSourceDataString());
+        IngestDocument document = processor.execute(ingestDocument);
+        assert document.getSourceAndMetadata().containsKey(OUTPUT_FIELD);
+        Object passages = document.getSourceAndMetadata().get(OUTPUT_FIELD);
+        assert (passages instanceof List<?>);
+        List<String> expectedPassages = new ArrayList<>();
+        expectedPassages.add("This is an example document to be chunked. The document ");
+        expectedPassages.add("contains a single paragraph, two sentences and 24 tokens by ");
+        expectedPassages.add("standard tokenizer in OpenSearch.");
+        assertEquals(expectedPassages, passages);
+
+        Map<EventStatName, TimestampedEventStatSnapshot> snapshots = EventStatsManager.instance()
+            .getTimestampedEventStatSnapshots(EnumSet.allOf(EventStatName.class));
+
+        assertEquals(0L, snapshots.get(EventStatName.TEXT_CHUNKING_PROCESSOR_EXECUTIONS).getValue().longValue());
+        assertEquals(0L, snapshots.get(EventStatName.TEXT_CHUNKING_FIXED_LENGTH_EXECUTIONS).getValue().longValue());
+    }
+
+    @SneakyThrows
+    public void testExecute_statsEnabled_withFixedTokenLength_andSourceDataString_thenSucceed() {
+        TextChunkingProcessor processor = createFixedTokenLengthInstance(createStringFieldMap());
+        IngestDocument ingestDocument = createIngestDocumentWithSourceData(createSourceDataString());
+        IngestDocument document = processor.execute(ingestDocument);
+        assert document.getSourceAndMetadata().containsKey(OUTPUT_FIELD);
+        Object passages = document.getSourceAndMetadata().get(OUTPUT_FIELD);
+        assert (passages instanceof List<?>);
+        List<String> expectedPassages = new ArrayList<>();
+        expectedPassages.add("This is an example document to be chunked. The document ");
+        expectedPassages.add("contains a single paragraph, two sentences and 24 tokens by ");
+        expectedPassages.add("standard tokenizer in OpenSearch.");
+        assertEquals(expectedPassages, passages);
+
+        Map<EventStatName, TimestampedEventStatSnapshot> snapshots = EventStatsManager.instance()
+            .getTimestampedEventStatSnapshots(EnumSet.allOf(EventStatName.class));
+
+        assertEquals(1L, snapshots.get(EventStatName.TEXT_CHUNKING_PROCESSOR_EXECUTIONS).getValue().longValue());
+        assertEquals(1L, snapshots.get(EventStatName.TEXT_CHUNKING_FIXED_LENGTH_EXECUTIONS).getValue().longValue());
+    }
+
+    @SneakyThrows
+    public void testExecute_statsEnabled_withDelimiter_andSourceDataString_thenSucceed() {
+        TextChunkingProcessor processor = createDelimiterInstance();
+        IngestDocument ingestDocument = createIngestDocumentWithSourceData(createSourceDataString());
+        IngestDocument document = processor.execute(ingestDocument);
+        assert document.getSourceAndMetadata().containsKey(OUTPUT_FIELD);
+        Object passages = document.getSourceAndMetadata().get(OUTPUT_FIELD);
+        assert (passages instanceof List<?>);
+        List<String> expectedPassages = new ArrayList<>();
+        expectedPassages.add("This is an example document to be chunked.");
+        expectedPassages.add(" The document contains a single paragraph, two sentences and 24 tokens by standard tokenizer in OpenSearch.");
+        assertEquals(expectedPassages, passages);
+
+        Map<EventStatName, TimestampedEventStatSnapshot> snapshots = EventStatsManager.instance()
+            .getTimestampedEventStatSnapshots(EnumSet.allOf(EventStatName.class));
+
+        assertEquals(1L, snapshots.get(EventStatName.TEXT_CHUNKING_PROCESSOR_EXECUTIONS).getValue().longValue());
+        assertEquals(1L, snapshots.get(EventStatName.TEXT_CHUNKING_DELIMITER_EXECUTIONS).getValue().longValue());
+    }
 }

Original file line number	Diff line number	Diff line change
`@@ -195,7 +195,6 @@ public IngestDocument execute(final IngestDocument ingestDocument) {`
`195`	`195`	`runtimeParameters.put(MAX_CHUNK_LIMIT_FIELD, maxChunkLimit);`
`196`	`196`	`runtimeParameters.put(CHUNK_STRING_COUNT_FIELD, chunkStringCount);`
`197`	`197`	`chunkMapType(sourceAndMetadataMap, fieldMap, runtimeParameters);`
`198`		`-`
`199`	`198`	`recordChunkingExecutionStats(chunker.getAlgorithmName());`
`200`	`199`	`return ingestDocument;`
`201`	`200`	`}`