[TEST] Add integration test codes for fixed_char_length chunking algorithm

Signed-off-by: yeonghyeonKo <[email protected]>

Author: YeonghyeonKO

src/test/java/org/opensearch/neuralsearch/processor/TextChunkingProcessorIT.java

@@ -39,6 +39,8 @@ public class TextChunkingProcessorIT extends BaseNeuralSearchIT {
 
     private static final String DELIMITER_PIPELINE_NAME = "pipeline-text-chunking-delimiter";
 
+    private static final String FIXED_CHAR_LENGTH_PIPELINE_NAME = "pipeline-text-chunking-fixed-char-length";
+
     private static final String CASCADE_PIPELINE_NAME = "pipeline-text-chunking-cascade";
 
     private static final String TEST_DOCUMENT = "processor/chunker/TextChunkingTestDocument.json";
@@ -54,6 +56,8 @@ public class TextChunkingProcessorIT extends BaseNeuralSearchIT {
         "processor/chunker/PipelineForFixedTokenLengthChunkerWithLowercaseTokenizer.json",
         DELIMITER_PIPELINE_NAME,
         "processor/chunker/PipelineForDelimiterChunker.json",
+        FIXED_CHAR_LENGTH_PIPELINE_NAME,
+        "processor/chunker/PipelineForFixedCharLengthChunker.json",
         CASCADE_PIPELINE_NAME,
         "processor/chunker/PipelineForCascadedChunker.json"
     );
@@ -137,6 +141,22 @@ public void testTextChunkingProcessor_withDelimiterAlgorithm_successful() {
         validateIndexIngestResults(INDEX_NAME, OUTPUT_FIELD, expectedPassages);
     }
 
+    @SneakyThrows
+    public void testTextChunkingProcessor_withFixedCharLengthAlgorithm_thenSucceed() {
+        createPipelineProcessor(FIXED_CHAR_LENGTH_PIPELINE_NAME);
+        createTextChunkingIndex(INDEX_NAME, FIXED_CHAR_LENGTH_PIPELINE_NAME);
+
+        String document = getDocumentFromFilePath(TEST_DOCUMENT);
+        ingestDocument(INDEX_NAME, document);
+
+        List<String> expectedPassages = new ArrayList<>();
+        expectedPassages.add("This is an example document to be chunked. The doc");
+        expectedPassages.add("d. The document contains a single paragraph, two s");
+        expectedPassages.add("aph, two sentences and 24 tokens by standard token");
+        expectedPassages.add("dard tokenizer in OpenSearch.");
+        validateIndexIngestResults(INDEX_NAME, OUTPUT_FIELD, expectedPassages);
+    }
+
     @SneakyThrows
     public void testTextChunkingProcessor_withCascadePipeline_successful() {
         createPipelineProcessor(CASCADE_PIPELINE_NAME);
@@ -209,11 +229,11 @@ public void testTextChunkingProcessor_processorStats_successful() {
         // Parse json to get stats
         assertEquals(5, getNestedValue(allNodesStats, EventStatName.TEXT_CHUNKING_PROCESSOR_EXECUTIONS));
         assertEquals(3, getNestedValue(allNodesStats, EventStatName.TEXT_CHUNKING_DELIMITER_EXECUTIONS));
-        assertEquals(2, getNestedValue(allNodesStats, EventStatName.TEXT_CHUNKING_FIXED_LENGTH_EXECUTIONS));
+        assertEquals(2, getNestedValue(allNodesStats, EventStatName.TEXT_CHUNKING_FIXED_TOKEN_LENGTH_EXECUTIONS));
 
         assertEquals(3, getNestedValue(stats, InfoStatName.TEXT_CHUNKING_PROCESSORS));
         assertEquals(1, getNestedValue(stats, InfoStatName.TEXT_CHUNKING_DELIMITER_PROCESSORS));
-        assertEquals(2, getNestedValue(stats, InfoStatName.TEXT_CHUNKING_FIXED_LENGTH_PROCESSORS));
+        assertEquals(2, getNestedValue(stats, InfoStatName.TEXT_CHUNKING_FIXED_TOKEN_LENGTH_PROCESSORS));
 
         // Reset stats
         updateClusterSettings("plugins.neural_search.stats_enabled", false);

src/test/java/org/opensearch/neuralsearch/processor/TextChunkingProcessorTests.java

@@ -36,6 +36,7 @@
 import org.opensearch.ingest.IngestDocument;
 import org.opensearch.ingest.Processor;
 import org.opensearch.neuralsearch.processor.chunker.DelimiterChunker;
+import org.opensearch.neuralsearch.processor.chunker.FixedCharLengthChunker;
 import org.opensearch.neuralsearch.processor.chunker.FixedTokenLengthChunker;
 import org.opensearch.neuralsearch.processor.factory.TextChunkingProcessorFactory;
 import org.opensearch.neuralsearch.settings.NeuralSearchSettingsAccessor;
@@ -127,6 +128,13 @@ private Map<String, Object> createDelimiterParameters() {
         return parameters;
     }
 
+    private Map<String, Object> createFixedCharLengthParameters() {
+        Map<String, Object> parameters = new HashMap<>();
+        parameters.put(FixedCharLengthChunker.CHAR_LIMIT_FIELD, 50);
+        parameters.put(FixedCharLengthChunker.OVERLAP_RATE_FIELD, 0.2);
+        return parameters;
+    }
+
     private Map<String, Object> createStringFieldMap() {
         Map<String, Object> fieldMap = new HashMap<>();
         fieldMap.put(INPUT_FIELD, OUTPUT_FIELD);
@@ -190,6 +198,17 @@ private TextChunkingProcessor createDelimiterInstance() {
         return textChunkingProcessorFactory.create(registry, PROCESSOR_TAG, DESCRIPTION, config);
     }
 
+    @SneakyThrows
+    private TextChunkingProcessor createFixedCharLengthInstance(Map<String, Object> fieldMap) {
+        Map<String, Object> config = new HashMap<>();
+        Map<String, Object> algorithmMap = new HashMap<>();
+        algorithmMap.put(FixedCharLengthChunker.ALGORITHM_NAME, createFixedCharLengthParameters());
+        config.put(FIELD_MAP_FIELD, fieldMap);
+        config.put(ALGORITHM_FIELD, algorithmMap);
+        Map<String, Processor.Factory> registry = new HashMap<>();
+        return textChunkingProcessorFactory.create(registry, PROCESSOR_TAG, DESCRIPTION, config);
+    }
+
     @SneakyThrows
     private TextChunkingProcessor createIgnoreMissingInstance() {
         Map<String, Object> config = new HashMap<>();
@@ -1003,7 +1022,7 @@ public void testExecute_statsDisabled_thenSucceed() {
             .getTimestampedEventStatSnapshots(EnumSet.allOf(EventStatName.class));
 
         assertEquals(0L, snapshots.get(EventStatName.TEXT_CHUNKING_PROCESSOR_EXECUTIONS).getValue().longValue());
-        assertEquals(0L, snapshots.get(EventStatName.TEXT_CHUNKING_FIXED_LENGTH_EXECUTIONS).getValue().longValue());
+        assertEquals(0L, snapshots.get(EventStatName.TEXT_CHUNKING_FIXED_TOKEN_LENGTH_EXECUTIONS).getValue().longValue());
     }
 
     @SneakyThrows
@@ -1024,7 +1043,7 @@ public void testExecute_statsEnabled_withFixedTokenLength_andSourceDataString_th
             .getTimestampedEventStatSnapshots(EnumSet.allOf(EventStatName.class));
 
         assertEquals(1L, snapshots.get(EventStatName.TEXT_CHUNKING_PROCESSOR_EXECUTIONS).getValue().longValue());
-        assertEquals(1L, snapshots.get(EventStatName.TEXT_CHUNKING_FIXED_LENGTH_EXECUTIONS).getValue().longValue());
+        assertEquals(1L, snapshots.get(EventStatName.TEXT_CHUNKING_FIXED_TOKEN_LENGTH_EXECUTIONS).getValue().longValue());
     }
 
     @SneakyThrows
@@ -1046,4 +1065,26 @@ public void testExecute_statsEnabled_withDelimiter_andSourceDataString_thenSucce
         assertEquals(1L, snapshots.get(EventStatName.TEXT_CHUNKING_PROCESSOR_EXECUTIONS).getValue().longValue());
         assertEquals(1L, snapshots.get(EventStatName.TEXT_CHUNKING_DELIMITER_EXECUTIONS).getValue().longValue());
     }
+
+    @SneakyThrows
+    public void testExecute_statsEnabled_withFixedCharLength_andSourceDataString_thenSucceed() {
+        TextChunkingProcessor processor = createFixedCharLengthInstance(createStringFieldMap());
+        IngestDocument ingestDocument = createIngestDocumentWithSourceData(createSourceDataString());
+        IngestDocument document = processor.execute(ingestDocument);
+        assert document.getSourceAndMetadata().containsKey(OUTPUT_FIELD);
+        Object passages = document.getSourceAndMetadata().get(OUTPUT_FIELD);
+        assert (passages instanceof List<?>);
+        List<String> expectedPassages = new ArrayList<>();
+        expectedPassages.add("This is an example document to be chunked. The doc");
+        expectedPassages.add("d. The document contains a single paragraph, two s");
+        expectedPassages.add("aph, two sentences and 24 tokens by standard token");
+        expectedPassages.add("dard tokenizer in OpenSearch.");
+        assertEquals(expectedPassages, passages);
+
+        Map<EventStatName, TimestampedEventStatSnapshot> snapshots = EventStatsManager.instance()
+            .getTimestampedEventStatSnapshots(EnumSet.allOf(EventStatName.class));
+
+        assertEquals(1L, snapshots.get(EventStatName.TEXT_CHUNKING_PROCESSOR_EXECUTIONS).getValue().longValue());
+        assertEquals(1L, snapshots.get(EventStatName.TEXT_CHUNKING_FIXED_CHAR_LENGTH_EXECUTIONS).getValue().longValue());
+    }
 }

src/test/java/org/opensearch/neuralsearch/processor/chunker/ChunkerFactoryTests.java

@@ -30,6 +30,12 @@ public void testCreate_Delimiter() {
         assert (chunker instanceof DelimiterChunker);
     }
 
+    public void testCreate_FixedCharLength() {
+        Chunker chunker = ChunkerFactory.create(FixedCharLengthChunker.ALGORITHM_NAME, createChunkParameters());
+        assertNotNull(chunker);
+        assert (chunker instanceof FixedCharLengthChunker);
+    }
+
     public void testCreate_Invalid() {
         String invalidChunkerName = "Invalid Chunker Algorithm";
         assertThrows(NullPointerException.class, () -> ChunkerFactory.create(invalidChunkerName, createChunkParameters()));

src/test/resources/processor/chunker/PipelineForFixedCharLengthChunker.json

@@ -0,0 +1,18 @@
+{
+  "description": "An example fixed character length chunker pipeline with overlap_rate",
+  "processors" : [
+    {
+      "text_chunking": {
+        "field_map": {
+          "body": "body_chunk"
+        },
+        "algorithm": {
+          "fixed_char_length": {
+            "char_limit": 50,
+            "overlap_rate": 0.2
+          }
+        }
+      }
+    }
+  ]
+}