add validation for invalid nested hybrid query

Signed-off-by: Will Hwang <[email protected]>

Author: will-hwang

src/main/java/org/opensearch/neuralsearch/query/HybridQueryBuilder.java

@@ -225,7 +225,14 @@ public static HybridQueryBuilder fromXContent(XContentParser parser) throws IOEx
                                 String.format(Locale.ROOT, "Number of sub-queries exceeds maximum supported by [%s] query", NAME)
                             );
                         }
-                        queries.add(parseInnerQueryBuilder(parser));
+                        QueryBuilder query = parseInnerQueryBuilder(parser);
+                        if (query.getName().equals(NAME)) {
+                            throw new ParsingException(
+                                parser.getTokenLocation(),
+                                String.format(Locale.ROOT, "[%s] query cannot be nested in another [%s] query", NAME, NAME)
+                            );
+                        }
+                        queries.add(query);
                         token = parser.nextToken();
                     }
                 } else {

src/test/java/org/opensearch/neuralsearch/query/HybridQueryBuilderTests.java

@@ -354,6 +354,68 @@ public void testDoToQuery_whenTooManySubqueries_thenFail() {
         assertThat(exception.getMessage(), containsString("Number of sub-queries exceeds maximum supported"));
     }
 
+    @SneakyThrows
+    public void testDoToQuery_whenNestedHybridSubqueries_thenFail() {
+        // create query with nested hybrid subqueries, which should not be allowed
+        XContentBuilder xContentBuilder = XContentFactory.jsonBuilder()
+            .startObject()
+            .startArray("queries")
+            .startObject()
+            .startObject(HybridQueryBuilder.NAME)
+            .startArray("queries")
+            .startObject()
+            .startObject(NeuralQueryBuilder.NAME)
+            .startObject(VECTOR_FIELD_NAME)
+            .field(QUERY_TEXT_FIELD.getPreferredName(), QUERY_TEXT)
+            .field(MODEL_ID_FIELD.getPreferredName(), MODEL_ID)
+            .field(K_FIELD.getPreferredName(), K)
+            .field(BOOST_FIELD.getPreferredName(), BOOST)
+            .endObject()
+            .endObject()
+            .endObject()
+            .startObject()
+            .startObject(TermQueryBuilder.NAME)
+            .field(TEXT_FIELD_NAME, TERM_QUERY_TEXT)
+            .endObject()
+            .endObject()
+            .endArray()
+            .endObject()
+            .endObject()
+            .startObject()
+            .startObject(TermQueryBuilder.NAME)
+            .field(TEXT_FIELD_NAME, TERM_QUERY_TEXT)
+            .endObject()
+            .endObject()
+            .endArray()
+            .field("pagination_depth", 10)
+            .endObject();
+
+        NamedXContentRegistry namedXContentRegistry = new NamedXContentRegistry(
+            List.of(
+                new NamedXContentRegistry.Entry(QueryBuilder.class, new ParseField(TermQueryBuilder.NAME), TermQueryBuilder::fromXContent),
+                new NamedXContentRegistry.Entry(
+                    QueryBuilder.class,
+                    new ParseField(NeuralQueryBuilder.NAME),
+                    NeuralQueryBuilder::fromXContent
+                ),
+                new NamedXContentRegistry.Entry(
+                    QueryBuilder.class,
+                    new ParseField(HybridQueryBuilder.NAME),
+                    HybridQueryBuilder::fromXContent
+                )
+            )
+        );
+        XContentParser contentParser = createParser(
+            namedXContentRegistry,
+            xContentBuilder.contentType().xContent(),
+            BytesReference.bytes(xContentBuilder)
+        );
+        contentParser.nextToken();
+
+        ParsingException exception = expectThrows(ParsingException.class, () -> HybridQueryBuilder.fromXContent(contentParser));
+        assertThat(exception.getMessage(), containsString("[hybrid] query cannot be nested in another [hybrid] query"));
+    }
+
     /**
      * Tests basic query:
      * {