Skip to content

Commit 8270b8d

Browse files
gaobinlonggaobinlong
committed
Merge remote-tracking branch 'upstream/main' into fix_url
2 parents 7974120 + 7673a77 commit 8270b8d

File tree

40 files changed

+3392
-1511
lines changed

40 files changed

+3392
-1511
lines changed

‎.github/workflows/add-performance-comment.yml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,10 @@ on:
66

77
jobs:
88
add-comment:
9-
if: github.event.label.name == 'Performance'
9+
if: |
10+
github.event.label.name == 'Performance' ||
11+
github.event.label.name == 'Search:Performance' ||
12+
github.event.label.name == 'Indexing:Performance'
1013
runs-on: ubuntu-latest
1114
permissions:
1215
pull-requests: write

‎.github/workflows/benchmark-pull-request.yml

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -77,18 +77,6 @@ jobs:
7777
run: |
7878
echo "Invalid comment format detected. Failing the workflow."
7979
exit 1
80-
- id: get_approvers
81-
run: |
82-
echo "approvers=$(cat .github/CODEOWNERS | grep '^\*' | tr -d '* ' | sed 's/@/,/g' | sed 's/,//1')" >> $GITHUB_OUTPUT
83-
- uses: trstringer/manual-approval@v1
84-
if: (!contains(steps.get_approvers.outputs.approvers, github.event.comment.user.login))
85-
with:
86-
secret: ${{ github.TOKEN }}
87-
approvers: ${{ steps.get_approvers.outputs.approvers }}
88-
minimum-approvals: 1
89-
issue-title: 'Request to approve/deny benchmark run for PR #${{ env.PR_NUMBER }}'
90-
issue-body: "Please approve or deny the benchmark run for PR #${{ env.PR_NUMBER }}"
91-
exclude-workflow-initiator-as-approver: false
9280
- name: Get PR Details
9381
id: get_pr
9482
uses: actions/github-script@v7
@@ -106,21 +94,33 @@ jobs:
10694
10795
return {
10896
"headRepoFullName": pull_request.head.repo.full_name,
109-
"headRef": pull_request.head.ref
97+
"headRefSha": pull_request.head.sha
11098
};
11199
- name: Set pr details env vars
112100
run: |
113101
echo '${{ steps.get_pr.outputs.result }}' | jq -r '.headRepoFullName'
114-
echo '${{ steps.get_pr.outputs.result }}' | jq -r '.headRef'
102+
echo '${{ steps.get_pr.outputs.result }}' | jq -r '.headRefSha'
115103
headRepo=$(echo '${{ steps.get_pr.outputs.result }}' | jq -r '.headRepoFullName')
116-
headRef=$(echo '${{ steps.get_pr.outputs.result }}' | jq -r '.headRef')
104+
headRefSha=$(echo '${{ steps.get_pr.outputs.result }}' | jq -r '.headRefSha')
117105
echo "prHeadRepo=$headRepo" >> $GITHUB_ENV
118-
echo "prHeadRef=$headRef" >> $GITHUB_ENV
106+
echo "prHeadRefSha=$headRefSha" >> $GITHUB_ENV
107+
- id: get_approvers
108+
run: |
109+
echo "approvers=$(cat .github/CODEOWNERS | grep '^\*' | tr -d '* ' | sed 's/@/,/g' | sed 's/,//1')" >> $GITHUB_OUTPUT
110+
- uses: trstringer/manual-approval@v1
111+
if: (!contains(steps.get_approvers.outputs.approvers, github.event.comment.user.login))
112+
with:
113+
secret: ${{ github.TOKEN }}
114+
approvers: ${{ steps.get_approvers.outputs.approvers }}
115+
minimum-approvals: 1
116+
issue-title: 'Request to approve/deny benchmark run for PR #${{ env.PR_NUMBER }}'
117+
issue-body: "Please approve or deny the benchmark run for PR #${{ env.PR_NUMBER }}"
118+
exclude-workflow-initiator-as-approver: false
119119
- name: Checkout PR Repo
120120
uses: actions/checkout@v4
121121
with:
122122
repository: ${{ env.prHeadRepo }}
123-
ref: ${{ env.prHeadRef }}
123+
ref: ${{ env.prHeadRefSha }}
124124
token: ${{ secrets.GITHUB_TOKEN }}
125125
- name: Setup Java
126126
uses: actions/setup-java@v1

‎server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsNodeResponse.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -77,7 +77,7 @@ public ClusterStatsNodeResponse(StreamInput in) throws IOException {
7777
}
7878
this.nodeInfo = new NodeInfo(in);
7979
this.nodeStats = new NodeStats(in);
80-
if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
80+
if (in.getVersion().onOrAfter(Version.V_2_16_0)) {
8181
this.shardsStats = in.readOptionalArray(ShardStats::new, ShardStats[]::new);
8282
this.aggregatedNodeLevelStats = in.readOptionalWriteable(AggregatedNodeLevelStats::new);
8383
} else {
@@ -156,7 +156,7 @@ public void writeTo(StreamOutput out) throws IOException {
156156
}
157157
nodeInfo.writeTo(out);
158158
nodeStats.writeTo(out);
159-
if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
159+
if (out.getVersion().onOrAfter(Version.V_2_16_0)) {
160160
if (aggregatedNodeLevelStats != null) {
161161
out.writeOptionalArray(null);
162162
out.writeOptionalWriteable(aggregatedNodeLevelStats);

‎server/src/main/java/org/opensearch/action/admin/cluster/stats/ClusterStatsRequest.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ public class ClusterStatsRequest extends BaseNodesRequest<ClusterStatsRequest> {
5050

5151
public ClusterStatsRequest(StreamInput in) throws IOException {
5252
super(in);
53-
if (in.getVersion().onOrAfter(Version.V_3_0_0)) {
53+
if (in.getVersion().onOrAfter(Version.V_2_16_0)) {
5454
useAggregatedNodeLevelResponses = in.readOptionalBoolean();
5555
}
5656
}
@@ -76,7 +76,7 @@ public void useAggregatedNodeLevelResponses(boolean useAggregatedNodeLevelRespon
7676
@Override
7777
public void writeTo(StreamOutput out) throws IOException {
7878
super.writeTo(out);
79-
if (out.getVersion().onOrAfter(Version.V_3_0_0)) {
79+
if (out.getVersion().onOrAfter(Version.V_2_16_0)) {
8080
out.writeOptionalBoolean(useAggregatedNodeLevelResponses);
8181
}
8282
}

‎server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesReader.java

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,9 @@
1717
import org.apache.lucene.index.SortedNumericDocValues;
1818
import org.apache.lucene.index.SortedSetDocValues;
1919
import org.opensearch.common.annotation.ExperimentalApi;
20-
import org.opensearch.index.mapper.CompositeMappedFieldType;
2120

2221
import java.io.IOException;
22+
import java.util.ArrayList;
2323
import java.util.List;
2424

2525
/**
@@ -74,15 +74,13 @@ public void close() throws IOException {
7474
}
7575

7676
@Override
77-
public List<String> getCompositeIndexFields() {
77+
public List<CompositeIndexFieldInfo> getCompositeIndexFields() {
7878
// todo : read from file formats and get the field names.
79-
throw new UnsupportedOperationException();
80-
79+
return new ArrayList<>();
8180
}
8281

8382
@Override
84-
public CompositeIndexValues getCompositeIndexValues(String field, CompositeMappedFieldType.CompositeFieldType fieldType)
85-
throws IOException {
83+
public CompositeIndexValues getCompositeIndexValues(CompositeIndexFieldInfo compositeIndexFieldInfo) throws IOException {
8684
// TODO : read compositeIndexValues [starTreeValues] from star tree files
8785
throw new UnsupportedOperationException();
8886
}

‎server/src/main/java/org/opensearch/index/codec/composite/Composite99DocValuesWriter.java

Lines changed: 91 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,20 +8,29 @@
88

99
package org.opensearch.index.codec.composite;
1010

11+
import org.apache.logging.log4j.LogManager;
12+
import org.apache.logging.log4j.Logger;
1113
import org.apache.lucene.codecs.DocValuesConsumer;
1214
import org.apache.lucene.codecs.DocValuesProducer;
15+
import org.apache.lucene.index.DocValues;
16+
import org.apache.lucene.index.DocValuesType;
17+
import org.apache.lucene.index.EmptyDocValuesProducer;
1318
import org.apache.lucene.index.FieldInfo;
1419
import org.apache.lucene.index.MergeState;
1520
import org.apache.lucene.index.SegmentWriteState;
21+
import org.apache.lucene.index.SortedNumericDocValues;
1622
import org.opensearch.common.annotation.ExperimentalApi;
23+
import org.opensearch.index.codec.composite.datacube.startree.StarTreeValues;
24+
import org.opensearch.index.compositeindex.datacube.startree.StarTreeField;
1725
import org.opensearch.index.compositeindex.datacube.startree.builder.StarTreesBuilder;
1826
import org.opensearch.index.mapper.CompositeMappedFieldType;
1927
import org.opensearch.index.mapper.MapperService;
20-
import org.opensearch.index.mapper.StarTreeMapper;
2128

2229
import java.io.IOException;
30+
import java.util.Collections;
2331
import java.util.HashMap;
2432
import java.util.HashSet;
33+
import java.util.List;
2534
import java.util.Map;
2635
import java.util.Set;
2736
import java.util.concurrent.atomic.AtomicReference;
@@ -40,8 +49,10 @@ public class Composite99DocValuesWriter extends DocValuesConsumer {
4049
AtomicReference<MergeState> mergeState = new AtomicReference<>();
4150
private final Set<CompositeMappedFieldType> compositeMappedFieldTypes;
4251
private final Set<String> compositeFieldSet;
52+
private final Set<String> segmentFieldSet;
4353

4454
private final Map<String, DocValuesProducer> fieldProducerMap = new HashMap<>();
55+
private static final Logger logger = LogManager.getLogger(Composite99DocValuesWriter.class);
4556

4657
public Composite99DocValuesWriter(DocValuesConsumer delegate, SegmentWriteState segmentWriteState, MapperService mapperService) {
4758

@@ -50,6 +61,12 @@ public Composite99DocValuesWriter(DocValuesConsumer delegate, SegmentWriteState
5061
this.mapperService = mapperService;
5162
this.compositeMappedFieldTypes = mapperService.getCompositeFieldTypes();
5263
compositeFieldSet = new HashSet<>();
64+
segmentFieldSet = new HashSet<>();
65+
for (FieldInfo fi : segmentWriteState.fieldInfos) {
66+
if (DocValuesType.SORTED_NUMERIC.equals(fi.getDocValuesType())) {
67+
segmentFieldSet.add(fi.name);
68+
}
69+
}
5370
for (CompositeMappedFieldType type : compositeMappedFieldTypes) {
5471
compositeFieldSet.addAll(type.fields());
5572
}
@@ -95,23 +112,91 @@ private void createCompositeIndicesIfPossible(DocValuesProducer valuesProducer,
95112
fieldProducerMap.put(field.name, valuesProducer);
96113
compositeFieldSet.remove(field.name);
97114
}
115+
segmentFieldSet.remove(field.name);
116+
if (segmentFieldSet.isEmpty()) {
117+
Set<String> compositeFieldSetCopy = new HashSet<>(compositeFieldSet);
118+
for (String compositeField : compositeFieldSetCopy) {
119+
fieldProducerMap.put(compositeField, new EmptyDocValuesProducer() {
120+
@Override
121+
public SortedNumericDocValues getSortedNumeric(FieldInfo field) {
122+
return DocValues.emptySortedNumeric();
123+
}
124+
});
125+
compositeFieldSet.remove(compositeField);
126+
}
127+
}
98128
// we have all the required fields to build composite fields
99129
if (compositeFieldSet.isEmpty()) {
100130
for (CompositeMappedFieldType mappedType : compositeMappedFieldTypes) {
101-
if (mappedType instanceof StarTreeMapper.StarTreeFieldType) {
102-
try (StarTreesBuilder starTreesBuilder = new StarTreesBuilder(fieldProducerMap, state, mapperService)) {
103-
starTreesBuilder.build();
131+
if (mappedType.getCompositeIndexType().equals(CompositeMappedFieldType.CompositeFieldType.STAR_TREE)) {
132+
try (StarTreesBuilder starTreesBuilder = new StarTreesBuilder(state, mapperService)) {
133+
starTreesBuilder.build(fieldProducerMap);
104134
}
105135
}
106136
}
107137
}
138+
108139
}
109140

110141
@Override
111142
public void merge(MergeState mergeState) throws IOException {
112143
this.mergeState.compareAndSet(null, mergeState);
113144
super.merge(mergeState);
114-
// TODO : handle merge star tree
115-
// mergeStarTreeFields(mergeState);
145+
mergeCompositeFields(mergeState);
146+
}
147+
148+
/**
149+
* Merges composite fields from multiple segments
150+
* @param mergeState merge state
151+
*/
152+
private void mergeCompositeFields(MergeState mergeState) throws IOException {
153+
mergeStarTreeFields(mergeState);
154+
}
155+
156+
/**
157+
* Merges star tree data fields from multiple segments
158+
* @param mergeState merge state
159+
*/
160+
private void mergeStarTreeFields(MergeState mergeState) throws IOException {
161+
Map<String, List<StarTreeValues>> starTreeSubsPerField = new HashMap<>();
162+
StarTreeField starTreeField = null;
163+
for (int i = 0; i < mergeState.docValuesProducers.length; i++) {
164+
CompositeIndexReader reader = null;
165+
if (mergeState.docValuesProducers[i] == null) {
166+
continue;
167+
}
168+
if (mergeState.docValuesProducers[i] instanceof CompositeIndexReader) {
169+
reader = (CompositeIndexReader) mergeState.docValuesProducers[i];
170+
} else {
171+
continue;
172+
}
173+
174+
List<CompositeIndexFieldInfo> compositeFieldInfo = reader.getCompositeIndexFields();
175+
for (CompositeIndexFieldInfo fieldInfo : compositeFieldInfo) {
176+
if (fieldInfo.getType().equals(CompositeMappedFieldType.CompositeFieldType.STAR_TREE)) {
177+
CompositeIndexValues compositeIndexValues = reader.getCompositeIndexValues(fieldInfo);
178+
if (compositeIndexValues instanceof StarTreeValues) {
179+
StarTreeValues starTreeValues = (StarTreeValues) compositeIndexValues;
180+
List<StarTreeValues> fieldsList = starTreeSubsPerField.getOrDefault(fieldInfo.getField(), Collections.emptyList());
181+
if (starTreeField == null) {
182+
starTreeField = starTreeValues.getStarTreeField();
183+
}
184+
// assert star tree configuration is same across segments
185+
else {
186+
if (starTreeField.equals(starTreeValues.getStarTreeField()) == false) {
187+
throw new IllegalArgumentException(
188+
"star tree field configuration must match the configuration of the field being merged"
189+
);
190+
}
191+
}
192+
fieldsList.add(starTreeValues);
193+
starTreeSubsPerField.put(fieldInfo.getField(), fieldsList);
194+
}
195+
}
196+
}
197+
}
198+
try (StarTreesBuilder starTreesBuilder = new StarTreesBuilder(state, mapperService)) {
199+
starTreesBuilder.buildDuringMerge(starTreeSubsPerField);
200+
}
116201
}
117202
}

‎server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexFieldInfo.java

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
2+
/*
3+
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* The OpenSearch Contributors require contributions made to
6+
* this file be licensed under the Apache-2.0 license or a
7+
* compatible open source license.
8+
*/
9+
10+
package org.opensearch.index.codec.composite;
11+
12+
import org.opensearch.common.annotation.ExperimentalApi;
13+
import org.opensearch.index.mapper.CompositeMappedFieldType;
14+
15+
/**
16+
* Field info details of composite index fields
17+
*
18+
* @opensearch.experimental
19+
*/
20+
@ExperimentalApi
21+
public class CompositeIndexFieldInfo {
22+
private final String field;
23+
private final CompositeMappedFieldType.CompositeFieldType type;
24+
25+
public CompositeIndexFieldInfo(String field, CompositeMappedFieldType.CompositeFieldType type) {
26+
this.field = field;
27+
this.type = type;
28+
}
29+
30+
public String getField() {
31+
return field;
32+
}
33+
34+
public CompositeMappedFieldType.CompositeFieldType getType() {
35+
return type;
36+
}
37+
}

‎server/src/main/java/org/opensearch/index/codec/composite/CompositeIndexReader.java

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
package org.opensearch.index.codec.composite;
1010

1111
import org.opensearch.common.annotation.ExperimentalApi;
12-
import org.opensearch.index.mapper.CompositeMappedFieldType;
1312

1413
import java.io.IOException;
1514
import java.util.List;
@@ -25,10 +24,10 @@ public interface CompositeIndexReader {
2524
* Get list of composite index fields from the segment
2625
*
2726
*/
28-
List<String> getCompositeIndexFields();
27+
List<CompositeIndexFieldInfo> getCompositeIndexFields();
2928

3029
/**
3130
* Get composite index values based on the field name and the field type
3231
*/
33-
CompositeIndexValues getCompositeIndexValues(String field, CompositeMappedFieldType.CompositeFieldType fieldType) throws IOException;
32+
CompositeIndexValues getCompositeIndexValues(CompositeIndexFieldInfo fieldInfo) throws IOException;
3433
}

0 commit comments

Comments
 (0)