Skip to content

Commit 747c06e

Browse files
bharath-techiebharath-techie
committed
Merge branch 'main' of github.com:opensearch-project/OpenSearch into avgfix
2 parents 3ec11d3 + d28fbc3 commit 747c06e

File tree

89 files changed

+2204
-761
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

89 files changed

+2204
-761
lines changed

‎CHANGELOG.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
5050
- Optimise snapshot deletion to speed up snapshot deletion and creation ([#15568](https://github.com/opensearch-project/OpenSearch/pull/15568))
5151
- [Remote Publication] Added checksum validation for cluster state behind a cluster setting ([#15218](https://github.com/opensearch-project/OpenSearch/pull/15218))
5252
- Add canRemain method to TargetPoolAllocationDecider to move shards from local to remote pool for hot to warm tiering ([#15010](https://github.com/opensearch-project/OpenSearch/pull/15010))
53+
- ClusterManagerTaskThrottler Improvements ([#15508](https://github.com/opensearch-project/OpenSearch/pull/15508))
54+
- Reset DiscoveryNodes in all transport node actions request ([#15131](https://github.com/opensearch-project/OpenSearch/pull/15131))
5355

5456
### Dependencies
5557
- Bump `netty` from 4.1.111.Final to 4.1.112.Final ([#15081](https://github.com/opensearch-project/OpenSearch/pull/15081))

‎modules/mapper-extras/src/main/java/org/opensearch/index/mapper/ScaledFloatFieldMapper.java

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
import org.opensearch.common.xcontent.support.XContentMapValues;
5050
import org.opensearch.core.xcontent.XContentParser;
5151
import org.opensearch.core.xcontent.XContentParser.Token;
52+
import org.opensearch.index.compositeindex.datacube.DimensionType;
5253
import org.opensearch.index.fielddata.FieldData;
5354
import org.opensearch.index.fielddata.IndexFieldData;
5455
import org.opensearch.index.fielddata.IndexNumericFieldData;
@@ -71,10 +72,12 @@
7172
import java.util.Collections;
7273
import java.util.List;
7374
import java.util.Map;
75+
import java.util.Optional;
7476
import java.util.function.Supplier;
7577

7678
/** A {@link FieldMapper} for scaled floats. Values are internally multiplied
77-
* by a scaling factor and rounded to the closest long. */
79+
* by a scaling factor and rounded to the closest long.
80+
*/
7881
public class ScaledFloatFieldMapper extends ParametrizedFieldMapper {
7982

8083
public static final String CONTENT_TYPE = "scaled_float";
@@ -162,11 +165,21 @@ public ScaledFloatFieldMapper build(BuilderContext context) {
162165
);
163166
return new ScaledFloatFieldMapper(name, type, multiFieldsBuilder.build(this, context), copyTo.build(), this);
164167
}
168+
169+
@Override
170+
public Optional<DimensionType> getSupportedDataCubeDimensionType() {
171+
return Optional.of(DimensionType.NUMERIC);
172+
}
173+
174+
@Override
175+
public boolean isDataCubeMetricSupported() {
176+
return true;
177+
}
165178
}
166179

167180
public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n, c.getSettings()));
168181

169-
public static final class ScaledFloatFieldType extends SimpleMappedFieldType implements NumericPointEncoder {
182+
public static final class ScaledFloatFieldType extends SimpleMappedFieldType implements NumericPointEncoder, FieldValueConverter {
170183

171184
private final double scalingFactor;
172185
private final Double nullValue;
@@ -340,6 +353,12 @@ public DocValueFormat docValueFormat(String format, ZoneId timeZone) {
340353
private double scale(Object input) {
341354
return new BigDecimal(Double.toString(parse(input))).multiply(BigDecimal.valueOf(scalingFactor)).doubleValue();
342355
}
356+
357+
@Override
358+
public double toDoubleValue(long value) {
359+
double inverseScalingFactor = 1d / scalingFactor;
360+
return value * inverseScalingFactor;
361+
}
343362
}
344363

345364
private final Explicit<Boolean> ignoreMalformed;

‎modules/mapper-extras/src/test/java/org/opensearch/index/mapper/ScaledFloatFieldMapperTests.java

Lines changed: 101 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -34,18 +34,24 @@
3434

3535
import org.apache.lucene.index.DocValuesType;
3636
import org.apache.lucene.index.IndexableField;
37+
import org.opensearch.common.settings.Settings;
38+
import org.opensearch.common.util.FeatureFlags;
3739
import org.opensearch.common.xcontent.XContentFactory;
3840
import org.opensearch.core.common.bytes.BytesReference;
3941
import org.opensearch.core.xcontent.MediaTypeRegistry;
4042
import org.opensearch.core.xcontent.XContentBuilder;
43+
import org.opensearch.index.compositeindex.datacube.startree.StarTreeIndexSettings;
4144
import org.opensearch.plugins.Plugin;
45+
import org.junit.AfterClass;
46+
import org.junit.BeforeClass;
4247

4348
import java.io.IOException;
4449
import java.util.Arrays;
4550
import java.util.Collection;
4651
import java.util.List;
4752

4853
import static java.util.Collections.singletonList;
54+
import static org.opensearch.common.util.FeatureFlags.STAR_TREE_INDEX;
4955
import static org.hamcrest.Matchers.containsString;
5056

5157
public class ScaledFloatFieldMapperTests extends MapperTestCase {
@@ -91,24 +97,112 @@ public void testExistsQueryDocValuesDisabled() throws IOException {
9197
assertParseMinimalWarnings();
9298
}
9399

94-
public void testDefaults() throws Exception {
95-
XContentBuilder mapping = fieldMapping(b -> b.field("type", "scaled_float").field("scaling_factor", 10.0));
100+
@BeforeClass
101+
public static void createMapper() {
102+
FeatureFlags.initializeFeatureFlags(Settings.builder().put(STAR_TREE_INDEX, "true").build());
103+
}
104+
105+
@AfterClass
106+
public static void clearMapper() {
107+
FeatureFlags.initializeFeatureFlags(Settings.EMPTY);
108+
}
109+
110+
public void testScaledFloatWithStarTree() throws Exception {
111+
112+
double scalingFactorField1 = randomDouble() * 100;
113+
double scalingFactorField2 = randomDouble() * 100;
114+
double scalingFactorField3 = randomDouble() * 100;
115+
116+
XContentBuilder mapping = getStarTreeMappingWithScaledFloat(scalingFactorField1, scalingFactorField2, scalingFactorField3);
96117
DocumentMapper mapper = createDocumentMapper(mapping);
97-
assertEquals(mapping.toString(), mapper.mappingSource().toString());
118+
assertTrue(mapping.toString().contains("startree"));
98119

99-
ParsedDocument doc = mapper.parse(source(b -> b.field("field", 123)));
100-
IndexableField[] fields = doc.rootDoc().getFields("field");
120+
long randomLongField1 = randomLong();
121+
long randomLongField2 = randomLong();
122+
long randomLongField3 = randomLong();
123+
ParsedDocument doc = mapper.parse(
124+
source(b -> b.field("field1", randomLongField1).field("field2", randomLongField2).field("field3", randomLongField3))
125+
);
126+
validateScaledFloatFields(doc, "field1", randomLongField1, scalingFactorField1);
127+
validateScaledFloatFields(doc, "field2", randomLongField2, scalingFactorField2);
128+
validateScaledFloatFields(doc, "field3", randomLongField3, scalingFactorField3);
129+
}
130+
131+
@Override
132+
protected Settings getIndexSettings() {
133+
return Settings.builder()
134+
.put(StarTreeIndexSettings.IS_COMPOSITE_INDEX_SETTING.getKey(), true)
135+
.put(super.getIndexSettings())
136+
.build();
137+
}
138+
139+
private static void validateScaledFloatFields(ParsedDocument doc, String field, long value, double scalingFactor) {
140+
IndexableField[] fields = doc.rootDoc().getFields(field);
101141
assertEquals(2, fields.length);
102142
IndexableField pointField = fields[0];
103143
assertEquals(1, pointField.fieldType().pointDimensionCount());
104144
assertFalse(pointField.fieldType().stored());
105-
assertEquals(1230, pointField.numericValue().longValue());
145+
assertEquals((long) (value * scalingFactor), pointField.numericValue().longValue());
106146
IndexableField dvField = fields[1];
107147
assertEquals(DocValuesType.SORTED_NUMERIC, dvField.fieldType().docValuesType());
108-
assertEquals(1230, dvField.numericValue().longValue());
148+
assertEquals((long) (value * scalingFactor), dvField.numericValue().longValue());
109149
assertFalse(dvField.fieldType().stored());
110150
}
111151

152+
private XContentBuilder getStarTreeMappingWithScaledFloat(
153+
double scalingFactorField1,
154+
double scalingFactorField2,
155+
double scalingFactorField3
156+
) throws IOException {
157+
return topMapping(b -> {
158+
b.startObject("composite");
159+
b.startObject("startree");
160+
b.field("type", "star_tree");
161+
b.startObject("config");
162+
b.field("max_leaf_docs", 100);
163+
b.startArray("ordered_dimensions");
164+
b.startObject();
165+
b.field("name", "field1");
166+
b.endObject();
167+
b.startObject();
168+
b.field("name", "field2");
169+
b.endObject();
170+
b.endArray();
171+
b.startArray("metrics");
172+
b.startObject();
173+
b.field("name", "field3");
174+
b.startArray("stats");
175+
b.value("sum");
176+
b.value("value_count");
177+
b.endArray();
178+
b.endObject();
179+
b.endArray();
180+
b.endObject();
181+
b.endObject();
182+
b.endObject();
183+
b.startObject("properties");
184+
b.startObject("field1");
185+
b.field("type", "scaled_float").field("scaling_factor", scalingFactorField1);
186+
b.endObject();
187+
b.startObject("field2");
188+
b.field("type", "scaled_float").field("scaling_factor", scalingFactorField2);
189+
b.endObject();
190+
b.startObject("field3");
191+
b.field("type", "scaled_float").field("scaling_factor", scalingFactorField3);
192+
b.endObject();
193+
b.endObject();
194+
});
195+
}
196+
197+
public void testDefaults() throws Exception {
198+
XContentBuilder mapping = fieldMapping(b -> b.field("type", "scaled_float").field("scaling_factor", 10.0));
199+
DocumentMapper mapper = createDocumentMapper(mapping);
200+
assertEquals(mapping.toString(), mapper.mappingSource().toString());
201+
202+
ParsedDocument doc = mapper.parse(source(b -> b.field("field", 123)));
203+
validateScaledFloatFields(doc, "field", 123, 10.0);
204+
}
205+
112206
public void testMissingScalingFactor() {
113207
Exception e = expectThrows(
114208
MapperParsingException.class,

‎rest-api-spec/src/main/resources/rest-api-spec/test/snapshot.status/10_basic.yml

Lines changed: 36 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -25,38 +25,40 @@ setup:
2525
snapshot: test_snapshot
2626
wait_for_completion: true
2727

28-
- do:
29-
snapshot.status:
30-
repository: test_repo_status_1
31-
snapshot: test_snapshot
32-
33-
- is_true: snapshots
34-
- match: { snapshots.0.snapshot: test_snapshot }
35-
- match: { snapshots.0.state: SUCCESS }
36-
- gt: { snapshots.0.stats.incremental.file_count: 0 }
37-
- gt: { snapshots.0.stats.incremental.size_in_bytes: 0 }
38-
- gt: { snapshots.0.stats.total.file_count: 0 }
39-
- gt: { snapshots.0.stats.total.size_in_bytes: 0 }
40-
- is_true: snapshots.0.stats.start_time_in_millis
41-
## fast in memory snapshots can take less than one millisecond to complete.
42-
- gte: { snapshots.0.stats.time_in_millis: 0 }
43-
44-
---
45-
"Get missing snapshot status throws an exception":
46-
47-
- do:
48-
catch: /snapshot_missing_exception.+ is missing/
49-
snapshot.status:
50-
repository: test_repo_status_1
51-
snapshot: test_nonexistent_snapshot
52-
53-
---
54-
"Get missing snapshot status succeeds when ignoreUnavailable is true":
55-
56-
- do:
57-
snapshot.status:
58-
repository: test_repo_status_1
59-
snapshot: test_nonexistent_snapshot
60-
ignore_unavailable: true
28+
# TODO: fix and unmute tests
6129

62-
- is_true: snapshots
30+
# - do:
31+
# snapshot.status:
32+
# repository: test_repo_status_1
33+
# snapshot: test_snapshot
34+
#
35+
# - is_true: snapshots
36+
# - match: { snapshots.0.snapshot: test_snapshot }
37+
# - match: { snapshots.0.state: SUCCESS }
38+
# - gt: { snapshots.0.stats.incremental.file_count: 0 }
39+
# - gt: { snapshots.0.stats.incremental.size_in_bytes: 0 }
40+
# - gt: { snapshots.0.stats.total.file_count: 0 }
41+
# - gt: { snapshots.0.stats.total.size_in_bytes: 0 }
42+
# - is_true: snapshots.0.stats.start_time_in_millis
43+
### fast in memory snapshots can take less than one millisecond to complete.
44+
# - gte: { snapshots.0.stats.time_in_millis: 0 }
45+
#
46+
#---
47+
#"Get missing snapshot status throws an exception":
48+
#
49+
# - do:
50+
# catch: /snapshot_missing_exception.+ is missing/
51+
# snapshot.status:
52+
# repository: test_repo_status_1
53+
# snapshot: test_nonexistent_snapshot
54+
#
55+
#---
56+
#"Get missing snapshot status succeeds when ignoreUnavailable is true":
57+
#
58+
# - do:
59+
# snapshot.status:
60+
# repository: test_repo_status_1
61+
# snapshot: test_nonexistent_snapshot
62+
# ignore_unavailable: true
63+
#
64+
# - is_true: snapshots

‎server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteRoutingTableServiceIT.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,10 @@ protected Settings nodeSettings(int nodeOrdinal) {
6767
)
6868
.put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, REMOTE_ROUTING_TABLE_REPO)
6969
.put(REMOTE_PUBLICATION_EXPERIMENTAL, true)
70-
.put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_ENABLED_SETTING.getKey(), true)
70+
.put(
71+
RemoteClusterStateService.REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_MODE_SETTING.getKey(),
72+
RemoteClusterStateService.RemoteClusterStateValidationMode.FAILURE
73+
)
7174
.build();
7275
}
7376

‎server/src/internalClusterTest/java/org/opensearch/gateway/remote/RemoteStatePublicationIT.java

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,10 @@ protected Settings nodeSettings(int nodeOrdinal) {
9090
.put("node.attr." + REMOTE_STORE_ROUTING_TABLE_REPOSITORY_NAME_ATTRIBUTE_KEY, routingTableRepoName)
9191
.put(routingTableRepoTypeAttributeKey, ReloadableFsRepository.TYPE)
9292
.put(routingTableRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath)
93-
.put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_ENABLED_SETTING.getKey(), true)
93+
.put(
94+
RemoteClusterStateService.REMOTE_CLUSTER_STATE_CHECKSUM_VALIDATION_MODE_SETTING.getKey(),
95+
RemoteClusterStateService.RemoteClusterStateValidationMode.FAILURE
96+
)
9497
.build();
9598
}
9699

0 commit comments

Comments
 (0)