Skip to content

Commit 8686d28

Browse files
Gagan6164Gagan6164
authored
Add Snapshot and Remote Restore ITs for Warm Index (opensearch-project#18281)
Signed-off-by: Gagan Singh Saini <[email protected]>Signed-off-by: TJ Neuenfeldt <[email protected]>
1 parent e5af998 commit 8686d28

File tree

11 files changed

+374
-110
lines changed

11 files changed

+374
-110
lines changed

‎server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java

Lines changed: 58 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,9 @@
88

99
package org.opensearch.remotestore;
1010

11+
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
12+
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
13+
1114
import org.opensearch.action.DocWriteResponse;
1215
import org.opensearch.action.LatchedActionListener;
1316
import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest;
@@ -30,17 +33,20 @@
3033
import org.opensearch.common.util.io.IOUtils;
3134
import org.opensearch.core.action.ActionListener;
3235
import org.opensearch.core.common.unit.ByteSizeUnit;
36+
import org.opensearch.core.common.unit.ByteSizeValue;
3337
import org.opensearch.core.index.Index;
3438
import org.opensearch.core.rest.RestStatus;
3539
import org.opensearch.index.IndexService;
3640
import org.opensearch.index.IndexSettings;
3741
import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
3842
import org.opensearch.index.remote.RemoteStoreEnums.PathType;
3943
import org.opensearch.index.shard.IndexShard;
44+
import org.opensearch.index.store.remote.file.CleanerDaemonThreadLeakFilter;
4045
import org.opensearch.indices.IndicesService;
4146
import org.opensearch.indices.RemoteStoreSettings;
4247
import org.opensearch.indices.recovery.RecoveryState;
4348
import org.opensearch.indices.replication.common.ReplicationType;
49+
import org.opensearch.node.Node;
4450
import org.opensearch.node.remotestore.RemoteStorePinnedTimestampService;
4551
import org.opensearch.repositories.RepositoriesService;
4652
import org.opensearch.repositories.Repository;
@@ -60,6 +66,7 @@
6066
import java.nio.file.Path;
6167
import java.util.ArrayList;
6268
import java.util.Arrays;
69+
import java.util.Collection;
6370
import java.util.Collections;
6471
import java.util.List;
6572
import java.util.Map;
@@ -77,6 +84,7 @@
7784
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY;
7885
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED;
7986
import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY;
87+
import static org.opensearch.common.util.FeatureFlags.WRITABLE_WARM_INDEX_SETTING;
8088
import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.SEGMENTS;
8189
import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.TRANSLOG;
8290
import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA;
@@ -89,9 +97,31 @@
8997
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
9098
import static org.hamcrest.Matchers.lessThanOrEqualTo;
9199

100+
@ThreadLeakFilters(filters = CleanerDaemonThreadLeakFilter.class)
92101
@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
93102
public class RemoteRestoreSnapshotIT extends RemoteSnapshotIT {
94103

104+
public RemoteRestoreSnapshotIT(Settings nodeSettings) {
105+
super(nodeSettings);
106+
}
107+
108+
@ParametersFactory
109+
public static Collection<Object[]> parameters() {
110+
return Arrays.asList(
111+
new Object[] { Settings.builder().put(WRITABLE_WARM_INDEX_SETTING.getKey(), false).build() },
112+
new Object[] { Settings.builder().put(WRITABLE_WARM_INDEX_SETTING.getKey(), true).build() }
113+
);
114+
}
115+
116+
@Override
117+
protected Settings nodeSettings(int nodeOrdinal) {
118+
ByteSizeValue cacheSize = new ByteSizeValue(16, ByteSizeUnit.GB);
119+
return Settings.builder()
120+
.put(super.nodeSettings(nodeOrdinal))
121+
.put(Node.NODE_SEARCH_CACHE_SIZE_SETTING.getKey(), cacheSize.toString())
122+
.build();
123+
}
124+
95125
private void assertDocsPresentInIndex(Client client, String indexName, int numOfDocs) {
96126
for (int i = 0; i < numOfDocs; i++) {
97127
String id = Integer.toString(i);
@@ -102,7 +132,7 @@ private void assertDocsPresentInIndex(Client client, String indexName, int numOf
102132

103133
public void testRestoreOperationsShallowCopyEnabled() throws Exception {
104134
String clusterManagerNode = internalCluster().startClusterManagerOnlyNode();
105-
String primary = internalCluster().startDataOnlyNode();
135+
String primary = internalCluster().startDataAndWarmNodes(1).get(0);
106136
String indexName1 = "testindex1";
107137
String indexName2 = "testindex2";
108138
String snapshotRepoName = "test-restore-snapshot-repo";
@@ -128,7 +158,7 @@ public void testRestoreOperationsShallowCopyEnabled() throws Exception {
128158
indexDocuments(client, indexName2, numDocsInIndex2);
129159
ensureGreen(indexName1, indexName2);
130160

131-
internalCluster().startDataOnlyNode();
161+
internalCluster().startDataAndWarmNodes(1);
132162
logger.info("--> snapshot");
133163

134164
SnapshotInfo snapshotInfo = createSnapshot(snapshotRepoName, snapshotName1, new ArrayList<>(Arrays.asList(indexName1, indexName2)));
@@ -200,7 +230,7 @@ public void testRestoreOperationsShallowCopyEnabled() throws Exception {
200230
*/
201231
public void testRemoteStoreCustomDataOnIndexCreationAndRestore() {
202232
String clusterManagerNode = internalCluster().startClusterManagerOnlyNode();
203-
internalCluster().startDataOnlyNode();
233+
internalCluster().startDataAndWarmNodes(1);
204234
String indexName1 = "testindex1";
205235
String indexName2 = "testindex2";
206236
String snapshotRepoName = "test-restore-snapshot-repo";
@@ -313,7 +343,7 @@ private void validatePathType(String index, PathType pathType, @Nullable PathHas
313343

314344
public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException {
315345
String clusterManagerNode = internalCluster().startClusterManagerOnlyNode();
316-
String primary = internalCluster().startDataOnlyNode();
346+
String primary = internalCluster().startDataAndWarmNodes(1).get(0);
317347
String indexName1 = "testindex1";
318348
String indexName2 = "testindex2";
319349
String snapshotRepoName = "test-restore-snapshot-repo";
@@ -339,7 +369,7 @@ public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException {
339369
indexDocuments(client, indexName2, numDocsInIndex2);
340370
ensureGreen(indexName1, indexName2);
341371

342-
internalCluster().startDataOnlyNode();
372+
internalCluster().startDataAndWarmNodes(1);
343373
logger.info("--> snapshot");
344374
SnapshotInfo snapshotInfo1 = createSnapshot(
345375
snapshotRepoName,
@@ -441,7 +471,7 @@ void assertRemoteSegmentsAndTranslogUploaded(String idx) throws IOException {
441471

442472
public void testRemoteRestoreIndexRestoredFromSnapshot() throws IOException, ExecutionException, InterruptedException {
443473
internalCluster().startClusterManagerOnlyNode();
444-
internalCluster().startDataOnlyNodes(2);
474+
internalCluster().startDataAndWarmNodes(2);
445475

446476
String indexName1 = "testindex1";
447477
String snapshotRepoName = "test-restore-snapshot-repo";
@@ -500,7 +530,7 @@ public void testRemoteRestoreIndexRestoredFromSnapshot() throws IOException, Exe
500530

501531
public void testSuccessfulIndexRestoredFromSnapshotWithUpdatedSetting() throws IOException, ExecutionException, InterruptedException {
502532
internalCluster().startClusterManagerOnlyNode();
503-
internalCluster().startDataOnlyNodes(2);
533+
internalCluster().startDataAndWarmNodes(2);
504534

505535
String indexName1 = "testindex1";
506536
String snapshotRepoName = "test-restore-snapshot-repo";
@@ -716,7 +746,7 @@ public void testRestoreShallowSnapshotIndexAfterSnapshot() throws ExecutionExcep
716746

717747
public void testInvalidRestoreRequestScenarios() throws Exception {
718748
internalCluster().startClusterManagerOnlyNode();
719-
internalCluster().startDataOnlyNode();
749+
internalCluster().startDataAndWarmNodes(1);
720750
String index = "test-index";
721751
String snapshotRepo = "test-restore-snapshot-repo";
722752
String newRemoteStoreRepo = "test-new-rs-repo";
@@ -736,7 +766,7 @@ public void testInvalidRestoreRequestScenarios() throws Exception {
736766
indexDocuments(client, index, numDocsInIndex);
737767
ensureGreen(index);
738768

739-
internalCluster().startDataOnlyNode();
769+
internalCluster().startDataAndWarmNodes(1);
740770
logger.info("--> snapshot");
741771

742772
SnapshotInfo snapshotInfo = createSnapshot(snapshotRepo, snapshotName1, new ArrayList<>(List.of(index)));
@@ -896,8 +926,8 @@ public void testInvalidRestoreRequestScenarios() throws Exception {
896926

897927
public void testCreateSnapshotV2_Orphan_Timestamp_Cleanup() throws Exception {
898928
internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
899-
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
900-
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
929+
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
930+
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
901931
String indexName1 = "testindex1";
902932
String indexName2 = "testindex2";
903933
String snapshotRepoName = "test-create-snapshot-repo";
@@ -962,8 +992,8 @@ public void onFailure(Exception e) {}
962992
public void testMixedSnapshotCreationWithV2RepositorySetting() throws Exception {
963993

964994
internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
965-
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
966-
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
995+
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
996+
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
967997
String indexName1 = "testindex1";
968998
String indexName2 = "testindex2";
969999
String indexName3 = "testindex3";
@@ -1034,8 +1064,8 @@ public void testMixedSnapshotCreationWithV2RepositorySetting() throws Exception
10341064

10351065
public void testConcurrentSnapshotV2CreateOperation() throws InterruptedException, ExecutionException {
10361066
internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
1037-
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
1038-
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
1067+
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
1068+
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
10391069
String indexName1 = "testindex1";
10401070
String indexName2 = "testindex2";
10411071
String snapshotRepoName = "test-create-snapshot-repo";
@@ -1108,8 +1138,8 @@ public void testConcurrentSnapshotV2CreateOperation_MasterChange() throws Except
11081138
internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
11091139
internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
11101140
internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
1111-
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
1112-
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
1141+
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
1142+
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
11131143
String indexName1 = "testindex1";
11141144
String indexName2 = "testindex2";
11151145
String snapshotRepoName = "test-create-snapshot-repo";
@@ -1184,8 +1214,8 @@ public void testConcurrentSnapshotV2CreateOperation_MasterChange() throws Except
11841214

11851215
public void testCreateSnapshotV2() throws Exception {
11861216
internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
1187-
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
1188-
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
1217+
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
1218+
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
11891219
String indexName1 = "testindex1";
11901220
String indexName2 = "testindex2";
11911221
String indexName3 = "testindex3";
@@ -1254,8 +1284,8 @@ public void forceSyncPinnedTimestamps() {
12541284

12551285
public void testCreateSnapshotV2WithRedIndex() throws Exception {
12561286
internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
1257-
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
1258-
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
1287+
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
1288+
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
12591289
String indexName1 = "testindex1";
12601290
String indexName2 = "testindex2";
12611291
String snapshotRepoName = "test-create-snapshot-repo";
@@ -1302,8 +1332,8 @@ public void testCreateSnapshotV2WithRedIndex() throws Exception {
13021332

13031333
public void testCreateSnapshotV2WithIndexingLoad() throws Exception {
13041334
internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
1305-
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
1306-
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
1335+
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
1336+
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
13071337
String indexName1 = "testindex1";
13081338
String indexName2 = "testindex2";
13091339
String snapshotRepoName = "test-create-snapshot-repo";
@@ -1370,8 +1400,8 @@ public void testCreateSnapshotV2WithIndexingLoad() throws Exception {
13701400

13711401
public void testCreateSnapshotV2WithShallowCopySettingDisabled() throws Exception {
13721402
internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
1373-
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
1374-
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
1403+
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
1404+
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
13751405
String indexName1 = "testindex1";
13761406
String indexName2 = "testindex2";
13771407
String snapshotRepoName = "test-create-snapshot-repo";
@@ -1419,7 +1449,7 @@ public void testCreateSnapshotV2WithShallowCopySettingDisabled() throws Exceptio
14191449
public void testClusterManagerFailoverDuringSnapshotCreation() throws Exception {
14201450

14211451
internalCluster().startClusterManagerOnlyNodes(3, pinnedTimestampSettings());
1422-
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
1452+
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
14231453
String indexName1 = "testindex1";
14241454
String indexName2 = "testindex2";
14251455
String snapshotRepoName = "test-create-snapshot-repo";
@@ -1493,8 +1523,8 @@ public void testClusterManagerFailoverDuringSnapshotCreation() throws Exception
14931523

14941524
public void testConcurrentV1SnapshotAndV2RepoSettingUpdate() throws Exception {
14951525
internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
1496-
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
1497-
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
1526+
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
1527+
internalCluster().startDataAndWarmNodes(1, pinnedTimestampSettings());
14981528
String snapshotRepoName = "test-create-snapshot-repo";
14991529
String snapshotName1 = "test-create-snapshot-v1";
15001530
Path absolutePath1 = randomRepoPath().toAbsolutePath();

‎server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteSnapshotIT.java

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,8 +13,10 @@
1313
import org.opensearch.cluster.metadata.IndexMetadata;
1414
import org.opensearch.cluster.metadata.RepositoryMetadata;
1515
import org.opensearch.common.settings.Settings;
16+
import org.opensearch.index.IndexModule;
1617
import org.opensearch.index.IndexSettings;
1718
import org.opensearch.indices.RemoteStoreSettings;
19+
import org.opensearch.node.Node;
1820
import org.opensearch.repositories.fs.ReloadableFsRepository;
1921
import org.opensearch.snapshots.AbstractSnapshotIntegTestCase;
2022
import org.opensearch.transport.client.Client;
@@ -24,9 +26,20 @@
2426
import java.nio.file.Path;
2527
import java.util.concurrent.ExecutionException;
2628

29+
import static org.opensearch.common.util.FeatureFlags.WRITABLE_WARM_INDEX_SETTING;
2730
import static org.opensearch.repositories.fs.ReloadableFsRepository.REPOSITORIES_FAILRATE_SETTING;
31+
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
2832

2933
public abstract class RemoteSnapshotIT extends AbstractSnapshotIntegTestCase {
34+
35+
public RemoteSnapshotIT(Settings nodeSettings) {
36+
super(nodeSettings);
37+
}
38+
39+
public RemoteSnapshotIT() {
40+
super();
41+
}
42+
3043
protected static final String BASE_REMOTE_REPO = "test-rs-repo" + TEST_REMOTE_STORE_REPO_SUFFIX;
3144
protected Path remoteRepoPath;
3245

@@ -38,6 +51,16 @@ public void setup() {
3851
@After
3952
public void teardown() {
4053
clusterAdmin().prepareCleanupRepository(BASE_REMOTE_REPO).get();
54+
if (WRITABLE_WARM_INDEX_SETTING.get(settings)) {
55+
assertAcked(client().admin().indices().prepareDelete("_all").get());
56+
var nodes = internalCluster().getDataNodeInstances(Node.class);
57+
for (var node : nodes) {
58+
var fileCache = node.fileCache();
59+
if (fileCache != null) {
60+
fileCache.clear();
61+
}
62+
}
63+
}
4164
}
4265

4366
@Override
@@ -61,6 +84,9 @@ protected Settings.Builder getIndexSettings(int numOfShards, int numOfReplicas)
6184
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards)
6285
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas)
6386
.put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "300s");
87+
if (WRITABLE_WARM_INDEX_SETTING.get(settings)) {
88+
settingsBuilder.put(IndexModule.IS_WARM_INDEX_SETTING.getKey(), true);
89+
}
6490
return settingsBuilder;
6591
}
6692

0 commit comments

Comments
 (0)