Skip to content

Commit e8b02c9

Browse files
gbbafnagbbafna
authored
Shallow snapshot v2 - create snapshot validations in a cluster state update (#15939)
--------- Signed-off-by: Gaurav Bafna <[email protected]>
1 parent 7caca26 commit e8b02c9

File tree

7 files changed

+1134
-230
lines changed

7 files changed

+1134
-230
lines changed

‎server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java

Lines changed: 74 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
import org.opensearch.action.DocWriteResponse;
1212
import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest;
1313
import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
14+
import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
15+
import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
1416
import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
1517
import org.opensearch.action.admin.indices.delete.DeleteIndexRequest;
1618
import org.opensearch.action.admin.indices.recovery.RecoveryResponse;
@@ -25,6 +27,7 @@
2527
import org.opensearch.common.blobstore.BlobPath;
2628
import org.opensearch.common.io.PathUtils;
2729
import org.opensearch.common.settings.Settings;
30+
import org.opensearch.common.unit.TimeValue;
2831
import org.opensearch.common.util.io.IOUtils;
2932
import org.opensearch.core.common.unit.ByteSizeUnit;
3033
import org.opensearch.core.index.Index;
@@ -43,14 +46,11 @@
4346
import org.opensearch.repositories.RepositoryData;
4447
import org.opensearch.repositories.blobstore.BlobStoreRepository;
4548
import org.opensearch.repositories.fs.FsRepository;
46-
import org.opensearch.snapshots.AbstractSnapshotIntegTestCase;
4749
import org.opensearch.snapshots.SnapshotInfo;
4850
import org.opensearch.snapshots.SnapshotRestoreException;
4951
import org.opensearch.snapshots.SnapshotState;
5052
import org.opensearch.test.InternalTestCluster;
5153
import org.opensearch.test.OpenSearchIntegTestCase;
52-
import org.junit.After;
53-
import org.junit.Before;
5454

5555
import java.io.IOException;
5656
import java.nio.file.Files;
@@ -63,6 +63,7 @@
6363
import java.util.Objects;
6464
import java.util.Optional;
6565
import java.util.concurrent.ExecutionException;
66+
import java.util.concurrent.TimeUnit;
6667
import java.util.stream.Collectors;
6768
import java.util.stream.Stream;
6869

@@ -79,48 +80,7 @@
7980
import static org.hamcrest.Matchers.lessThanOrEqualTo;
8081

8182
@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
82-
public class RemoteRestoreSnapshotIT extends AbstractSnapshotIntegTestCase {
83-
private static final String BASE_REMOTE_REPO = "test-rs-repo" + TEST_REMOTE_STORE_REPO_SUFFIX;
84-
private Path remoteRepoPath;
85-
86-
@Before
87-
public void setup() {
88-
remoteRepoPath = randomRepoPath().toAbsolutePath();
89-
}
90-
91-
@After
92-
public void teardown() {
93-
clusterAdmin().prepareCleanupRepository(BASE_REMOTE_REPO).get();
94-
}
95-
96-
@Override
97-
protected Settings nodeSettings(int nodeOrdinal) {
98-
return Settings.builder()
99-
.put(super.nodeSettings(nodeOrdinal))
100-
.put(remoteStoreClusterSettings(BASE_REMOTE_REPO, remoteRepoPath))
101-
.build();
102-
}
103-
104-
private Settings.Builder getIndexSettings(int numOfShards, int numOfReplicas) {
105-
Settings.Builder settingsBuilder = Settings.builder()
106-
.put(super.indexSettings())
107-
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards)
108-
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas)
109-
.put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "300s");
110-
return settingsBuilder;
111-
}
112-
113-
private void indexDocuments(Client client, String indexName, int numOfDocs) {
114-
indexDocuments(client, indexName, 0, numOfDocs);
115-
}
116-
117-
private void indexDocuments(Client client, String indexName, int fromId, int toId) {
118-
for (int i = fromId; i < toId; i++) {
119-
String id = Integer.toString(i);
120-
client.prepareIndex(indexName).setId(id).setSource("text", "sometext").get();
121-
}
122-
client.admin().indices().prepareFlush(indexName).get();
123-
}
83+
public class RemoteRestoreSnapshotIT extends RemoteSnapshotIT {
12484

12585
private void assertDocsPresentInIndex(Client client, String indexName, int numOfDocs) {
12686
for (int i = 0; i < numOfDocs; i++) {
@@ -997,6 +957,75 @@ public void testConcurrentSnapshotV2CreateOperation() throws InterruptedExceptio
997957
assertThat(repositoryData.getSnapshotIds().size(), greaterThanOrEqualTo(1));
998958
}
999959

960+
public void testConcurrentSnapshotV2CreateOperation_MasterChange() throws Exception {
961+
internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
962+
internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
963+
internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
964+
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
965+
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
966+
String indexName1 = "testindex1";
967+
String indexName2 = "testindex2";
968+
String snapshotRepoName = "test-create-snapshot-repo";
969+
Path absolutePath1 = randomRepoPath().toAbsolutePath();
970+
logger.info("Snapshot Path [{}]", absolutePath1);
971+
972+
Settings.Builder settings = Settings.builder()
973+
.put(FsRepository.LOCATION_SETTING.getKey(), absolutePath1)
974+
.put(FsRepository.COMPRESS_SETTING.getKey(), randomBoolean())
975+
.put(FsRepository.CHUNK_SIZE_SETTING.getKey(), randomIntBetween(100, 1000), ByteSizeUnit.BYTES)
976+
.put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), true)
977+
.put(BlobStoreRepository.SHALLOW_SNAPSHOT_V2.getKey(), true);
978+
createRepository(snapshotRepoName, FsRepository.TYPE, settings);
979+
980+
Client client = client();
981+
Settings indexSettings = getIndexSettings(20, 0).build();
982+
createIndex(indexName1, indexSettings);
983+
984+
Settings indexSettings2 = getIndexSettings(15, 0).build();
985+
createIndex(indexName2, indexSettings2);
986+
987+
final int numDocsInIndex1 = 10;
988+
final int numDocsInIndex2 = 20;
989+
indexDocuments(client, indexName1, numDocsInIndex1);
990+
indexDocuments(client, indexName2, numDocsInIndex2);
991+
ensureGreen(indexName1, indexName2);
992+
993+
Thread thread = new Thread(() -> {
994+
try {
995+
String snapshotName = "snapshot-earlier-master";
996+
internalCluster().nonClusterManagerClient()
997+
.admin()
998+
.cluster()
999+
.prepareCreateSnapshot(snapshotRepoName, snapshotName)
1000+
.setWaitForCompletion(true)
1001+
.setMasterNodeTimeout(TimeValue.timeValueSeconds(60))
1002+
.get();
1003+
1004+
} catch (Exception ignored) {}
1005+
});
1006+
thread.start();
1007+
1008+
// stop existing master
1009+
final String clusterManagerNode = internalCluster().getClusterManagerName();
1010+
stopNode(clusterManagerNode);
1011+
1012+
// Validate that we have greater one snapshot has been created
1013+
String snapshotName = "new-snapshot";
1014+
try {
1015+
client().admin().cluster().prepareCreateSnapshot(snapshotRepoName, snapshotName).setWaitForCompletion(true).get();
1016+
} catch (Exception e) {
1017+
logger.info("Exception while creating new-snapshot", e);
1018+
}
1019+
1020+
// Validate that snapshot is present in repository data
1021+
assertBusy(() -> {
1022+
GetSnapshotsRequest request = new GetSnapshotsRequest(snapshotRepoName);
1023+
GetSnapshotsResponse response2 = client().admin().cluster().getSnapshots(request).actionGet();
1024+
assertThat(response2.getSnapshots().size(), greaterThanOrEqualTo(1));
1025+
}, 30, TimeUnit.SECONDS);
1026+
thread.join();
1027+
}
1028+
10001029
public void testCreateSnapshotV2WithRedIndex() throws Exception {
10011030
internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
10021031
internalCluster().startDataOnlyNode(pinnedTimestampSettings());
@@ -1315,11 +1344,4 @@ public void testConcurrentV1SnapshotAndV2RepoSettingUpdate() throws Exception {
13151344
createV1SnapshotThread.join();
13161345
}
13171346

1318-
private Settings pinnedTimestampSettings() {
1319-
Settings settings = Settings.builder()
1320-
.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true)
1321-
.build();
1322-
return settings;
1323-
}
1324-
13251347
}

‎server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteSnapshotIT.java

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.remotestore;
10+
11+
import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
12+
import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesResponse;
13+
import org.opensearch.client.Client;
14+
import org.opensearch.cluster.metadata.IndexMetadata;
15+
import org.opensearch.cluster.metadata.RepositoryMetadata;
16+
import org.opensearch.common.settings.Settings;
17+
import org.opensearch.index.IndexSettings;
18+
import org.opensearch.indices.RemoteStoreSettings;
19+
import org.opensearch.repositories.fs.ReloadableFsRepository;
20+
import org.opensearch.snapshots.AbstractSnapshotIntegTestCase;
21+
import org.junit.After;
22+
import org.junit.Before;
23+
24+
import java.nio.file.Path;
25+
import java.util.concurrent.ExecutionException;
26+
27+
import static org.opensearch.repositories.fs.ReloadableFsRepository.REPOSITORIES_FAILRATE_SETTING;
28+
29+
public abstract class RemoteSnapshotIT extends AbstractSnapshotIntegTestCase {
30+
protected static final String BASE_REMOTE_REPO = "test-rs-repo" + TEST_REMOTE_STORE_REPO_SUFFIX;
31+
protected Path remoteRepoPath;
32+
33+
@Before
34+
public void setup() {
35+
remoteRepoPath = randomRepoPath().toAbsolutePath();
36+
}
37+
38+
@After
39+
public void teardown() {
40+
clusterAdmin().prepareCleanupRepository(BASE_REMOTE_REPO).get();
41+
}
42+
43+
@Override
44+
protected Settings nodeSettings(int nodeOrdinal) {
45+
return Settings.builder()
46+
.put(super.nodeSettings(nodeOrdinal))
47+
.put(remoteStoreClusterSettings(BASE_REMOTE_REPO, remoteRepoPath))
48+
.build();
49+
}
50+
51+
protected Settings pinnedTimestampSettings() {
52+
Settings settings = Settings.builder()
53+
.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), true)
54+
.build();
55+
return settings;
56+
}
57+
58+
protected Settings.Builder getIndexSettings(int numOfShards, int numOfReplicas) {
59+
Settings.Builder settingsBuilder = Settings.builder()
60+
.put(super.indexSettings())
61+
.put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numOfShards)
62+
.put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, numOfReplicas)
63+
.put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "300s");
64+
return settingsBuilder;
65+
}
66+
67+
protected void indexDocuments(Client client, String indexName, int numOfDocs) {
68+
indexDocuments(client, indexName, 0, numOfDocs);
69+
}
70+
71+
void indexDocuments(Client client, String indexName, int fromId, int toId) {
72+
for (int i = fromId; i < toId; i++) {
73+
String id = Integer.toString(i);
74+
client.prepareIndex(indexName).setId(id).setSource("text", "sometext").get();
75+
}
76+
client.admin().indices().prepareFlush(indexName).get();
77+
}
78+
79+
protected void setFailRate(String repoName, int value) throws ExecutionException, InterruptedException {
80+
GetRepositoriesRequest gr = new GetRepositoriesRequest(new String[] { repoName });
81+
GetRepositoriesResponse res = client().admin().cluster().getRepositories(gr).get();
82+
RepositoryMetadata rmd = res.repositories().get(0);
83+
Settings.Builder settings = Settings.builder()
84+
.put("location", rmd.settings().get("location"))
85+
.put(REPOSITORIES_FAILRATE_SETTING.getKey(), value);
86+
createRepository(repoName, ReloadableFsRepository.TYPE, settings);
87+
}
88+
89+
}

0 commit comments

Comments
 (0)