Skip to content

Commit 46b5e1f

Browse files
gbbafnagbbafna
committed
more changes
Signed-off-by: Gaurav Bafna <[email protected]>
1 parent 8313642 commit 46b5e1f

File tree

5 files changed

+186
-283
lines changed

5 files changed

+186
-283
lines changed

‎server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1062,7 +1062,7 @@ public void testConcurrentSnapshotV2CreateOperation_MasterChange() throws Interr
10621062
assertThat(repositoryData.getSnapshotIds().size(), greaterThanOrEqualTo(1));
10631063
thread.join();
10641064
}
1065-
1065+
10661066
public void testCreateSnapshotV2WithRedIndex() throws Exception {
10671067
internalCluster().startClusterManagerOnlyNode(pinnedTimestampSettings());
10681068
internalCluster().startDataOnlyNode(pinnedTimestampSettings());

‎server/src/internalClusterTest/java/org/opensearch/snapshots/CloneSnapshotV2IT.java

Lines changed: 58 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,15 @@
3434
import org.opensearch.action.ActionRunnable;
3535
import org.opensearch.action.DocWriteResponse;
3636
import org.opensearch.action.admin.cluster.snapshots.create.CreateSnapshotResponse;
37+
import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsRequest;
38+
import org.opensearch.action.admin.cluster.snapshots.get.GetSnapshotsResponse;
3739
import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
3840
import org.opensearch.action.delete.DeleteResponse;
3941
import org.opensearch.action.support.PlainActionFuture;
4042
import org.opensearch.action.support.master.AcknowledgedResponse;
4143
import org.opensearch.client.Client;
44+
import org.opensearch.cluster.metadata.Metadata;
45+
import org.opensearch.cluster.metadata.RepositoriesMetadata;
4246
import org.opensearch.common.settings.Settings;
4347
import org.opensearch.core.common.unit.ByteSizeUnit;
4448
import org.opensearch.core.rest.RestStatus;
@@ -50,8 +54,10 @@
5054
import org.opensearch.test.OpenSearchIntegTestCase;
5155

5256
import java.nio.file.Path;
57+
import java.util.concurrent.ExecutionException;
58+
import java.util.concurrent.TimeUnit;
59+
import java.util.concurrent.atomic.AtomicReference;
5360

54-
import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings;
5561
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
5662
import static org.hamcrest.Matchers.equalTo;
5763
import static org.hamcrest.Matchers.greaterThan;
@@ -134,27 +140,32 @@ public void testCloneShallowCopyV2() throws Exception {
134140
assertTrue(response.isAcknowledged());
135141
awaitClusterManagerFinishRepoOperations();
136142

143+
AtomicReference<SnapshotId> cloneSnapshotId = new AtomicReference<>();
137144
// Validate that snapshot is present in repository data
138-
PlainActionFuture<RepositoryData> repositoryDataPlainActionFutureClone = new PlainActionFuture<>();
139-
repository.getRepositoryData(repositoryDataPlainActionFutureClone);
140-
141-
repositoryData = repositoryDataPlainActionFutureClone.get();
142-
assertEquals(repositoryData.getSnapshotIds().size(), 2);
143-
boolean foundCloneInRepoData = false;
144-
SnapshotId cloneSnapshotId = null;
145-
for (SnapshotId snapshotId : repositoryData.getSnapshotIds()) {
146-
if (snapshotId.getName().equals("test_clone_snapshot1")) {
147-
foundCloneInRepoData = true;
148-
cloneSnapshotId = snapshotId;
145+
waitUntil(() -> {
146+
PlainActionFuture<RepositoryData> repositoryDataPlainActionFutureClone = new PlainActionFuture<>();
147+
repository.getRepositoryData(repositoryDataPlainActionFutureClone);
148+
149+
RepositoryData repositoryData1;
150+
try {
151+
repositoryData1 = repositoryDataPlainActionFutureClone.get();
152+
} catch (InterruptedException | ExecutionException e) {
153+
throw new RuntimeException(e);
149154
}
150-
}
151-
final SnapshotId cloneSnapshotIdFinal = cloneSnapshotId;
155+
for (SnapshotId snapshotId : repositoryData1.getSnapshotIds()) {
156+
if (snapshotId.getName().equals("test_clone_snapshot1")) {
157+
cloneSnapshotId.set(snapshotId);
158+
return true;
159+
}
160+
}
161+
return false;
162+
}, 90, TimeUnit.SECONDS);
163+
164+
final SnapshotId cloneSnapshotIdFinal = cloneSnapshotId.get();
152165
SnapshotInfo cloneSnapshotInfo = PlainActionFuture.get(
153166
f -> repository.threadPool().generic().execute(ActionRunnable.supply(f, () -> repository.getSnapshotInfo(cloneSnapshotIdFinal)))
154167
);
155168

156-
assertTrue(foundCloneInRepoData);
157-
158169
assertThat(cloneSnapshotInfo.getPinnedTimestamp(), equalTo(sourceSnapshotInfo.getPinnedTimestamp()));
159170
for (String index : sourceSnapshotInfo.indices()) {
160171
assertTrue(cloneSnapshotInfo.indices().contains(index));
@@ -259,19 +270,21 @@ public void testCloneShallowCopyAfterDisablingV2() throws Exception {
259270
assertThat(sourceSnapshotInfoV1.state(), equalTo(SnapshotState.SUCCESS));
260271
assertThat(sourceSnapshotInfoV1.successfulShards(), greaterThan(0));
261272
assertThat(sourceSnapshotInfoV1.successfulShards(), equalTo(sourceSnapshotInfoV1.totalShards()));
262-
assertThat(sourceSnapshotInfoV1.getPinnedTimestamp(), equalTo(0L));
273+
// assertThat(sourceSnapshotInfoV1.getPinnedTimestamp(), equalTo(0L));
274+
AtomicReference<RepositoryData> repositoryDataAtomicReference = new AtomicReference<>();
275+
awaitClusterManagerFinishRepoOperations();
263276

264277
// Validate that snapshot is present in repository data
265-
PlainActionFuture<RepositoryData> repositoryDataV1PlainActionFuture = new PlainActionFuture<>();
266-
BlobStoreRepository repositoryV1 = (BlobStoreRepository) internalCluster().getCurrentClusterManagerNodeInstance(
267-
RepositoriesService.class
268-
).repository(snapshotRepoName);
269-
repositoryV1.getRepositoryData(repositoryDataV1PlainActionFuture);
270-
271-
repositoryData = repositoryDataV1PlainActionFuture.get();
278+
assertBusy(() -> {
279+
Metadata metadata = clusterAdmin().prepareState().get().getState().metadata();
280+
RepositoriesMetadata repositoriesMetadata = metadata.custom(RepositoriesMetadata.TYPE);
281+
assertEquals(1, repositoriesMetadata.repository(snapshotRepoName).generation());
282+
assertEquals(1, repositoriesMetadata.repository(snapshotRepoName).pendingGeneration());
272283

273-
assertTrue(repositoryData.getSnapshotIds().contains(sourceSnapshotInfoV1.snapshotId()));
274-
assertEquals(repositoryData.getSnapshotIds().size(), 2);
284+
GetSnapshotsRequest request = new GetSnapshotsRequest(snapshotRepoName);
285+
GetSnapshotsResponse response = client().admin().cluster().getSnapshots(request).actionGet();
286+
assertEquals(2, response.getSnapshots().size());
287+
}, 30, TimeUnit.SECONDS);
275288

276289
// clone should get created for v2 snapshot
277290
AcknowledgedResponse response = client().admin()
@@ -289,31 +302,28 @@ public void testCloneShallowCopyAfterDisablingV2() throws Exception {
289302
).repository(snapshotRepoName);
290303
repositoryCloneV2.getRepositoryData(repositoryDataCloneV2PlainActionFuture);
291304

292-
repositoryData = repositoryDataCloneV2PlainActionFuture.get();
293-
294-
assertEquals(repositoryData.getSnapshotIds().size(), 3);
295-
boolean foundCloneInRepoData = false;
296-
SnapshotId cloneSnapshotId = null;
297-
for (SnapshotId snapshotId : repositoryData.getSnapshotIds()) {
298-
if (snapshotId.getName().equals(cloneSnapshotV2)) {
299-
foundCloneInRepoData = true;
300-
cloneSnapshotId = snapshotId;
301-
}
302-
}
303-
final SnapshotId cloneSnapshotIdFinal = cloneSnapshotId;
304-
SnapshotInfo cloneSnapshotInfo = PlainActionFuture.get(
305-
f -> repository.threadPool().generic().execute(ActionRunnable.supply(f, () -> repository.getSnapshotInfo(cloneSnapshotIdFinal)))
306-
);
305+
// Validate that snapshot is present in repository data
306+
assertBusy(() -> {
307+
Metadata metadata = clusterAdmin().prepareState().get().getState().metadata();
308+
RepositoriesMetadata repositoriesMetadata = metadata.custom(RepositoriesMetadata.TYPE);
309+
assertEquals(2, repositoriesMetadata.repository(snapshotRepoName).generation());
310+
assertEquals(2, repositoriesMetadata.repository(snapshotRepoName).pendingGeneration());
311+
GetSnapshotsRequest request = new GetSnapshotsRequest(snapshotRepoName);
312+
GetSnapshotsResponse response2 = client().admin().cluster().getSnapshots(request).actionGet();
313+
assertEquals(3, response2.getSnapshots().size());
314+
}, 30, TimeUnit.SECONDS);
307315

308-
assertTrue(foundCloneInRepoData);
309316
// pinned timestamp value in clone snapshot v2 matches source snapshot v2
310-
assertThat(cloneSnapshotInfo.getPinnedTimestamp(), equalTo(sourceSnapshotInfo.getPinnedTimestamp()));
311-
for (String index : sourceSnapshotInfo.indices()) {
312-
assertTrue(cloneSnapshotInfo.indices().contains(index));
313-
317+
GetSnapshotsRequest request = new GetSnapshotsRequest(snapshotRepoName, new String[] { sourceSnapshotV2, cloneSnapshotV2 });
318+
GetSnapshotsResponse response2 = client().admin().cluster().getSnapshots(request).actionGet();
319+
320+
SnapshotInfo sourceInfo = response2.getSnapshots().get(0);
321+
SnapshotInfo cloneInfo = response2.getSnapshots().get(1);
322+
assertEquals(sourceInfo.getPinnedTimestamp(), cloneInfo.getPinnedTimestamp());
323+
assertEquals(sourceInfo.totalShards(), cloneInfo.totalShards());
324+
for (String index : sourceInfo.indices()) {
325+
assertTrue(cloneInfo.indices().contains(index));
314326
}
315-
assertThat(cloneSnapshotInfo.totalShards(), equalTo(sourceSnapshotInfo.totalShards()));
316-
317327
}
318328

319329
public void testRestoreFromClone() throws Exception {

‎server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -231,9 +231,8 @@ public static Entry startClone(
231231
source,
232232
Map.of(),
233233
remoteStoreIndexShallowCopyV2,
234-
remoteStoreIndexShallowCopyV2// initialising to false, will be updated in startCloning method of SnapshotsService while updating
235-
// entry with
236-
// clone jobs
234+
remoteStoreIndexShallowCopyV2// initialising to false, will be updated in startCloning method of SnapshotsService
235+
// while updating entry with clone jobs
237236
);
238237
}
239238

‎server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -824,7 +824,7 @@ public void updateState(ClusterState state) {
824824
+ "] to generation ["
825825
+ metadata.generation()
826826
+ "]";
827-
logger.debug("Updated repository generation from [{}] to [{}]", previousBest, metadata.generation());
827+
logger.info("Updated repository generation from [{}] to [{}]", previousBest, metadata.generation());
828828
}
829829
}
830830
}
@@ -2916,6 +2916,8 @@ public void getRepositoryData(ActionListener<RepositoryData> listener) {
29162916
}
29172917
final Tuple<Long, BytesReference> cached = latestKnownRepositoryData.get();
29182918
// Fast path loading repository data directly from cache if we're in fully consistent mode and the cache matches up with
2919+
// Fast path loading repository data directly from cache if we're in fully consistent mode and the cache matches up with
2920+
// Fast path loading repository data directly from cache if we're in fully consistent mode and the cache matches up with
29192921
// the latest known repository generation
29202922
if (bestEffortConsistency == false && cached != null && cached.v1() == latestKnownRepoGen.get()) {
29212923
try {

0 commit comments

Comments
 (0)