diff --git a/CHANGELOG.md b/CHANGELOG.md index 60820ab196261..9f4dbe23b1945 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Introduce a new search node role to hold search only shards ([#17620](https://github.com/opensearch-project/OpenSearch/pull/17620)) - Fix systemd integTest on deb regarding path ownership check ([#17641](https://github.com/opensearch-project/OpenSearch/pull/17641)) - Add dfs transformation function in XContentMapValues ([#17612](https://github.com/opensearch-project/OpenSearch/pull/17612)) +- Add tracking for long-running SearchTask post cancellation ([#17726](https://github.com/opensearch-project/OpenSearch/pull/17726)) - Added Kinesis support as a plugin for the pull-based ingestion ([#17615](https://github.com/opensearch-project/OpenSearch/pull/17615)) - Add FilterFieldType for developers who want to wrap MappedFieldType ([#17627](https://github.com/opensearch-project/OpenSearch/pull/17627)) - [Rule Based Auto-tagging] Add in-memory rule processing service ([#17365](https://github.com/opensearch-project/OpenSearch/pull/17365)) diff --git a/server/src/main/java/org/opensearch/tasks/BaseSearchTaskCancellationStats.java b/server/src/main/java/org/opensearch/tasks/BaseSearchTaskCancellationStats.java new file mode 100644 index 0000000000000..4d9fc982d3df7 --- /dev/null +++ b/server/src/main/java/org/opensearch/tasks/BaseSearchTaskCancellationStats.java @@ -0,0 +1,73 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.tasks; + +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.core.xcontent.ToXContentObject; +import org.opensearch.core.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Objects; + +/** + * Base class for search task cancellation statistics. + */ +public abstract class BaseSearchTaskCancellationStats implements ToXContentObject, Writeable { + + private final long currentLongRunningCancelledTaskCount; + private final long totalLongRunningCancelledTaskCount; + + public BaseSearchTaskCancellationStats(long currentTaskCount, long totalTaskCount) { + this.currentLongRunningCancelledTaskCount = currentTaskCount; + this.totalLongRunningCancelledTaskCount = totalTaskCount; + } + + public BaseSearchTaskCancellationStats(StreamInput in) throws IOException { + this.currentLongRunningCancelledTaskCount = in.readVLong(); + this.totalLongRunningCancelledTaskCount = in.readVLong(); + } + + protected long getCurrentLongRunningCancelledTaskCount() { + return this.currentLongRunningCancelledTaskCount; + } + + protected long getTotalLongRunningCancelledTaskCount() { + return this.totalLongRunningCancelledTaskCount; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("current_count_post_cancel", currentLongRunningCancelledTaskCount); + builder.field("total_count_post_cancel", totalLongRunningCancelledTaskCount); + return builder.endObject(); + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVLong(currentLongRunningCancelledTaskCount); + out.writeVLong(totalLongRunningCancelledTaskCount); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + BaseSearchTaskCancellationStats that = (BaseSearchTaskCancellationStats) o; + return currentLongRunningCancelledTaskCount == that.currentLongRunningCancelledTaskCount + && totalLongRunningCancelledTaskCount == that.totalLongRunningCancelledTaskCount; + } + + @Override + public int hashCode() { + return Objects.hash(currentLongRunningCancelledTaskCount, totalLongRunningCancelledTaskCount); + } +} diff --git a/server/src/main/java/org/opensearch/tasks/SearchShardTaskCancellationStats.java b/server/src/main/java/org/opensearch/tasks/SearchShardTaskCancellationStats.java index e6ce092d7516e..cd13acde7f613 100644 --- a/server/src/main/java/org/opensearch/tasks/SearchShardTaskCancellationStats.java +++ b/server/src/main/java/org/opensearch/tasks/SearchShardTaskCancellationStats.java @@ -9,67 +9,19 @@ package org.opensearch.tasks; import org.opensearch.core.common.io.stream.StreamInput; -import org.opensearch.core.common.io.stream.StreamOutput; -import org.opensearch.core.common.io.stream.Writeable; -import org.opensearch.core.xcontent.ToXContentObject; -import org.opensearch.core.xcontent.XContentBuilder; import java.io.IOException; -import java.util.Objects; /** * Holds monitoring service stats specific to search shard task. */ -public class SearchShardTaskCancellationStats implements ToXContentObject, Writeable { - - private final long currentLongRunningCancelledTaskCount; - private final long totalLongRunningCancelledTaskCount; +public class SearchShardTaskCancellationStats extends BaseSearchTaskCancellationStats { public SearchShardTaskCancellationStats(long currentTaskCount, long totalTaskCount) { - this.currentLongRunningCancelledTaskCount = currentTaskCount; - this.totalLongRunningCancelledTaskCount = totalTaskCount; + super(currentTaskCount, totalTaskCount); } public SearchShardTaskCancellationStats(StreamInput in) throws IOException { - this.currentLongRunningCancelledTaskCount = in.readVLong(); - this.totalLongRunningCancelledTaskCount = in.readVLong(); - } - - // package private for testing - protected long getCurrentLongRunningCancelledTaskCount() { - return this.currentLongRunningCancelledTaskCount; - } - - // package private for testing - protected long getTotalLongRunningCancelledTaskCount() { - return this.totalLongRunningCancelledTaskCount; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field("current_count_post_cancel", currentLongRunningCancelledTaskCount); - builder.field("total_count_post_cancel", totalLongRunningCancelledTaskCount); - return builder.endObject(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeVLong(currentLongRunningCancelledTaskCount); - out.writeVLong(totalLongRunningCancelledTaskCount); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - SearchShardTaskCancellationStats that = (SearchShardTaskCancellationStats) o; - return currentLongRunningCancelledTaskCount == that.currentLongRunningCancelledTaskCount - && totalLongRunningCancelledTaskCount == that.totalLongRunningCancelledTaskCount; - } - - @Override - public int hashCode() { - return Objects.hash(currentLongRunningCancelledTaskCount, totalLongRunningCancelledTaskCount); + super(in); } } diff --git a/server/src/main/java/org/opensearch/tasks/SearchTaskCancellationStats.java b/server/src/main/java/org/opensearch/tasks/SearchTaskCancellationStats.java new file mode 100644 index 0000000000000..9ae9a65877bb7 --- /dev/null +++ b/server/src/main/java/org/opensearch/tasks/SearchTaskCancellationStats.java @@ -0,0 +1,27 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.tasks; + +import org.opensearch.core.common.io.stream.StreamInput; + +import java.io.IOException; + +/** + * Holds monitoring service stats specific to search task. + */ +public class SearchTaskCancellationStats extends BaseSearchTaskCancellationStats { + + public SearchTaskCancellationStats(long currentTaskCount, long totalTaskCount) { + super(currentTaskCount, totalTaskCount); + } + + public SearchTaskCancellationStats(StreamInput in) throws IOException { + super(in); + } +} diff --git a/server/src/main/java/org/opensearch/tasks/TaskCancellationMonitoringService.java b/server/src/main/java/org/opensearch/tasks/TaskCancellationMonitoringService.java index 2040703d88c38..65ac4d3a0db6e 100644 --- a/server/src/main/java/org/opensearch/tasks/TaskCancellationMonitoringService.java +++ b/server/src/main/java/org/opensearch/tasks/TaskCancellationMonitoringService.java @@ -11,6 +11,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.search.SearchShardTask; +import org.opensearch.action.search.SearchTask; import org.opensearch.common.lifecycle.AbstractLifecycleComponent; import org.opensearch.common.metrics.CounterMetric; import org.opensearch.threadpool.Scheduler; @@ -32,7 +33,7 @@ public class TaskCancellationMonitoringService extends AbstractLifecycleComponent implements TaskManager.TaskEventListeners { private static final Logger logger = LogManager.getLogger(TaskCancellationMonitoringService.class); - private final static List> TASKS_TO_TRACK = Arrays.asList(SearchShardTask.class); + private final static List> TASKS_TO_TRACK = Arrays.asList(SearchShardTask.class, SearchTask.class); private volatile Scheduler.Cancellable scheduledFuture; private final ThreadPool threadPool; @@ -146,6 +147,10 @@ public TaskCancellationStats stats() { Map, List> currentRunningCancelledTasks = getCurrentRunningTasksPostCancellation(); return new TaskCancellationStats( + new SearchTaskCancellationStats( + Optional.of(currentRunningCancelledTasks).map(mapper -> mapper.get(SearchTask.class)).map(List::size).orElse(0), + cancellationStatsHolder.get(SearchTask.class).totalLongRunningCancelledTaskCount.count() + ), new SearchShardTaskCancellationStats( Optional.of(currentRunningCancelledTasks).map(mapper -> mapper.get(SearchShardTask.class)).map(List::size).orElse(0), cancellationStatsHolder.get(SearchShardTask.class).totalLongRunningCancelledTaskCount.count() diff --git a/server/src/main/java/org/opensearch/tasks/TaskCancellationStats.java b/server/src/main/java/org/opensearch/tasks/TaskCancellationStats.java index cca898fdd844f..49f4c8eb8e02e 100644 --- a/server/src/main/java/org/opensearch/tasks/TaskCancellationStats.java +++ b/server/src/main/java/org/opensearch/tasks/TaskCancellationStats.java @@ -8,6 +8,7 @@ package org.opensearch.tasks; +import org.opensearch.Version; import org.opensearch.core.common.io.stream.StreamInput; import org.opensearch.core.common.io.stream.StreamOutput; import org.opensearch.core.common.io.stream.Writeable; @@ -22,13 +23,23 @@ */ public class TaskCancellationStats implements ToXContentFragment, Writeable { + private final SearchTaskCancellationStats searchTaskCancellationStats; private final SearchShardTaskCancellationStats searchShardTaskCancellationStats; - public TaskCancellationStats(SearchShardTaskCancellationStats searchShardTaskCancellationStats) { + public TaskCancellationStats( + SearchTaskCancellationStats searchTaskCancellationStats, + SearchShardTaskCancellationStats searchShardTaskCancellationStats + ) { + this.searchTaskCancellationStats = searchTaskCancellationStats; this.searchShardTaskCancellationStats = searchShardTaskCancellationStats; } public TaskCancellationStats(StreamInput in) throws IOException { + if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + searchTaskCancellationStats = new SearchTaskCancellationStats(in); + } else { + searchTaskCancellationStats = new SearchTaskCancellationStats(0, 0); + } searchShardTaskCancellationStats = new SearchShardTaskCancellationStats(in); } @@ -37,15 +48,24 @@ protected SearchShardTaskCancellationStats getSearchShardTaskCancellationStats() return this.searchShardTaskCancellationStats; } + // package private for testing + protected SearchTaskCancellationStats getSearchTaskCancellationStats() { + return this.searchTaskCancellationStats; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject("task_cancellation"); + builder.field("search_task", searchTaskCancellationStats); builder.field("search_shard_task", searchShardTaskCancellationStats); return builder.endObject(); } @Override public void writeTo(StreamOutput out) throws IOException { + if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + searchTaskCancellationStats.writeTo(out); + } searchShardTaskCancellationStats.writeTo(out); } @@ -54,11 +74,12 @@ public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; TaskCancellationStats that = (TaskCancellationStats) o; - return Objects.equals(searchShardTaskCancellationStats, that.searchShardTaskCancellationStats); + return Objects.equals(searchTaskCancellationStats, that.searchTaskCancellationStats) + && Objects.equals(searchShardTaskCancellationStats, that.searchShardTaskCancellationStats); } @Override public int hashCode() { - return Objects.hash(searchShardTaskCancellationStats); + return Objects.hash(searchTaskCancellationStats, searchShardTaskCancellationStats); } } diff --git a/server/src/test/java/org/opensearch/tasks/SearchTaskCancellationStatsTests.java b/server/src/test/java/org/opensearch/tasks/SearchTaskCancellationStatsTests.java new file mode 100644 index 0000000000000..e4533582ca7f3 --- /dev/null +++ b/server/src/test/java/org/opensearch/tasks/SearchTaskCancellationStatsTests.java @@ -0,0 +1,28 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.tasks; + +import org.opensearch.core.common.io.stream.Writeable; +import org.opensearch.test.AbstractWireSerializingTestCase; + +public class SearchTaskCancellationStatsTests extends AbstractWireSerializingTestCase { + @Override + protected Writeable.Reader instanceReader() { + return SearchTaskCancellationStats::new; + } + + @Override + protected SearchTaskCancellationStats createTestInstance() { + return randomInstance(); + } + + public static SearchTaskCancellationStats randomInstance() { + return new SearchTaskCancellationStats(randomNonNegativeLong(), randomNonNegativeLong()); + } +} diff --git a/server/src/test/java/org/opensearch/tasks/TaskCancellationMonitoringServiceTests.java b/server/src/test/java/org/opensearch/tasks/TaskCancellationMonitoringServiceTests.java index bb154b95f9f01..a10d5bd285c8b 100644 --- a/server/src/test/java/org/opensearch/tasks/TaskCancellationMonitoringServiceTests.java +++ b/server/src/test/java/org/opensearch/tasks/TaskCancellationMonitoringServiceTests.java @@ -10,6 +10,7 @@ import org.opensearch.Version; import org.opensearch.action.search.SearchShardTask; +import org.opensearch.action.search.SearchTask; import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.Settings; import org.opensearch.core.common.io.stream.StreamOutput; @@ -31,6 +32,7 @@ import java.util.concurrent.CountDownLatch; import java.util.concurrent.Phaser; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import static org.opensearch.tasks.TaskCancellationMonitoringSettings.DURATION_MILLIS_SETTING; import static org.mockito.ArgumentMatchers.any; @@ -79,6 +81,17 @@ public void testWithNoCurrentRunningCancelledTasks() { } public void testWithNonZeroCancelledSearchShardTasksRunning() throws InterruptedException { + testWithNonZeroCancelledTasksRunning(SearchShardTask.class, TaskCancellationStats::getSearchShardTaskCancellationStats); + } + + public void testWithNonZeroCancelledSearchTasksRunning() throws InterruptedException { + testWithNonZeroCancelledTasksRunning(SearchTask.class, TaskCancellationStats::getSearchTaskCancellationStats); + } + + private void testWithNonZeroCancelledTasksRunning( + Class taskType, + Function statsExtractor + ) throws InterruptedException { Settings settings = Settings.builder() .put(DURATION_MILLIS_SETTING.getKey(), 0) // Setting to zero for testing .build(); @@ -93,7 +106,7 @@ public void testWithNonZeroCancelledSearchShardTasksRunning() throws Interrupted taskCancellationMonitoringSettings ); int numTasks = randomIntBetween(5, 50); - List tasks = createTasks(numTasks); + List tasks = createTasks(taskType, numTasks); int cancelFromIdx = randomIntBetween(0, numTasks - 1); int cancelTillIdx = randomIntBetween(cancelFromIdx, numTasks - 1); @@ -105,19 +118,19 @@ public void testWithNonZeroCancelledSearchShardTasksRunning() throws Interrupted taskCancellationMonitoringService.doRun(); // 1st run to verify whether we are able to track running cancelled // tasks. TaskCancellationStats stats = taskCancellationMonitoringService.stats(); - assertEquals(numberOfTasksCancelled, stats.getSearchShardTaskCancellationStats().getCurrentLongRunningCancelledTaskCount()); - assertEquals(numberOfTasksCancelled, stats.getSearchShardTaskCancellationStats().getTotalLongRunningCancelledTaskCount()); + assertEquals(numberOfTasksCancelled, statsExtractor.apply(stats).getCurrentLongRunningCancelledTaskCount()); + assertEquals(numberOfTasksCancelled, statsExtractor.apply(stats).getTotalLongRunningCancelledTaskCount()); taskCancellationMonitoringService.doRun(); // 2nd run. Verify same. stats = taskCancellationMonitoringService.stats(); - assertEquals(numberOfTasksCancelled, stats.getSearchShardTaskCancellationStats().getCurrentLongRunningCancelledTaskCount()); - assertEquals(numberOfTasksCancelled, stats.getSearchShardTaskCancellationStats().getTotalLongRunningCancelledTaskCount()); + assertEquals(numberOfTasksCancelled, statsExtractor.apply(stats).getCurrentLongRunningCancelledTaskCount()); + assertEquals(numberOfTasksCancelled, statsExtractor.apply(stats).getTotalLongRunningCancelledTaskCount()); completeTasksConcurrently(tasks, 0, tasks.size() - 1).await(); taskCancellationMonitoringService.doRun(); // 3rd run to verify current count is 0 and total remains the same. stats = taskCancellationMonitoringService.stats(); assertTrue(taskCancellationMonitoringService.getCancelledTaskTracker().isEmpty()); - assertEquals(0, stats.getSearchShardTaskCancellationStats().getCurrentLongRunningCancelledTaskCount()); - assertEquals(numberOfTasksCancelled, stats.getSearchShardTaskCancellationStats().getTotalLongRunningCancelledTaskCount()); + assertEquals(0, statsExtractor.apply(stats).getCurrentLongRunningCancelledTaskCount()); + assertEquals(numberOfTasksCancelled, statsExtractor.apply(stats).getTotalLongRunningCancelledTaskCount()); } public void testShouldRunGetsDisabledAfterTaskCompletion() throws InterruptedException { @@ -138,7 +151,7 @@ public void testShouldRunGetsDisabledAfterTaskCompletion() throws InterruptedExc // Start few tasks. int numTasks = randomIntBetween(5, 50); - List tasks = createTasks(numTasks); + List tasks = createTasks(SearchShardTask.class, numTasks); taskCancellationMonitoringService.doRun(); TaskCancellationStats stats = taskCancellationMonitoringService.stats(); @@ -176,7 +189,7 @@ public void testWithVaryingCancelledTasksDuration() throws InterruptedException ); int numTasks = randomIntBetween(5, 50); - List tasks = createTasks(numTasks); + List tasks = createTasks(SearchShardTask.class, numTasks); int numTasksToBeCancelledInFirstIteration = randomIntBetween(1, numTasks - 1); CountDownLatch countDownLatch = cancelTasksConcurrently(tasks, 0, numTasksToBeCancelledInFirstIteration - 1); @@ -232,10 +245,13 @@ public void testTasksAreGettingEvictedCorrectlyAfterCompletion() throws Interrup ); // Start few tasks. - int numTasks = randomIntBetween(5, 50); - List tasks = createTasks(numTasks); + int searchShardTaskNum = randomIntBetween(5, 50); + int searchTaskNum = randomIntBetween(5, 50); + int taskNum = searchShardTaskNum + searchTaskNum; + List tasks = new ArrayList<>(createTasks(SearchShardTask.class, searchShardTaskNum)); + tasks.addAll(createTasks(SearchTask.class, searchTaskNum)); assertTrue(taskCancellationMonitoringService.getCancelledTaskTracker().isEmpty()); - int numTasksToBeCancelledInFirstIteration = randomIntBetween(2, numTasks - 1); + int numTasksToBeCancelledInFirstIteration = randomIntBetween(2, taskNum - 1); CountDownLatch countDownLatch = cancelTasksConcurrently(tasks, 0, numTasksToBeCancelledInFirstIteration - 1); countDownLatch.await(); // Wait for all tasks to be cancelled in first iteration @@ -245,7 +261,7 @@ public void testTasksAreGettingEvictedCorrectlyAfterCompletion() throws Interrup assertTrue(taskCancellationMonitoringService.getCancelledTaskTracker().containsKey(tasks.get(itr).getId())); } // Cancel rest of the tasks - cancelTasksConcurrently(tasks, numTasksToBeCancelledInFirstIteration, numTasks - 1).await(); + cancelTasksConcurrently(tasks, numTasksToBeCancelledInFirstIteration, taskNum - 1).await(); for (int itr = 0; itr < tasks.size(); itr++) { assertTrue(taskCancellationMonitoringService.getCancelledTaskTracker().containsKey(tasks.get(itr).getId())); } @@ -294,15 +310,16 @@ public void testDoStartAndStop() { verify(scheduleFuture, times(1)).cancel(); } - private List createTasks(int numTasks) { - List tasks = new ArrayList<>(numTasks); + @SuppressWarnings("unchecked") + private List createTasks(Class taskType, int numTasks) { + List tasks = new ArrayList<>(numTasks); for (int i = 0; i < numTasks; i++) { - tasks.add((SearchShardTask) taskManager.register("type-" + i, "action-" + i, new MockQuerySearchRequest())); + tasks.add((T) taskManager.register("type-" + i, "action-" + i, new MockQuerySearchRequest(taskType))); } return tasks; } - // Caller can this with the list of tasks specifically mentioning which ones to cancel. And can call CountDownLatch + // Caller can call this method with a list of tasks specifically mentioning which ones to cancel. And can call CountDownLatch // .await() to wait for all tasks be cancelled. private CountDownLatch cancelTasksConcurrently(List tasks, int cancelFromIdx, int cancelTillIdx) { assert cancelFromIdx >= 0; @@ -347,10 +364,12 @@ private CountDownLatch completeTasksConcurrently(List } public static class MockQuerySearchRequest extends TransportRequest { + private Class taskType; protected String requestName; - public MockQuerySearchRequest() { + public MockQuerySearchRequest(Class taskType) { super(); + this.taskType = taskType; } @Override @@ -366,8 +385,13 @@ public String getDescription() { @Override public Task createTask(long id, String type, String action, TaskId parentTaskId, Map headers) { - return new SearchShardTask(id, type, action, getDescription(), parentTaskId, headers); + if (taskType == SearchTask.class) { + return new SearchTask(id, type, action, this::getDescription, parentTaskId, headers); + } else if (taskType == SearchShardTask.class) { + return new SearchShardTask(id, type, action, getDescription(), parentTaskId, headers); + } else { + throw new IllegalArgumentException("Unsupported task type: " + taskType); + } } } - } diff --git a/server/src/test/java/org/opensearch/tasks/TaskCancellationStatsTests.java b/server/src/test/java/org/opensearch/tasks/TaskCancellationStatsTests.java index beef05b04ab7a..7c3dbe900cb1f 100644 --- a/server/src/test/java/org/opensearch/tasks/TaskCancellationStatsTests.java +++ b/server/src/test/java/org/opensearch/tasks/TaskCancellationStatsTests.java @@ -23,6 +23,9 @@ protected TaskCancellationStats createTestInstance() { } public static TaskCancellationStats randomInstance() { - return new TaskCancellationStats(SearchShardTaskCancellationStatsTests.randomInstance()); + return new TaskCancellationStats( + SearchTaskCancellationStatsTests.randomInstance(), + SearchShardTaskCancellationStatsTests.randomInstance() + ); } }