Skip to content

Commit 2016216

Browse files
ansjcykkewwei
authored andcommitted
[Backport 2.x] Core and Plugin changes for query-level resource usages tracking (opensearch-project#14085)
* Query-level resource usages tracking (opensearch-project#13172) * Query-level resource usages tracking Signed-off-by: Chenyang Ji <[email protected]> * Moving TaskResourceTrackingService to clusterService Signed-off-by: Chenyang Ji <[email protected]> * use shard response header to piggyback task resource usages Signed-off-by: Chenyang Ji <[email protected]> * split changes for query insights plugin Signed-off-by: Chenyang Ji <[email protected]> * improve the supplier logic and other misc items Signed-off-by: Chenyang Ji <[email protected]> * track resource usage for failed requests Signed-off-by: Chenyang Ji <[email protected]> * move resource usages interactions into TaskResourceTrackingService Signed-off-by: Chenyang Ji <[email protected]> --------- Signed-off-by: Chenyang Ji <[email protected]> (cherry picked from commit 3d1fa98) * fix concurrent modification issue in thread context (opensearch-project#14084) Signed-off-by: Chenyang Ji <[email protected]> (cherry picked from commit c8f0b6d) * consume query level cpu and memory usage in query insights (opensearch-project#13739) * consume query level cpu and memory usage in query insights Signed-off-by: Chenyang Ji <[email protected]> * handle failed requests metrics in query insights Signed-off-by: Chenyang Ji <[email protected]> * refactor the code to make it more maintainable Signed-off-by: Chenyang Ji <[email protected]> --------- Signed-off-by: Chenyang Ji <[email protected]> (cherry picked from commit 04a417a) * fix japicmp check for threadContext Signed-off-by: Chenyang Ji <[email protected]> (cherry picked from commit b403fdc) Signed-off-by: kkewwei <[email protected]>
1 parent a3ce11d commit 2016216

File tree

44 files changed

+1200
-164
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+1200
-164
lines changed

CHANGELOG.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
2121
- Move Remote Store Migration from DocRep to GA and modify remote migration settings name ([#14100](https://github.com/opensearch-project/OpenSearch/pull/14100))
2222
- [Remote State] Add async remote state deletion task running on an interval, configurable by a setting ([#13995](https://github.com/opensearch-project/OpenSearch/pull/13995))
2323
- Add remote routing table for remote state publication with experimental feature flag ([#13304](https://github.com/opensearch-project/OpenSearch/pull/13304))
24+
- Add support for query level resource usage tracking ([#13172](https://github.com/opensearch-project/OpenSearch/pull/13172))
25+
- [Query Insights] Add cpu and memory metrics to top n queries ([#13739](https://github.com/opensearch-project/OpenSearch/pull/13739))
2426

2527
### Dependencies
2628
- Bump `com.github.spullara.mustache.java:compiler` from 0.9.10 to 0.9.13 ([#13329](https://github.com/opensearch-project/OpenSearch/pull/13329), [#13559](https://github.com/opensearch-project/OpenSearch/pull/13559))

libs/core/src/main/java/org/opensearch/core/tasks/resourcetracker/ResourceUsageInfo.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,10 @@ public long getTotalValue() {
104104
return endValue.get() - startValue;
105105
}
106106

107+
public long getStartValue() {
108+
return startValue;
109+
}
110+
107111
@Override
108112
public String toString() {
109113
return String.valueOf(getTotalValue());
Lines changed: 225 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,225 @@
1+
/*
2+
* SPDX-License-Identifier: Apache-2.0
3+
*
4+
* The OpenSearch Contributors require contributions made to
5+
* this file be licensed under the Apache-2.0 license or a
6+
* compatible open source license.
7+
*/
8+
9+
package org.opensearch.core.tasks.resourcetracker;
10+
11+
import org.opensearch.common.annotation.PublicApi;
12+
import org.opensearch.core.ParseField;
13+
import org.opensearch.core.common.Strings;
14+
import org.opensearch.core.common.io.stream.StreamInput;
15+
import org.opensearch.core.common.io.stream.StreamOutput;
16+
import org.opensearch.core.common.io.stream.Writeable;
17+
import org.opensearch.core.xcontent.ConstructingObjectParser;
18+
import org.opensearch.core.xcontent.MediaTypeRegistry;
19+
import org.opensearch.core.xcontent.ToXContentObject;
20+
import org.opensearch.core.xcontent.XContentBuilder;
21+
22+
import java.io.IOException;
23+
import java.util.Objects;
24+
25+
import static org.opensearch.core.xcontent.ConstructingObjectParser.constructorArg;
26+
27+
/**
28+
* Task resource usage information with minimal information about the task
29+
* <p>
30+
* Writeable TaskResourceInfo objects are used to represent resource usage
31+
* information of running tasks, which can be propagated to coordinator node
32+
* to infer query-level resource usage
33+
*
34+
* @opensearch.api
35+
*/
36+
@PublicApi(since = "2.15.0")
37+
public class TaskResourceInfo implements Writeable, ToXContentObject {
38+
private final String action;
39+
private final long taskId;
40+
private final long parentTaskId;
41+
private final String nodeId;
42+
private final TaskResourceUsage taskResourceUsage;
43+
44+
private static final ParseField ACTION = new ParseField("action");
45+
private static final ParseField TASK_ID = new ParseField("taskId");
46+
private static final ParseField PARENT_TASK_ID = new ParseField("parentTaskId");
47+
private static final ParseField NODE_ID = new ParseField("nodeId");
48+
private static final ParseField TASK_RESOURCE_USAGE = new ParseField("taskResourceUsage");
49+
50+
public TaskResourceInfo(
51+
final String action,
52+
final long taskId,
53+
final long parentTaskId,
54+
final String nodeId,
55+
final TaskResourceUsage taskResourceUsage
56+
) {
57+
this.action = action;
58+
this.taskId = taskId;
59+
this.parentTaskId = parentTaskId;
60+
this.nodeId = nodeId;
61+
this.taskResourceUsage = taskResourceUsage;
62+
}
63+
64+
public static final ConstructingObjectParser<TaskResourceInfo, Void> PARSER = new ConstructingObjectParser<>(
65+
"task_resource_info",
66+
a -> new Builder().setAction((String) a[0])
67+
.setTaskId((Long) a[1])
68+
.setParentTaskId((Long) a[2])
69+
.setNodeId((String) a[3])
70+
.setTaskResourceUsage((TaskResourceUsage) a[4])
71+
.build()
72+
);
73+
74+
static {
75+
PARSER.declareString(constructorArg(), ACTION);
76+
PARSER.declareLong(constructorArg(), TASK_ID);
77+
PARSER.declareLong(constructorArg(), PARENT_TASK_ID);
78+
PARSER.declareString(constructorArg(), NODE_ID);
79+
PARSER.declareObject(constructorArg(), TaskResourceUsage.PARSER, TASK_RESOURCE_USAGE);
80+
}
81+
82+
@Override
83+
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
84+
builder.startObject();
85+
builder.field(ACTION.getPreferredName(), this.action);
86+
builder.field(TASK_ID.getPreferredName(), this.taskId);
87+
builder.field(PARENT_TASK_ID.getPreferredName(), this.parentTaskId);
88+
builder.field(NODE_ID.getPreferredName(), this.nodeId);
89+
builder.startObject(TASK_RESOURCE_USAGE.getPreferredName());
90+
this.taskResourceUsage.toXContent(builder, params);
91+
builder.endObject();
92+
builder.endObject();
93+
return builder;
94+
}
95+
96+
/**
97+
* Builder for {@link TaskResourceInfo}
98+
*/
99+
public static class Builder {
100+
private TaskResourceUsage taskResourceUsage;
101+
private String action;
102+
private long taskId;
103+
private long parentTaskId;
104+
private String nodeId;
105+
106+
public Builder setTaskResourceUsage(final TaskResourceUsage taskResourceUsage) {
107+
this.taskResourceUsage = taskResourceUsage;
108+
return this;
109+
}
110+
111+
public Builder setAction(final String action) {
112+
this.action = action;
113+
return this;
114+
}
115+
116+
public Builder setTaskId(final long taskId) {
117+
this.taskId = taskId;
118+
return this;
119+
}
120+
121+
public Builder setParentTaskId(final long parentTaskId) {
122+
this.parentTaskId = parentTaskId;
123+
return this;
124+
}
125+
126+
public Builder setNodeId(final String nodeId) {
127+
this.nodeId = nodeId;
128+
return this;
129+
}
130+
131+
public TaskResourceInfo build() {
132+
return new TaskResourceInfo(action, taskId, parentTaskId, nodeId, taskResourceUsage);
133+
}
134+
}
135+
136+
/**
137+
* Read task info from a stream.
138+
*
139+
* @param in StreamInput to read
140+
* @return {@link TaskResourceInfo}
141+
* @throws IOException IOException
142+
*/
143+
public static TaskResourceInfo readFromStream(StreamInput in) throws IOException {
144+
return new TaskResourceInfo.Builder().setAction(in.readString())
145+
.setTaskId(in.readLong())
146+
.setParentTaskId(in.readLong())
147+
.setNodeId(in.readString())
148+
.setTaskResourceUsage(TaskResourceUsage.readFromStream(in))
149+
.build();
150+
}
151+
152+
/**
153+
* Get TaskResourceUsage
154+
*
155+
* @return taskResourceUsage
156+
*/
157+
public TaskResourceUsage getTaskResourceUsage() {
158+
return taskResourceUsage;
159+
}
160+
161+
/**
162+
* Get parent task id
163+
*
164+
* @return parent task id
165+
*/
166+
public long getParentTaskId() {
167+
return parentTaskId;
168+
}
169+
170+
/**
171+
* Get task id
172+
* @return task id
173+
*/
174+
public long getTaskId() {
175+
return taskId;
176+
}
177+
178+
/**
179+
* Get node id
180+
* @return node id
181+
*/
182+
public String getNodeId() {
183+
return nodeId;
184+
}
185+
186+
/**
187+
* Get task action
188+
* @return task action
189+
*/
190+
public String getAction() {
191+
return action;
192+
}
193+
194+
@Override
195+
public void writeTo(StreamOutput out) throws IOException {
196+
out.writeString(action);
197+
out.writeLong(taskId);
198+
out.writeLong(parentTaskId);
199+
out.writeString(nodeId);
200+
taskResourceUsage.writeTo(out);
201+
}
202+
203+
@Override
204+
public String toString() {
205+
return Strings.toString(MediaTypeRegistry.JSON, this);
206+
}
207+
208+
@Override
209+
public boolean equals(Object obj) {
210+
if (obj == null || obj.getClass() != TaskResourceInfo.class) {
211+
return false;
212+
}
213+
TaskResourceInfo other = (TaskResourceInfo) obj;
214+
return action.equals(other.action)
215+
&& taskId == other.taskId
216+
&& parentTaskId == other.parentTaskId
217+
&& Objects.equals(nodeId, other.nodeId)
218+
&& taskResourceUsage.equals(other.taskResourceUsage);
219+
}
220+
221+
@Override
222+
public int hashCode() {
223+
return Objects.hash(action, taskId, parentTaskId, nodeId, taskResourceUsage);
224+
}
225+
}

plugins/query-insights/src/main/java/org/opensearch/plugin/insights/QueryInsightsPlugin.java

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,15 @@ public List<Setting<?>> getSettings() {
111111
QueryInsightsSettings.TOP_N_LATENCY_QUERIES_ENABLED,
112112
QueryInsightsSettings.TOP_N_LATENCY_QUERIES_SIZE,
113113
QueryInsightsSettings.TOP_N_LATENCY_QUERIES_WINDOW_SIZE,
114-
QueryInsightsSettings.TOP_N_LATENCY_EXPORTER_SETTINGS
114+
QueryInsightsSettings.TOP_N_LATENCY_EXPORTER_SETTINGS,
115+
QueryInsightsSettings.TOP_N_CPU_QUERIES_ENABLED,
116+
QueryInsightsSettings.TOP_N_CPU_QUERIES_SIZE,
117+
QueryInsightsSettings.TOP_N_CPU_QUERIES_WINDOW_SIZE,
118+
QueryInsightsSettings.TOP_N_CPU_EXPORTER_SETTINGS,
119+
QueryInsightsSettings.TOP_N_MEMORY_QUERIES_ENABLED,
120+
QueryInsightsSettings.TOP_N_MEMORY_QUERIES_SIZE,
121+
QueryInsightsSettings.TOP_N_MEMORY_QUERIES_WINDOW_SIZE,
122+
QueryInsightsSettings.TOP_N_MEMORY_EXPORTER_SETTINGS
115123
);
116124
}
117125
}

plugins/query-insights/src/main/java/org/opensearch/plugin/insights/core/exporter/QueryInsightsExporterFactory.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
import java.util.Locale;
2020
import java.util.Set;
2121

22-
import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.DEFAULT_TOP_N_LATENCY_QUERIES_INDEX_PATTERN;
22+
import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.DEFAULT_TOP_N_QUERIES_INDEX_PATTERN;
2323
import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.DEFAULT_TOP_QUERIES_EXPORTER_TYPE;
2424
import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.EXPORTER_TYPE;
2525
import static org.opensearch.plugin.insights.settings.QueryInsightsSettings.EXPORT_INDEX;
@@ -71,7 +71,7 @@ public void validateExporterConfig(final Settings settings) throws IllegalArgume
7171
}
7272
switch (type) {
7373
case LOCAL_INDEX:
74-
final String indexPattern = settings.get(EXPORT_INDEX, DEFAULT_TOP_N_LATENCY_QUERIES_INDEX_PATTERN);
74+
final String indexPattern = settings.get(EXPORT_INDEX, DEFAULT_TOP_N_QUERIES_INDEX_PATTERN);
7575
if (indexPattern.length() == 0) {
7676
throw new IllegalArgumentException("Empty index pattern configured for the exporter");
7777
}

0 commit comments

Comments
 (0)