Skip to content

Commit 5086fa3

Browse files
doks5doks5
authored
control-service: Add python_version to Control Service API (#1806)
As part of the effort to provide users with the ability to specify the python release version, which they want their data jobs to be deployed with, we need to expose a property in the Control Service's API and ensure that the configuration is stored in the database and the kubernetes objects. This change exposes a `python_version` property in the api schemas of the DataJobDeployment and DataJobDeploymentStatus components, and propagates the property to the database model and kubernetes resources. All of this is done as one change, to avoid unexpected behaviour or regressions. VEP for Reference: https://github.com/vmware/versatile-data-kit/tree/main/specs/vep-1739-multiple-python-versions Testing Done: New and modified existing tests. --------- Signed-off-by: Andon Andonov <andonova@vmware.com> Co-authored-by: github-actions <>
1 parent fd861fc commit 5086fa3

File tree

16 files changed

+122
-9
lines changed

16 files changed

+122
-9
lines changed

‎projects/control-service/projects/model/apidefs/datajob-api/api.yaml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1077,6 +1077,10 @@ components:
10771077
description: Job version (can be Git commit)
10781078
type: string
10791079
example: 11a403ba
1080+
python_version:
1081+
description: A python release version (supported by the service) to be used for job deployments.
1082+
type: string
1083+
example: '3.9'
10801084
mode:
10811085
$ref: '#/components/schemas/DataJobMode'
10821086
id:
@@ -1110,6 +1114,10 @@ components:
11101114
description: Job version (can be Git commit)
11111115
type: string
11121116
example: 11a403ba
1117+
python_version:
1118+
description: A python release version (supported by the service) to be used for job deployments.
1119+
type: string
1120+
example: '3.9'
11131121
mode:
11141122
$ref: '#/components/schemas/DataJobMode'
11151123
id:

‎projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/DataJobsDeploymentController.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,7 @@ public ResponseEntity<Void> deploymentDelete(
6868
@Override
6969
public ResponseEntity<Void> deploymentPatch(
7070
String teamName, String jobName, String deploymentId, DataJobDeployment dataJobDeployment) {
71+
deploymentService.validatePythonVersionIsSupported(dataJobDeployment.getPythonVersion());
7172
if (jobsService.jobWithTeamExists(jobName, teamName)) {
7273
// TODO: deploymentId not implemented
7374
Optional<com.vmware.taurus.service.model.DataJob> job = jobsService.getByName(jobName);
@@ -120,6 +121,7 @@ public ResponseEntity<Void> deploymentUpdate(
120121
String jobName,
121122
Boolean sendNotification,
122123
DataJobDeployment dataJobDeployment) {
124+
deploymentService.validatePythonVersionIsSupported(dataJobDeployment.getPythonVersion());
123125
if (jobsService.jobWithTeamExists(jobName, teamName)) {
124126
Optional<com.vmware.taurus.service.model.DataJob> job =
125127
jobsService.getByName(jobName.toLowerCase());

‎projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/DeploymentModelConverter.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
package com.vmware.taurus.datajobs;
77

88
import com.vmware.taurus.controlplane.model.data.DataJobResources;
9+
import com.vmware.taurus.service.deploy.SupportedPythonVersions;
910
import com.vmware.taurus.service.model.JobDeployment;
1011
import com.vmware.taurus.service.model.JobDeploymentStatus;
1112
import lombok.AllArgsConstructor;
@@ -16,6 +17,8 @@
1617
@AllArgsConstructor
1718
public class DeploymentModelConverter {
1819

20+
private final SupportedPythonVersions supportedPythonVersions;
21+
1922
public static JobDeployment toJobDeployment(
2023
String teamName, String jobName, JobDeploymentStatus jobDeploymentStatus) {
2124
JobDeployment deployment = new JobDeployment();
@@ -28,6 +31,7 @@ public static JobDeployment toJobDeployment(
2831
deployment.setMode(jobDeploymentStatus.getMode());
2932
deployment.setGitCommitSha(jobDeploymentStatus.getGitCommitSha());
3033
deployment.setVdkVersion(jobDeploymentStatus.getVdkVersion());
34+
deployment.setPythonVersion(jobDeploymentStatus.getPythonVersion());
3135

3236
return deployment;
3337
}
@@ -101,6 +105,11 @@ public static JobDeployment mergeDeployments(
101105
newDeployment.getVdkVersion() != null
102106
? newDeployment.getVdkVersion()
103107
: oldDeployment.getVdkVersion());
108+
mergedDeployment.setPythonVersion(
109+
newDeployment.getPythonVersion() != null
110+
? newDeployment.getPythonVersion()
111+
: oldDeployment.getPythonVersion());
112+
104113
return mergedDeployment;
105114
}
106115
}

‎projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/ToApiModelConverter.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@ public static DataJobDeploymentStatus toDataJobDeploymentStatus(
109109
deployment.setLastDeployedBy(jobDeploymentStatus.getLastDeployedBy());
110110
deployment.setLastDeployedDate(jobDeploymentStatus.getLastDeployedDate());
111111
deployment.setVdkVersion(jobDeploymentStatus.getVdkVersion());
112+
deployment.setPythonVersion(jobDeploymentStatus.getPythonVersion());
112113

113114
return deployment;
114115
}

‎projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/datajobs/ToModelApiConverter.java‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@ public static JobDeployment toJobDeployment(
2727
}
2828
jobDeployment.setGitCommitSha(dataJobDeployment.getJobVersion());
2929
jobDeployment.setVdkVersion(dataJobDeployment.getVdkVersion());
30+
if (dataJobDeployment.getPythonVersion() != null) {
31+
jobDeployment.setPythonVersion(dataJobDeployment.getPythonVersion());
32+
}
3033

3134
return jobDeployment;
3235
}

‎projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/exception/UnsupportedPythonVersionException.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
/*
2+
* Copyright 2021-2023 VMware, Inc.
3+
* SPDX-License-Identifier: Apache-2.0
4+
*/
5+
6+
package com.vmware.taurus.exception;
7+
8+
import org.springframework.http.HttpStatus;
9+
10+
import java.util.Set;
11+
12+
public class UnsupportedPythonVersionException extends DomainError implements UserFacingError {
13+
public UnsupportedPythonVersionException(
14+
String pythonVersion, Set<String> supportedPythonVersions) {
15+
super(
16+
String.format("Not supported python version: '%s'", pythonVersion),
17+
"The selected python version is not supported by the platform.",
18+
"The deployment of the data job will not proceed. ",
19+
String.format(
20+
"To deploy the data job, please set the python_version property to one of the supported"
21+
+ " versions: '%s'.",
22+
supportedPythonVersions.toString()),
23+
null);
24+
}
25+
26+
@Override
27+
public HttpStatus getHttpStatus() {
28+
return HttpStatus.CONFLICT;
29+
}
30+
}

‎projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/KubernetesService.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2521,6 +2521,7 @@ private Optional<JobDeploymentStatus> mapV1beta1CronJobToDeploymentStatus(
25212521
if (annotations != null) {
25222522
deployment.setLastDeployedBy(annotations.get(JobAnnotation.DEPLOYED_BY.getValue()));
25232523
deployment.setLastDeployedDate(annotations.get(JobAnnotation.DEPLOYED_DATE.getValue()));
2524+
deployment.setPythonVersion(annotations.get(JobAnnotation.PYTHON_VERSION.getValue()));
25242525
}
25252526

25262527
List<V1Container> containers =
@@ -2582,6 +2583,7 @@ private Optional<JobDeploymentStatus> mapV1CronJobToDeploymentStatus(
25822583
if (annotations != null) {
25832584
deployment.setLastDeployedBy(annotations.get(JobAnnotation.DEPLOYED_BY.getValue()));
25842585
deployment.setLastDeployedDate(annotations.get(JobAnnotation.DEPLOYED_DATE.getValue()));
2586+
deployment.setPythonVersion(annotations.get(JobAnnotation.PYTHON_VERSION.getValue()));
25852587
}
25862588

25872589
List<V1Container> containers =

‎projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/DeploymentService.java‎

Lines changed: 33 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,7 @@
66
package com.vmware.taurus.service.deploy;
77

88
import com.vmware.taurus.datajobs.DeploymentModelConverter;
9-
import com.vmware.taurus.exception.ApiConstraintError;
10-
import com.vmware.taurus.exception.DataJobDeploymentNotFoundException;
11-
import com.vmware.taurus.exception.ErrorMessage;
12-
import com.vmware.taurus.exception.KubernetesException;
9+
import com.vmware.taurus.exception.*;
1310
import com.vmware.taurus.service.JobsRepository;
1411
import com.vmware.taurus.service.diag.OperationContext;
1512
import com.vmware.taurus.service.diag.methodintercept.Measurable;
@@ -46,6 +43,7 @@ public class DeploymentService {
4643
private final OperationContext operationContext;
4744
private final JobsRepository jobsRepository;
4845
private final DataJobMetrics dataJobMetrics;
46+
private final SupportedPythonVersions supportedPythonVersions;
4947

5048
public Optional<JobDeploymentStatus> readDeployment(String jobName) {
5149
return jobImageDeployer.readScheduledJob(jobName);
@@ -64,11 +62,13 @@ public List<JobDeploymentStatus> readDeployments() {
6462
* are null)
6563
*/
6664
public void patchDeployment(DataJob dataJob, JobDeployment jobDeployment) {
65+
6766
var deploymentStatus = readDeployment(dataJob.getName());
6867
if (deploymentStatus.isPresent()) {
6968
var oldDeployment =
7069
DeploymentModelConverter.toJobDeployment(
7170
dataJob.getJobConfig().getTeam(), dataJob.getName(), deploymentStatus.get());
71+
setPythonVersionIfNull(oldDeployment, jobDeployment);
7272
var mergedDeployment =
7373
DeploymentModelConverter.mergeDeployments(oldDeployment, jobDeployment);
7474
validateFieldsCanBePatched(oldDeployment, mergedDeployment);
@@ -141,14 +141,20 @@ public void updateDeployment(
141141
try {
142142
log.info("Starting deployment of job {}", jobDeployment.getDataJobName());
143143
deploymentProgress.started(dataJob.getJobConfig(), jobDeployment);
144+
144145
var deploymentStatus = readDeployment(dataJob.getName());
145146
if (deploymentStatus.isPresent()) {
146147
var oldDeployment =
147148
DeploymentModelConverter.toJobDeployment(
148149
dataJob.getJobConfig().getTeam(), dataJob.getName(), deploymentStatus.get());
150+
setPythonVersionIfNull(oldDeployment, jobDeployment);
149151
jobDeployment = DeploymentModelConverter.mergeDeployments(oldDeployment, jobDeployment);
150152
}
151153

154+
if (jobDeployment.getPythonVersion() == null) {
155+
jobDeployment.setPythonVersion(supportedPythonVersions.getDefaultPythonVersion());
156+
}
157+
152158
String imageName =
153159
dockerRegistryService.dataJobImage(
154160
jobDeployment.getDataJobName(), jobDeployment.getGitCommitSha());
@@ -193,6 +199,21 @@ private void saveDeployment(DataJob dataJob, JobDeployment jobDeployment) {
193199
}
194200
}
195201

202+
/**
203+
* As pythonVersion is optional, we need to check if it is passed. And if it is, we need to
204+
* validate that the python version is supported by the Control Service. If it is not, we need to
205+
* fail the operation, as we don't have sufficient information for the user's intent to deploy the
206+
* data job.
207+
*
208+
* @param pythonVersion The python version to be used for the data job deployment.
209+
*/
210+
public void validatePythonVersionIsSupported(String pythonVersion) {
211+
if (pythonVersion != null && !supportedPythonVersions.isPythonVersionSupported(pythonVersion)) {
212+
throw new UnsupportedPythonVersionException(
213+
pythonVersion, supportedPythonVersions.getSupportedPythonVersions());
214+
}
215+
}
216+
196217
private void handleException(
197218
DataJob dataJob, JobDeployment jobDeployment, Boolean sendNotification, Throwable e) {
198219
ErrorMessage message =
@@ -230,4 +251,12 @@ private boolean deploymentExistsOrInProgress(String dataJobName) {
230251
return jobImageBuilder.isBuildingJobInProgress(dataJobName)
231252
|| readDeployment(dataJobName).isPresent();
232253
}
254+
255+
private JobDeployment setPythonVersionIfNull(
256+
JobDeployment oldDeployment, JobDeployment newDeployment) {
257+
if (oldDeployment.getPythonVersion() == null && newDeployment.getPythonVersion() == null) {
258+
newDeployment.setPythonVersion(supportedPythonVersions.getDefaultPythonVersion());
259+
}
260+
return newDeployment;
261+
}
233262
}

‎projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/deploy/JobImageDeployer.java‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -279,7 +279,8 @@ private void updateCronJob(DataJob dataJob, JobDeployment jobDeployment, String
279279

280280
Map<String, String> jobDeploymentAnnotations = new HashMap<>();
281281
var jobLabels = getJobLabels(dataJob, jobDeployment);
282-
var jobAnnotations = getJobAnnotations(dataJob, lastDeployedBy);
282+
var jobAnnotations =
283+
getJobAnnotations(dataJob, lastDeployedBy, jobDeployment.getPythonVersion());
283284

284285
String cronJobName = getCronJobName(jobName);
285286
boolean enabled = jobDeployment.getEnabled() == null || jobDeployment.getEnabled();
@@ -379,7 +380,8 @@ private Map<String, String> getJobLabels(DataJob dataJob, JobDeployment jobDeplo
379380
return jobPodLabels;
380381
}
381382

382-
private Map<String, String> getJobAnnotations(DataJob dataJob, String deployedBy) {
383+
private Map<String, String> getJobAnnotations(
384+
DataJob dataJob, String deployedBy, String pythonVersion) {
383385
Map<String, String> jobPodAnnotations = new HashMap<>();
384386
jobPodAnnotations.put(JobAnnotation.SCHEDULE.getValue(), dataJob.getJobConfig().getSchedule());
385387
jobPodAnnotations.put(JobAnnotation.EXECUTION_TYPE.getValue(), "scheduled");
@@ -390,6 +392,7 @@ private Map<String, String> getJobAnnotations(DataJob dataJob, String deployedBy
390392
jobPodAnnotations.put(
391393
JobAnnotation.UNSCHEDULED.getValue(),
392394
(StringUtils.isEmpty(dataJob.getJobConfig().getSchedule()) ? "true" : "false"));
395+
jobPodAnnotations.put(JobAnnotation.PYTHON_VERSION.getValue(), pythonVersion);
393396
return jobPodAnnotations;
394397
}
395398

‎projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/service/model/JobAnnotation.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ public enum JobAnnotation {
1515
DEPLOYED_BY("deployed-by"),
1616
EXECUTION_TYPE("execution-type"),
1717
OP_ID("op-id"),
18+
PYTHON_VERSION("python-version"),
1819
UNSCHEDULED("unscheduled");
1920

2021
@Getter private String value;

0 commit comments

Comments
 (0)