control-service: secrets service integration test

.github/ISSUE_TEMPLATE/config.yaml → .github/ISSUE_TEMPLATE/config.yml

@@ -1,8 +1,11 @@
+# Copyright 2021-2023 VMware, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
 blank_issues_enabled: false
 contact_links:
   - name: "\U0001F914 All other questions, including if you're not sure what to do."
     url: https://github.com/jupyterlab/jupyterlab/discussions/new?category=q-a
     about: Search Github Discussions for similar questions or ask for help there.
   - name: "\U0001F4AC Chat with the devs on Versatile Data Kit slack channel in CNCF slack "
-    url: https://communityinviter.com/apps/cloud-native/cncf 
-    about: Ask short questions about using Versatile Data Kit
+    url: https://communityinviter.com/apps/cloud-native/cncf
+    about: Ask short questions about using Versatile Data Kit

projects/control-service/projects/pipelines_control_service/build.gradle

@@ -92,6 +92,8 @@ dependencies { // Implementation dependencies are found on compile classpath of
     testImplementation versions.'net.bytebuddy:byte-buddy'
     testImplementation versions.'org.testcontainers:testcontainers'
     testImplementation versions.'org.awaitility:awaitility'
+    testImplementation versions.'org.testcontainers:vault'
+    testImplementation versions.'org.testcontainers:junit-jupiter'
     testImplementation 'com.github.kirviq:dumbster:1.7.1'
     testImplementation versions.'org.junit.jupiter:junit-jupiter-api'
     testImplementation versions.'org.junit.platform:junit-platform-suite-api'

projects/control-service/projects/pipelines_control_service/src/integration-test/java/com/vmware/taurus/secrets/service/vault/TestVaultJobSecretsServiceIT.java

@@ -0,0 +1,98 @@
+/*
+ * Copyright 2021-2023 VMware, Inc.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package com.vmware.taurus.secrets.service.vault;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.vmware.taurus.ControlplaneApplication;
+import com.vmware.taurus.datajobs.it.common.BaseIT;
+import com.vmware.taurus.exception.DataJobSecretsSizeLimitException;
+import org.apache.commons.lang3.RandomStringUtils;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.vault.authentication.TokenAuthentication;
+import org.springframework.vault.client.VaultEndpoint;
+import org.springframework.vault.core.VaultTemplate;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+import org.testcontainers.vault.VaultContainer;
+
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+@SpringBootTest(
+    webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT,
+    classes = ControlplaneApplication.class)
+@Testcontainers
+public class TestVaultJobSecretsServiceIT extends BaseIT {
+
+  @Container
+  private static final VaultContainer vaultContainer =
+      new VaultContainer<>("vault:1.0.2").withVaultToken("root");
+
+  private static VaultJobSecretsService vaultJobSecretService;
+
+  @BeforeAll
+  public static void init() throws URISyntaxException {
+    String vaultUri = vaultContainer.getHttpHostAddress();
+
+    VaultEndpoint vaultEndpoint = VaultEndpoint.from(new URI(vaultUri));
+    TokenAuthentication clientAuthentication = new TokenAuthentication("root");
+
+    VaultTemplate vaultTemplate = new VaultTemplate(vaultEndpoint, clientAuthentication);
+
+    vaultJobSecretService = new VaultJobSecretsService(vaultTemplate);
+  }
+
+  @Test
+  public void testGetEmptyDataJobSecrets() throws Exception {
+    Map<String, Object> result = vaultJobSecretService.readJobSecrets("testJob");
+    Assertions.assertEquals(Collections.emptyMap(), result);
+  }
+
+  @Test
+  public void testSetDataJobSecrets() throws Exception {
+    Map<String, Object> temp = new HashMap<>();
+    temp.put("key1", "value1");
+
+    Map<String, Object> secrets = Collections.unmodifiableMap(temp);
+
+    vaultJobSecretService.updateJobSecrets("testJob2", secrets);
+
+    Map<String, Object> readResult = vaultJobSecretService.readJobSecrets("testJob2");
+    Assertions.assertEquals(secrets, readResult);
+  }
+
+  @Test
+  void testUpdateJobSecretsLimit() throws JsonProcessingException {
+    Map<String, Object> temp = new HashMap<>();
+    temp.put("key1", "value1");
+
+    Map<String, Object> secrets = Collections.unmodifiableMap(temp);
+
+    vaultJobSecretService.updateJobSecrets("testJob2", secrets);
+
+    Map<String, Object> largeSecrets = new HashMap<>();
+    largeSecrets.put("key1", null);
+    largeSecrets.put(
+        "key2",
+        RandomStringUtils.randomAlphabetic(VaultJobSecretsService.VAULT_SIZE_LIMIT_DEFAULT));
+
+    assertThrows(
+        DataJobSecretsSizeLimitException.class,
+        () -> vaultJobSecretService.updateJobSecrets("testJob2", largeSecrets));
+
+    // check secrets were not updated
+    Map<String, Object> readResult = vaultJobSecretService.readJobSecrets("testJob2");
+    Assertions.assertEquals(secrets, readResult);
+  }
+}

projects/control-service/projects/pipelines_control_service/src/main/java/com/vmware/taurus/secrets/service/vault/VaultJobSecretsService.java

@@ -27,7 +27,8 @@
 @ConditionalOnProperty(value = "featureflag.vault.integration.enabled")
 public class VaultJobSecretsService implements com.vmware.taurus.secrets.service.JobSecretsService {
 
-  private static final int VAULT_SIZE_LIMIT_DEFAULT = 1048576; // 1 MB
+  //  package private so it can be used in tests
+  static final int VAULT_SIZE_LIMIT_DEFAULT = 1048576; // 1 MB
   private static final String SECRET = "secret";
 
   @Value("${datajobs.vault.size.limit.bytes}")

projects/control-service/projects/versions-of-external-dependencies.gradle

@@ -34,6 +34,8 @@ project.ext {
             'net.javacrumbs.shedlock:shedlock-spring'                            : 'net.javacrumbs.shedlock:shedlock-spring:5.4.0',
             'net.javacrumbs.shedlock:shedlock-provider-jdbc-template'            : 'net.javacrumbs.shedlock:shedlock-provider-jdbc-template:5.4.0',
             'org.testcontainers:testcontainers'                                  : 'org.testcontainers:testcontainers:1.18.3',
+            'org.testcontainers:vault'                                           : 'org.testcontainers:vault:1.18.3',
+            'org.testcontainers:junit-jupiter'                                   : 'org.testcontainers:junit-jupiter:1.18.3',
             'org.mock-server:mockserver-netty'                                   : 'org.mock-server:mockserver-netty:5.15.0', //5.11.2
             'org.awaitility:awaitility'                                          : 'org.awaitility:awaitility:4.2.0',
             'org.apache.commons:commons-lang3'                                   : 'org.apache.commons:commons-lang3:3.12.0',
@@ -45,7 +47,7 @@ project.ext {
             'com.amazonaws:aws-java-sdk-core'                                    : 'com.amazonaws:aws-java-sdk-core:1.12.490',
             'com.amazonaws:aws-java-sdk-sts'                                     : 'com.amazonaws:aws-java-sdk-sts:1.12.490',
             'com.amazonaws:aws-java-sdk-ecr'                                     : 'com.amazonaws:aws-java-sdk-ecr:1.12.490',
-            'org.springframework.vault:spring-vault-core'                        : 'org.springframework.vault:spring-vault-core:3.0.2',
+            'org.springframework.vault:spring-vault-core'                        : 'org.springframework.vault:spring-vault-core:2.3.3',
 
             // transitive dependencies version force (freeze)
             // on next upgrade, revise if those still need to be set explicitly