Implement MVP for content repos

docs/configuration.schema.json

@@ -150,6 +150,55 @@
         "examples" : {
           "type" : [ "boolean", "null" ],
           "description" : "Deploy example content: source repos, GitOps repos, Jenkins Job, Argo CD apps/project"
+        },
+        "namespaces" : {
+          "description" : "Additional kubernetes namespaces. These are authorized to Argo CD, supplied with image pull secrets, monitored by prometheus, etc. Namespaces can be templates, e.g. ${config.application.namePrefix}staging}",
+          "type" : [ "array", "null" ],
+          "items" : {
+            "type" : "string"
+          }
+        },
+        "repos" : {
+          "description" : "Content repos to push into target environment",
+          "type" : [ "array", "null" ],
+          "items" : {
+            "type" : "object",
+            "properties" : {
+              "folderBased" : {
+                "type" : [ "boolean", "null" ],
+                "description" : "When true, interpret the folder structure of each repo as repos. That is, root folder becomes namespace in SCM, sub folders become repository names in SCM"
+              },
+              "password" : {
+                "type" : [ "string", "null" ],
+                "description" : "Password to authenticate against content repo"
+              },
+              "path" : {
+                "type" : [ "string", "null" ],
+                "description" : "Path within the content repo to process"
+              },
+              "ref" : {
+                "type" : [ "string", "null" ],
+                "description" : "Reference for a specific branch, tag, or commit"
+              },
+              "target" : {
+                "type" : [ "string", "null" ],
+                "description" : "Target path for the repository"
+              },
+              "templating" : {
+                "type" : [ "boolean", "null" ],
+                "description" : "When true, template all files ending in .ftl within the repo"
+              },
+              "url" : {
+                "type" : [ "string", "null" ],
+                "description" : "URL of the content repo"
+              },
+              "username" : {
+                "type" : [ "string", "null" ],
+                "description" : "Username to authenticate against content repo"
+              }
+            },
+            "additionalProperties" : false
+          }
         }
       },
       "additionalProperties" : false,

docs/developers.md

@@ -13,7 +13,7 @@ The versions are also specified in the `Config.groovy` file, so it is recommende
 
 ## Table of contents
 
-<!-- Update with ` doctoc --notitle docs/developers.md --maxlevel 4      `. See https://github.com/thlorenz/doctoc -->
+<!-- Update with `doctoc --notitle docs/developers.md --maxlevel 4`. See https://github.com/thlorenz/doctoc -->
 <!-- START doctoc generated TOC please keep comment here to allow auto update -->
 <!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
 

src/main/groovy/com/cloudogu/gitops/Application.groovy

@@ -1,6 +1,9 @@
 package com.cloudogu.gitops
 
 import com.cloudogu.gitops.config.Config
+import com.cloudogu.gitops.utils.TemplatingEngine
+import freemarker.template.Configuration
+import freemarker.template.DefaultObjectWrapperBuilder
 import groovy.util.logging.Slf4j
 import jakarta.inject.Singleton
 
@@ -35,13 +38,14 @@ class Application {
 
     void setNamespaceListToConfig(Config config) {
         Set<String> namespaces = new HashSet<>()
-        String namePrefix = config.application.namePrefix
-
-        if (config.content.examples) {
-            namespaces.addAll(Arrays.asList(
-                    namePrefix + "example-apps-staging",
-                    namePrefix + "example-apps-production"
-            ))
+        def engine = new TemplatingEngine()
+
+        config.content.namespaces.each { String ns ->
+            namespaces.add(engine.template(ns, [
+                    config              : config,
+                    // Allow for using static classes inside the templates
+                    statics             : new DefaultObjectWrapperBuilder(Configuration.VERSION_2_3_32).build().getStaticModels()
+            ]))
         }
 
         //iterates over all FeatureWithImages and gets their namespaces

src/main/groovy/com/cloudogu/gitops/cli/GitopsPlaygroundCliMainScripted.groovy

@@ -92,7 +92,7 @@ class GitopsPlaygroundCliMainScripted {
                         new Jenkins(config, executor, fileSystemUtils, new GlobalPropertyManager(jenkinsApiClient),
                                 new JobManager(jenkinsApiClient), new UserManager(jenkinsApiClient),
                                 new PrometheusConfigurator(jenkinsApiClient), helmStrategy, k8sClient, networkingUtils),
-                        new Content(config, k8sClient),
+                        new Content(config, k8sClient, scmmRepoProvider, scmmApiClient),
                         new ArgoCD(config, k8sClient, helmClient, fileSystemUtils, scmmRepoProvider),
                         new IngressNginx(config, fileSystemUtils, deployer, k8sClient, airGappedUtils),
                         new CertManager(config, fileSystemUtils, deployer, k8sClient, airGappedUtils),

src/main/groovy/com/cloudogu/gitops/config/ApplicationConfigurator.groovy

@@ -52,9 +52,11 @@ class ApplicationConfigurator {
         if (newConfig.features.ingressNginx.active && !newConfig.application.baseUrl) {
             log.warn("Ingress-controller is activated without baseUrl parameter. Services will not be accessible by hostnames. To avoid this use baseUrl with ingress. ")
         }
-
-        if (newConfig.content.examples && !newConfig.registry.active) {
-            throw new RuntimeException("content.examples requires either registry.active or registry.url")
+        if (newConfig.content.examples) {
+            if  (!newConfig.registry.active) {
+                throw new RuntimeException("content.examples requires either registry.active or registry.url")
+            }
+            newConfig.content.namespaces += [ 'example-apps-staging', 'example-apps-production']
         }
     }
 
@@ -251,6 +253,7 @@ class ApplicationConfigurator {
     void validateConfig(Config configToSet) {
         validateScmmAndJenkinsAreBothSet(configToSet)
         validateMirrorReposHelmChartFolderSet(configToSet)
+        validateContent(configToSet)
     }
 
     private void validateMirrorReposHelmChartFolderSet(Config configToSet) {
@@ -261,6 +264,17 @@ class ApplicationConfigurator {
                     "LOCAL_HELM_CHART_FOLDER='charts' after running 'scripts/downloadHelmCharts.sh' from the repo")
         }
     }
+
+    static void validateContent(Config config) {
+        config.content.repos.each { repo ->
+            if (!repo.url) {
+                throw new RuntimeException('content.repos requires a url parameter')
+            }
+            if (!repo.folderBased && !repo.target) {
+                throw new RuntimeException('content.repos.folderBased: false requires folder content.repos.target to be set')
+            }
+        }
+    }
 
     private void validateScmmAndJenkinsAreBothSet(Config configToSet) {
         if (configToSet.jenkins.active && 

src/main/groovy/com/cloudogu/gitops/config/Config.groovy

@@ -85,13 +85,47 @@ class Config {
     @JsonPropertyDescription(CONTENT_DESCRIPTION)
     @Mixin
     ContentSchema content = new ContentSchema()
-
+    
     class ContentSchema {
+
         @Option(names = ['--content-examples'], description = CONTENT_EXAMPLES_DESCRIPTION)
         @JsonPropertyDescription(CONTENT_EXAMPLES_DESCRIPTION)
         Boolean examples = false
+
+        @JsonPropertyDescription(CONTENT_NAMESPACES_DESCRIPTION)
+        List<String> namespaces = []
+
+        @JsonPropertyDescription(CONTENT_REPO_DESCRIPTION)
+        List<ContentRepositorySchema> repos = []
+
+        static class ContentRepositorySchema {
+            @JsonPropertyDescription(CONTENT_REPO_URL_DESCRIPTION)
+            String url = ''
+
+            @JsonPropertyDescription(CONTENT_REPO_PATH_DESCRIPTION)
+            String path = '.'
+
+            @JsonPropertyDescription(CONTENT_REPO_REF_DESCRIPTION)
+            String ref = 'main'
+
+            @JsonPropertyDescription(CONTENT_REPO_USERNAME_DESCRIPTION)
+            String username = ''
+
+            @JsonPropertyDescription(CONTENT_REPO_PASSWORD_DESCRIPTION)
+            String password = ''
+
+            @JsonPropertyDescription(CONTENT_REPO_TEMPLATING_DESCRIPTION)
+            Boolean templating = false
+
+            @JsonPropertyDescription(CONTENT_REPO_FOLDER_BASED_REPOS_DESCRIPTION)
+            Boolean folderBased = false
+
+            @JsonPropertyDescription(CONTENT_REPO_TARGET_DESCRIPTION)
+            String target = ''
+        }
     }
 
+
     static class HelmConfig {
         @JsonPropertyDescription(HELM_CONFIG_CHART_DESCRIPTION)
         String chart = ''

src/main/groovy/com/cloudogu/gitops/config/ConfigConstants.groovy

@@ -29,6 +29,16 @@ interface ConfigConstants {
 
     // Content
     String CONTENT_EXAMPLES_DESCRIPTION = 'Deploy example content: source repos, GitOps repos, Jenkins Job, Argo CD apps/project'
+    String CONTENT_NAMESPACES_DESCRIPTION = 'Additional kubernetes namespaces. These are authorized to Argo CD, supplied with image pull secrets, monitored by prometheus, etc. Namespaces can be templates, e.g. ${config.application.namePrefix}staging}'
+    String CONTENT_REPO_DESCRIPTION = "Content repos to push into target environment"
+    String CONTENT_REPO_URL_DESCRIPTION = "URL of the content repo"
+    String CONTENT_REPO_PATH_DESCRIPTION = "Path within the content repo to process"
+    String CONTENT_REPO_REF_DESCRIPTION = "Reference for a specific branch, tag, or commit"
+    String CONTENT_REPO_USERNAME_DESCRIPTION = "Username to authenticate against content repo"
+    String CONTENT_REPO_PASSWORD_DESCRIPTION = "Password to authenticate against content repo"
+    String CONTENT_REPO_TEMPLATING_DESCRIPTION = "When true, template all files ending in .ftl within the repo"
+    String CONTENT_REPO_FOLDER_BASED_REPOS_DESCRIPTION = "When true, interpret the folder structure of each repo as repos. That is, root folder becomes namespace in SCM, sub folders become repository names in SCM"
+    String CONTENT_REPO_TARGET_DESCRIPTION = "Target path for the repository"
 
     // group jenkins
     String JENKINS_ENABLE_DESCRIPTION = 'Installs Jenkins as CI server'