feat: List expansion from environment variables (#833)

Adds syntax to specify substitution of a list of values provided through separate, indexed environment variables.

Author: johanandren

HOCON.md

@@ -26,13 +26,14 @@
       - [Self-Referential Substitutions](#self-referential-substitutions)
       - [The `+=` field separator](#the--field-separator)
       - [Examples of Self-Referential Substitutions](#examples-of-self-referential-substitutions)
+      - [List values from environment variables](#list-values-from-environment-variables)
     - [Includes](#includes)
-      - [Include syntax](#include-syntax)
-      - [Include semantics: merging](#include-semantics-merging)
-      - [Include semantics: substitution](#include-semantics-substitution)
-      - [Include semantics: missing files and required files](#include-semantics-missing-files-and-required-files)
-      - [Include semantics: file formats and extensions](#include-semantics-file-formats-and-extensions)
-      - [Include semantics: locating resources](#include-semantics-locating-resources)
+     - [Include syntax](#include-syntax)
+     - [Include semantics: merging](#include-semantics-merging)
+     - [Include semantics: substitution](#include-semantics-substitution)
+     - [Include semantics: missing files and required files](#include-semantics-missing-files-and-required-files)
+     - [Include semantics: file formats and extensions](#include-semantics-file-formats-and-extensions)
+     - [Include semantics: locating resources](#include-semantics-locating-resources)
     - [Conversion of numerically-indexed objects to arrays](#conversion-of-numerically-indexed-objects-to-arrays)
   - [MIME Type](#mime-type)
   - [API Recommendations](#api-recommendations)
@@ -888,6 +889,33 @@ rather than by the path inside the `${}` expression, because
 substitutions may be resolved differently depending on their
 position in the file.
 
+
+#### List values from environment variables
+
+Environment variables are inherently single string values, but
+sometimes it's useful to populate a list-valued configuration
+setting from the environment. The `[]` suffix on a substitution
+enables this:
+
+    my-list = ${MY_LIST[]}
+
+This is only supported for environment variables and not system properties or path expressions
+
+When a substitution with the `[]` suffix is resolved via
+environment variable fallback, the implementation will look up
+`MY_LIST_0`, `MY_LIST_1`, `MY_LIST_2`, and so on (incrementing
+the index) until an environment variable is not found. The
+collected values form a list.
+
+- If no element is found (i.e. `MY_LIST_0` is not set), a
+  required substitution `${MY_LIST[]}` is an error, while an
+  optional substitution `${?MY_LIST[]}` is treated as undefined
+  (the key is removed from the config).
+- Each element value is a string, as with all environment
+  variables. Automatic type conversion applies when the
+  application requests another type (e.g. `getIntList()`).
+- The separator between the base name and the index is `_`.
+
 ### Includes
 
 #### Include syntax

README.md

@@ -698,7 +698,22 @@ Using java properties you specify the exact position:
 
     -Ditems.0="a" -Ditems.1="b"
 
-as well as with environment variables:
+It is also possible to use environment variables to  define a list using the special `[]` suffix to the
+environment variable name:
+
+    path = [ "a" ]
+    path = ${?OPTIONAL_A[]}
+
+with the values set as individual environment values using the name and an index suffix:
+
+    OPTIONAL_A_1="a"
+    OPTIONAL_A_2="b"
+
+the index must be gap free, as soon as the next index is not found, the collection of values stops.
+
+This only works for environment variables, not java properties or relative paths.
+
+Finally, it is also possible to use the CONFIG_FORCE feature:
 
     export CONFIG_FORCE_items_0=a
     export CONFIG_FORCE_items_1=b

build.sbt

@@ -77,7 +77,13 @@ lazy val configLib =  Project("config", file("config"))
       "CONFIG_FORCE_akka_event__handler__dispatcher_max__pool__size" -> "10",
       "SECRET_A" -> "A", // ConfigTest.renderShowEnvVariableValues
       "SECRET_B" -> "B", // ConfigTest.renderShowEnvVariableValues
-      "SECRET_C" -> "C" // ConfigTest.renderShowEnvVariableValues
+      "SECRET_C" -> "C", // ConfigTest.renderShowEnvVariableValues
+      "MY_LIST_0" -> "a", // ConfigTest.envVariableListExpansion
+      "MY_LIST_1" -> "b", // ConfigTest.envVariableListExpansion
+      "MY_LIST_2" -> "c", // ConfigTest.envVariableListExpansion
+      "NUM_LIST_0" -> "1", // ConfigTest.envVariableListExpansion
+      "NUM_LIST_1" -> "2", // ConfigTest.envVariableListExpansion
+      "NUM_LIST_2" -> "3" // ConfigTest.envVariableListExpansion
     ),
 
     OsgiKeys.exportPackage                 := Seq("com.typesafe.config", "com.typesafe.config.impl"),

config/src/main/java/com/typesafe/config/impl/ConfigNodeSimpleValue.java

@@ -29,10 +29,25 @@ else if (Tokens.isUnquotedText(token))
             return new ConfigString.Unquoted(token.origin(), Tokens.getUnquotedText(token));
         else if (Tokens.isSubstitution(token)) {
             List<Token> expression = Tokens.getSubstitutionPathExpression(token);
-            Path path = PathParser.parsePathExpression(expression.iterator(), token.origin());
             boolean optional = Tokens.getSubstitutionOptional(token);
 
-            return new ConfigReference(token.origin(), new SubstitutionExpression(path, optional));
+            // Detect and strip trailing [] for list expansion syntax.
+            // Inside a substitution, [] is tokenized as OPEN_SQUARE + CLOSE_SQUARE.
+            boolean listExpansion = false;
+            List<Token> pathExpression = expression;
+            int size = expression.size();
+            if (size >= 2) {
+                Token secondLast = expression.get(size - 2);
+                Token last = expression.get(size - 1);
+                if (secondLast == Tokens.OPEN_SQUARE && last == Tokens.CLOSE_SQUARE) {
+                    listExpansion = true;
+                    pathExpression = expression.subList(0, size - 2);
+                }
+            }
+
+            Path path = PathParser.parsePathExpression(pathExpression.iterator(), token.origin());
+
+            return new ConfigReference(token.origin(), new SubstitutionExpression(path, optional, listExpansion));
         }
         throw new ConfigException.BugOrBroken("ConfigNodeSimpleValue did not contain a valid value token");
     }

config/src/main/java/com/typesafe/config/impl/ResolveSource.java

@@ -1,5 +1,8 @@
 package com.typesafe.config.impl;
 
+import java.util.ArrayList;
+import java.util.List;
+
 import com.typesafe.config.ConfigException;
 import com.typesafe.config.impl.AbstractConfigValue.NotPossibleToResolve;
 
@@ -110,9 +113,19 @@ ResultWithPath lookupSubst(ResolveContext context, SubstitutionExpression subst,
             }
 
             if (result.result.value == null && result.result.context.options().getUseSystemEnvironment()) {
-                if (ConfigImpl.traceSubstitutionsEnabled())
-                    ConfigImpl.trace(result.result.context.depth(), unprefixed + " - looking up in system environment");
-                result = findInObject(ConfigImpl.envVariablesAsConfigObject(), context, unprefixed);
+                if (subst.listExpansion()) {
+                    if (ConfigImpl.traceSubstitutionsEnabled())
+                        ConfigImpl.trace(result.result.context.depth(), unprefixed + " - looking up list expansion in system environment");
+                    AbstractConfigValue listValue = expandEnvVarList(ConfigImpl.envVariablesAsConfigObject(), unprefixed, context);
+                    if (listValue != null) {
+                        result = new ResultWithPath(ResolveResult.make(result.result.context, listValue),
+                                new Node<Container>(ConfigImpl.envVariablesAsConfigObject()));
+                    }
+                } else {
+                    if (ConfigImpl.traceSubstitutionsEnabled())
+                        ConfigImpl.trace(result.result.context.depth(), unprefixed + " - looking up in system environment");
+                    result = findInObject(ConfigImpl.envVariablesAsConfigObject(), context, unprefixed);
+                }
             }
         }
 
@@ -122,6 +135,30 @@ ResultWithPath lookupSubst(ResolveContext context, SubstitutionExpression subst,
         return result;
     }
 
+    private static AbstractConfigValue expandEnvVarList(AbstractConfigObject envObj, Path basePath, ResolveContext context) {
+        // Build the base env var name from the path (e.g. path "MY_LIST" -> "MY_LIST")
+        String baseName = basePath.render();
+        SimpleConfigOrigin origin = SimpleConfigOrigin.newSimple("env list expansion of " + baseName);
+
+        // look for _N environment variables until there is no next value
+        List<AbstractConfigValue> values = new ArrayList<AbstractConfigValue>();
+        for (int i = 0; ; i++) {
+            String key = baseName + "_" + i;
+            Path elementPath = new Path(key);
+            AbstractConfigValue v = findInObject(envObj, elementPath).value;
+            if (v == null) {
+                break;
+            }
+            values.add(v);
+        }
+
+        if (values.isEmpty()) {
+            return null;
+        }
+
+        return new SimpleConfigList(origin, values);
+    }
+
     ResolveSource pushParent(Container parent) {
         if (parent == null)
             throw new ConfigException.BugOrBroken("can't push null parent");

config/src/main/java/com/typesafe/config/impl/SubstitutionExpression.java

@@ -4,10 +4,16 @@ final class SubstitutionExpression {
 
     final private Path path;
     final private boolean optional;
+    final private boolean listExpansion;
 
     SubstitutionExpression(Path path, boolean optional) {
+        this(path, optional, false);
+    }
+
+    SubstitutionExpression(Path path, boolean optional, boolean listExpansion) {
         this.path = path;
         this.optional = optional;
+        this.listExpansion = listExpansion;
     }
 
     Path path() {
@@ -18,23 +24,35 @@ boolean optional() {
         return optional;
     }
 
+    boolean listExpansion() {
+        return listExpansion;
+    }
+
     SubstitutionExpression changePath(Path newPath) {
         if (newPath == path)
             return this;
         else
-            return new SubstitutionExpression(newPath, optional);
+            return new SubstitutionExpression(newPath, optional, listExpansion);
+    }
+
+    SubstitutionExpression changeListExpansion(boolean newListExpansion) {
+        if (newListExpansion == listExpansion)
+            return this;
+        else
+            return new SubstitutionExpression(path, optional, newListExpansion);
     }
 
     @Override
     public String toString() {
-        return "${" + (optional ? "?" : "") + path.render() + "}";
+        return "${" + (optional ? "?" : "") + path.render() + (listExpansion ? "[]" : "") + "}";
     }
 
     @Override
     public boolean equals(Object other) {
         if (other instanceof SubstitutionExpression) {
             SubstitutionExpression otherExp = (SubstitutionExpression) other;
-            return otherExp.path.equals(this.path) && otherExp.optional == this.optional;
+            return otherExp.path.equals(this.path) && otherExp.optional == this.optional
+                    && otherExp.listExpansion == this.listExpansion;
         } else {
             return false;
         }
@@ -44,6 +62,7 @@ public boolean equals(Object other) {
     public int hashCode() {
         int h = 41 * (41 + path.hashCode());
         h = 41 * (h + (optional ? 1 : 0));
+        h = 41 * (h + (listExpansion ? 1 : 0));
         return h;
     }
 }

config/src/test/resources/env-variables.conf

@@ -2,3 +2,15 @@ secret = a
 secret = ${?SECRET_A}
 secrets = ["b", "c"]
 secrets = [${?SECRET_B}, ${?SECRET_C}]
+
+myList = ${MY_LIST[]}
+myOptionalList = ${?MY_LIST[]}
+myOptionalUndefinedList = ${?UNDEFINED_LIST[]}
+numList = ${NUM_LIST[]}
+
+# list concatenation with env var list expansion
+prependedList = ["x", "y"] ${?MY_LIST[]}
+appendedList = ${?MY_LIST[]} ["x", "y"]
+selfAppendedList = ["x"]
+selfAppendedList = ${?selfAppendedList} ${?MY_LIST[]}
+appendedUndefined = ["x", "y"] ${?UNDEFINED_LIST[]}

config/src/test/scala/com/typesafe/config/impl/ConfigTest.scala

@@ -4,16 +4,16 @@
 package com.typesafe.config.impl
 
 import java.math.BigInteger
-import java.time.temporal.{ ChronoUnit, TemporalUnit }
-
+import java.time.temporal.{ChronoUnit, TemporalUnit}
+import java.util
+import java.util.concurrent.TimeUnit
+import java.util.concurrent.TimeUnit.{DAYS, HOURS, MICROSECONDS, MILLISECONDS, MINUTES, NANOSECONDS, SECONDS}
 import org.junit.Assert._
 import org.junit._
 import com.typesafe.config._
-import java.util.concurrent.TimeUnit
 
 import scala.collection.JavaConverters._
 import com.typesafe.config.ConfigResolveOptions
-import java.util.concurrent.TimeUnit.{ DAYS, HOURS, MICROSECONDS, MILLISECONDS, MINUTES, NANOSECONDS, SECONDS }
 
 class ConfigTest extends TestUtils {
 
@@ -1223,6 +1223,16 @@ class ConfigTest extends TestUtils {
                |        # env variables
                |        "<env variable>"
                |    ]""".stripMargin))
+      assertTrue(rendered1.contains(
+        """|    "myList" : [
+           |        # env variables
+           |        "<env variable>",
+           |        # env variables
+           |        "<env variable>",
+           |        # env variables
+           |        "<env variable>"
+           |    ]""".stripMargin
+      ))
 
         val showRenderOpt = ConfigRenderOptions.defaults()
         val rendered2 = config.root().render(showRenderOpt)
@@ -1236,6 +1246,57 @@ class ConfigTest extends TestUtils {
                |    ]""".stripMargin))
     }
 
+    @Test
+    def envVariableListExpansion(): Unit = {
+        val config = ConfigFactory.load("env-variables")
+
+        val myList = config.getStringList("myList")
+        assertEquals("basic  environment variable list expansion", 3, myList.size())
+        assertEquals("a", myList.get(0))
+        assertEquals("b", myList.get(1))
+        assertEquals("c", myList.get(2))
+
+        val myOptionalList = config.getStringList("myOptionalList")
+        assertEquals("optional environment variable list expansion", 3, myOptionalList.size())
+        assertEquals("a", myOptionalList.get(0))
+
+        assertFalse("optional environment variable list expansion (undefined)", config.hasPath("myOptionalUndefinedList"))
+
+        val numList = config.getIntList("numList")
+        assertEquals("environment variable list conversion", 3, numList.size())
+        assertEquals(1, numList.get(0).intValue())
+        assertEquals(2, numList.get(1).intValue())
+        assertEquals(3, numList.get(2).intValue())
+    }
+
+    @Test
+    def envVariableListExpansionAppend(): Unit = {
+        val config = ConfigFactory.load("env-variables")
+
+        // list: ["x", "y"] ${?MY_LIST[]}
+        val prepended = config.getStringList("prependedList")
+        assertEquals("prepend static values before env var list", util.Arrays.asList("x", "y", "a", "b", "c"), prepended)
+
+        // list: ${?MY_LIST[]} ["x", "y"]
+        val appended = config.getStringList("appendedList")
+        assertEquals("append static values after env var list", util.Arrays.asList("a", "b", "c", "x", "y"), appended)
+
+        // myList = ["x"]; myList = ${?myList} ${?MY_LIST[]}
+        val selfAppended = config.getStringList("selfAppendedList")
+        assertEquals("self-referential append", util.Arrays.asList("x", "a", "b", "c"), selfAppended)
+
+        val appendedUndefined = config.getStringList("appendedUndefined")
+        assertEquals("appending undefined optional list leaves original intact", util.Arrays.asList("x", "y"), appendedUndefined)
+    }
+
+    @Test
+    def envVariableListExpansionRequiredUndefined(): Unit = {
+        val e = intercept[ConfigException.UnresolvedSubstitution] {
+            ConfigFactory.parseString("foo = ${UNDEFINED_LIST[]}").resolve()
+        }
+        assertTrue(e.getMessage.contains("UNDEFINED_LIST"))
+    }
+
     @Test
     def serializeRoundTrip() {
         for (i <- 1 to 10) {