feat: uri decode function

Author: Fletcher Gornick

Makefile.am

@@ -141,7 +141,7 @@ endif
 
 ### Tests (make check)
 
-TESTS = tests/mantest tests/jqtest tests/shtest tests/utf8test tests/base64test
+TESTS = tests/mantest tests/jqtest tests/shtest tests/utf8test tests/base64test tests/uritest
 if !WIN32
 TESTS += tests/optionaltest
 endif
@@ -218,7 +218,6 @@ EXTRA_DIST = $(DOC_FILES) $(man_MANS) $(TESTS) $(TEST_LOG_COMPILER)     \
         jq.1.prebuilt jq.spec src/lexer.c src/lexer.h src/parser.c      \
         src/parser.h src/version.h src/builtin.jq scripts/version       \
         libjq.pc                                                        \
-        tests/base64.test tests/jq-f-test.sh tests/jq.test              \
         tests/modules/a.jq tests/modules/b/b.jq tests/modules/c/c.jq    \
         tests/modules/c/d.jq tests/modules/data.json                    \
         tests/modules/home1/.jq tests/modules/home2/.jq/g.jq            \
@@ -232,7 +231,7 @@ EXTRA_DIST = $(DOC_FILES) $(man_MANS) $(TESTS) $(TEST_LOG_COMPILER)     \
         tests/onig.supp tests/local.supp                                \
         tests/setup tests/torture/input0.json                           \
         tests/optional.test tests/man.test tests/manonig.test           \
-        tests/jq.test tests/onig.test tests/base64.test                 \
+        tests/jq.test tests/onig.test tests/base64.test tests/uri.test  \
         tests/utf8-truncate.jq tests/jq-f-test.sh                       \
         tests/no-main-program.jq tests/yes-main-program.jq
 

docs/content/manual/dev/manual.yml

@@ -1300,7 +1300,7 @@ sections:
             input: '[1,[[],{"a":2}]]'
             output: ['[[0],[1,1,"a"]]']
 
-      - title: "`add`"
+      - title: "`add`, `add(generator)`"
         body: |
 
           The filter `add` takes as input an array, and produces as
@@ -1311,6 +1311,9 @@ sections:
 
           If the input is an empty array, `add` returns `null`.
 
+          `add(generator)` operates on the given generator rather than
+          the input.
+
         examples:
           - program: add
             input: '["a","b","c"]'
@@ -1321,6 +1324,9 @@ sections:
           - program: add
             input: '[]'
             output: ["null"]
+          - program: add(.[].a)
+            input: '[{"a":3}, {"a":5}, {"b":6}]'
+            output: ['8']
 
       - title: "`any`, `any(condition)`, `any(generator; condition)`"
         body: |
@@ -2135,6 +2141,11 @@ sections:
             Applies percent-encoding, by mapping all reserved URI
             characters to a `%XX` sequence.
 
+          * `@urid`:
+
+            The inverse of `@uri`, applies percent-decoding, by mapping
+            all `%XX` sequences to their corresponding URI characters.
+
           * `@csv`:
 
             The input must be an array, and it is rendered as CSV

jq.1.prebuilt

@@ -657,6 +657,48 @@ static jv f_format(jq_state *jq, jv input, jv fmt) {
     }
     jv_free(input);
     return line;
+  } else if (!strcmp(fmt_s, "urid")) {
+    jv_free(fmt);
+    input = f_tostring(jq, input);
+
+    jv line = jv_string("");
+    const char *errmsg =  "is not a valid uri encoding";
+    const char *s = jv_string_value(input);
+    while (*s) {
+      if (*s != '%') {
+        line = jv_string_append_buf(line, s++, 1);
+      } else {
+        unsigned char unicode[4] = {0};
+        int b = 0;
+        // check leading bits of first octet to determine length of unicode character
+        // (https://datatracker.ietf.org/doc/html/rfc3629#section-3)
+        while (b == 0 || (b < 4 && unicode[0] >> 7 & 1 && unicode[0] >> (7-b) & 1)) {
+          if (*(s++) != '%') {
+            jv_free(line);
+            return type_error(input, errmsg);
+          }
+          for (int i=0; i<2; i++) {
+            unicode[b] <<= 4;
+            char c = *(s++);
+            if ('0' <= c && c <= '9') unicode[b] |= c - '0';
+            else if ('a' <= c && c <= 'f') unicode[b] |= c - 'a' + 10;
+            else if ('A' <= c && c <= 'F') unicode[b] |= c - 'A' + 10;
+            else {
+              jv_free(line);
+              return type_error(input, errmsg);
+            }
+          }
+          b++;
+        }
+        if (!jvp_utf8_is_valid((const char *)unicode, (const char *)unicode+b)) {
+          jv_free(line);
+          return type_error(input, errmsg);
+        }
+        line = jv_string_append_buf(line, (const char *)unicode, b);
+      }
+    }
+    jv_free(input);
+    return line;
   } else if (!strcmp(fmt_s, "sh")) {
     jv_free(fmt);
     if (jv_get_kind(input) != JV_KIND_ARRAY)

src/builtin.c

@@ -8,7 +8,8 @@ def unique: group_by(.) | map(.[0]);
 def unique_by(f): group_by(f) | map(.[0]);
 def max_by(f): _max_by_impl(map([f]));
 def min_by(f): _min_by_impl(map([f]));
-def add: reduce .[] as $x (null; . + $x);
+def add(f): reduce f as $x (null; . + $x);
+def add: add(.[]);
 def del(f): delpaths([path(f)]);
 def abs: if . < 0 then - . else . end;
 def _assign(paths; $value): reduce path(paths) as $p (.; setpath($p; $value));

src/builtin.jq

@@ -61,14 +61,15 @@ null
 null
 "interpolation"
 
-@text,@json,([1,.]|@csv,@tsv),@html,@uri,@sh,(@base64|.,@base64d)
+@text,@json,([1,.]|@csv,@tsv),@html,(@uri|.,@urid),@sh,(@base64|.,@base64d)
 "!()<>&'\"\t"
 "!()<>&'\"\t"
 "\"!()<>&'\\\"\\t\""
 "1,\"!()<>&'\"\"\t\""
 "1\t!()<>&'\"\\t"
 "!()&lt;&gt;&amp;&apos;&quot;\t"
 "%21%28%29%3C%3E%26%27%22%09"
+"!()<>&'\"\t"
 "'!()<>&'\\''\"\t'"
 "ISgpPD4mJyIJ"
 "!()<>&'\"\t"
@@ -86,6 +87,10 @@ null
 "\u03bc"
 "%CE%BC"
 
+@urid
+"%CE%BC"
+"\u03bc"
+
 @html "<b>\(.)</b>"
 "<script>hax</script>"
 "<b>&lt;script&gt;hax&lt;/script&gt;</b>"
@@ -642,6 +647,19 @@ map_values(.+1)
 [0,1,2]
 [1,2,3]
 
+[add(null), add(range(range(10))), add(empty), add(10,range(10))]
+null
+[null,120,null,55]
+
+# Real-world use case for add(empty)
+.sum = add(.arr[])
+{"arr":[]}
+{"arr":[],"sum":null}
+
+add({(.[]):1}) | keys
+["a","a","b","a","d","b","d","a","d"]
+["a","b","d"]
+
 #
 # User-defined functions
 # Oh god.

tests/jq.test

@@ -0,0 +1,38 @@
+# Tests are groups of three lines: program, input, expected output
+# Blank lines and lines starting with # are ignored
+
+@uri
+"<>&'\"\t"
+"%3C%3E%26%27%22%09"
+
+# decoding encoded output results in same text
+(@uri|@urid)
+"<>&'\"\t"
+"<>&'\"\t"
+
+# testing variable length unicode characters
+@uri
+"a \u03bc \u2230 \ud83d\ude0e"
+"a%20%CE%BC%20%E2%88%B0%20%F0%9F%98%8E"
+
+@urid
+"a%20%CE%BC%20%E2%88%B0%20%F0%9F%98%8E"
+"a \u03bc \u2230 \ud83d\ude0e"
+
+### invalid uri strings
+
+# unicode character should be length 4 (not 3)
+. | try @urid catch .
+"%F0%93%81"
+"string (\"%F0%93%81\") is not a valid uri encoding"
+
+# invalid hex value ('FX')
+. | try @urid catch .
+"%FX%9F%98%8E"
+"string (\"%FX%9F%98%8E\") is not a valid uri encoding"
+
+# trailing utf-8 octets must be formatted like 10xxxxxx
+# 'C0' = 11000000 invalid
+. | try @urid catch .
+"%F0%C0%81%8E"
+"string (\"%F0%C0%81%8E\") is not a valid uri encoding"

tests/man.test

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+. "${0%/*}/setup" "$@"
+
+$VALGRIND $Q $JQ -L "$mods" --run-tests $JQTESTDIR/uri.test