Odin test runner, conforms to test runner interface version 1 (#7)

* Odin test runner, conforms to test runner interface version 1

* send the correct status to jq

* remove INFO messages from test output

* patch test runner JSON output bug

Author: glennj

Dockerfile

@@ -1,8 +1,49 @@
-FROM alpine:3.18
+# dockerfile recipe from https://github.com/SnappyBeeBit/docker-odin/blob/main/Dockerfile.odin-dev-latest
 
-# install packages required to run the tests
-RUN apk add --no-cache jq coreutils
+ARG ODIN_REF=dev-2025-10
+
+# ---------- Build stage ----------
+FROM ubuntu:24.04 AS build
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y --no-install-recommends \
+      git build-essential make ca-certificates \
+      clang-18 llvm-18-dev \
+  && update-ca-certificates \
+  && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /src
+ARG ODIN_REF
+RUN git clone --depth=1 --branch "${ODIN_REF}" https://github.com/odin-lang/Odin.git
+
+WORKDIR /src/Odin
+# Tell the build which llvm-config to use to avoid symlink hacks
+RUN make -j"$(nproc)" release-native LLVM_CONFIG=llvm-config-18
+
+# Fix a bug in this Odin release.  When we upgrade, revisit this.
+RUN sed -E -i '983,984s/\<err\>/marshall_err/g' /src/Odin/core/testing/runner.odin
+
+# ---------- Runtime stage ----------
+FROM ubuntu:24.04 AS runtime
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y --no-install-recommends \
+      ca-certificates libstdc++6 libllvm18 \
+      clang lld build-essential \
+      jq gawk locales \
+  && update-ca-certificates \
+  && locale-gen en_US.UTF-8 \
+  && rm -rf /var/lib/apt/lists/*
+
+# Odin binary and libraries
+COPY --from=build /src/Odin/odin /usr/local/bin/odin
+COPY --from=build /src/Odin/core /usr/local/lib/odin/core
+COPY --from=build /src/Odin/vendor /usr/local/lib/odin/vendor
+COPY --from=build /src/Odin/base /usr/local/lib/odin/base
+
+ENV LC_ALL=en_US.UTF-8
+ENV ODIN_ROOT=/usr/local/lib/odin
 
 WORKDIR /opt/test-runner
-COPY . .
+COPY ./bin/ bin/
 ENTRYPOINT ["/opt/test-runner/bin/run.sh"]

bin/run.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env sh
+#!/usr/bin/env bash
 
 # Synopsis:
 # Run the test runner on a solution.
@@ -17,7 +17,7 @@
 
 # If any required arguments is missing, print the usage and exit
 if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then
-    echo "usage: ./bin/run.sh exercise-slug path/to/solution/folder/ path/to/output/directory/"
+    echo "usage: $0 exercise-slug path/to/solution/folder/ path/to/output/directory/"
     exit 1
 fi
 
@@ -26,26 +26,51 @@ solution_dir=$(realpath "${2%/}")
 output_dir=$(realpath "${3%/}")
 results_file="${output_dir}/results.json"
 
+cd "${solution_dir}" || exit 1
+
 # Create the output directory if it doesn't exist
 mkdir -p "${output_dir}"
 
 echo "${slug}: testing..."
 
 # Run the tests for the provided implementation file and redirect stdout and
 # stderr to capture it
-test_output=$(false)
-# TODO: substitute "false" with the actual command to run the test:
-# test_output=$(command_to_run_tests 2>&1)
+compile_options=(
+    # ref https://odin-lang.org/docs/testing/#compile-time-options
+    -define:ODIN_TEST_LOG_LEVEL=warning
+    -define:ODIN_TEST_SHORT_LOGS=true 
+    -define:ODIN_TEST_FANCY=false
+    -define:ODIN_TEST_RANDOM_SEED=1234567890
+    -define:ODIN_TEST_TRACK_MEMORY=false
+)
+
+raw_output=$( odin test . "${compile_options[@]}"  2>&1 )
+rc=$?
 
 # Write the results.json file based on the exit code of the command that was 
 # just executed that tested the implementation file
-if [ $? -eq 0 ]; then
-    jq -n '{version: 1, status: "pass"}' > ${results_file}
+if [ $rc -eq 0 ]; then
+    jq -n '{version: 1, status: "pass"}' > "${results_file}"
 else
-    # OPTIONAL: Sanitize the output
-    # In some cases, the test output might be overly verbose, in which case stripping
-    # the unneeded information can be very helpful to the student
-    # sanitized_test_output=$(printf "${test_output}" | sed -n '/Test results:/,$p')
+    # Sanitize the output:
+    # remove text that can change from run to run, or from system to system.
+    test_output=$(
+        gawk -v pwd="${PWD}/" '
+            /To run only the failed test,/ {exit}
+            /Finished [[:digit:]]+ tests in / { sub(/ in [[:digit:].]+.s/, "") }
+            {
+                gsub(pwd, "") # trim full paths from filenames
+                print
+            }
+        ' <<< "$raw_output"
+    )
+
+    if [[ $test_output =~ .*$'\nFinished '[[:digit:]]+' tests.'.* ]]; then
+        # successfully compiled, but test failures
+        status='fail'
+    else
+        status='error'
+    fi
 
     # OPTIONAL: Manually add colors to the output to help scanning the output for errors
     # If the test output does not contain colors to help identify failing (or passing)
@@ -54,7 +79,9 @@ else
     #      | GREP_COLOR='01;31' grep --color=always -E -e '^(ERROR:.*|.*failed)$|$' \
     #      | GREP_COLOR='01;32' grep --color=always -E -e '^.*passed$|$')
 
-    jq -n --arg output "${test_output}" '{version: 1, status: "fail", message: $output}' > ${results_file}
+    jq -n --arg output "${test_output}" \
+          --arg status "${status}" \
+        '{version: 1, status: $status, message: $output}' > "${results_file}"
 fi
 
 echo "${slug}: done"

tests/all-fail/expected_results.json

@@ -0,0 +1 @@
+partial-fail

tests/empty-file/expected_results.json

@@ -1,5 +1,5 @@
 {
   "version": 1,
   "status": "fail",
-  "message": "TODO: replace with correct output"
+  "message": "[ERROR] --- [partial_fail_test.odin:7] expected two_fer() to be One for you, one for me., got One for failure, one for me.\n\nFinished 3 tests. 1 test failed.\n - partial_fail.test_no_name_given      \texpected two_fer() to be One for you, one for me., got One for failure, one for me."
 }

tests/partial-fail/.gitignore

@@ -0,0 +1,8 @@
+package partial_fail
+
+import "core:fmt"
+
+two_fer :: proc(name: string = "failure") -> string {
+	return fmt.tprintf("One for {}, one for me.", name)
+}
+

tests/partial-fail/expected_results.json

@@ -0,0 +1,19 @@
+package partial_fail
+
+import "core:testing"
+
+@(test)
+test_no_name_given :: proc(t: ^testing.T) {
+	testing.expect_value(t, two_fer(), "One for you, one for me.")
+}
+
+@(test)
+test_a_name_given :: proc(t: ^testing.T) {
+	testing.expect_value(t, two_fer("Alice"), "One for Alice, one for me.")
+}
+
+@(test)
+test_another_name_given :: proc(t: ^testing.T) {
+	testing.expect_value(t, two_fer("Bob"), "One for Bob, one for me.")
+}
+

tests/partial-fail/partial_fail.odin

@@ -0,0 +1,7 @@
+package success
+
+import "core:fmt"
+
+two_fer :: proc(name: string = "you") -> string {
+	return fmt.tprintf("One for {}, one for me.", name)
+}

tests/partial-fail/partial_fail_test.odin

@@ -0,0 +1,18 @@
+package success
+
+import "core:testing"
+
+@(test)
+test_no_name_given :: proc(t: ^testing.T) {
+	testing.expect_value(t, two_fer(), "One for you, one for me.")
+}
+
+@(test)
+test_a_name_given :: proc(t: ^testing.T) {
+	testing.expect_value(t, two_fer("Alice"), "One for Alice, one for me.")
+}
+
+@(test)
+test_another_name_given :: proc(t: ^testing.T) {
+	testing.expect_value(t, two_fer("Bob"), "One for Bob, one for me.")
+}