vdk-core: add log stacktrace flag

Why?

When we log the stack trace, we're logging exceptions that we've already
logged during execution. This increases the logs length by about 30%.

What?

Add flag that disables/enables logging the stack trace on exit code 1

How has this been tested?

Ran vdk locally with a failing data job

What type of change are you making?

Feature/non-breaking

Author: Dilyan Marinov

projects/vdk-core/src/vdk/internal/builtin_plugins/config/vdk_config.py

@@ -20,6 +20,7 @@
 LOG_CONFIG = "LOG_CONFIG"
 LOG_LEVEL_VDK = "LOG_LEVEL_VDK"
 LOG_LEVEL_MODULE = "LOG_LEVEL_MODULE"
+LOG_STACK_TRACE_ON_EXIT = "LOG_STACK_TRACE_ON_EXIT"
 WORKING_DIR = "WORKING_DIR"
 ATTEMPT_ID = "ATTEMPT_ID"
 EXECUTION_ID = "EXECUTION_ID"
@@ -83,6 +84,13 @@ def vdk_configure(self, config_builder: ConfigurationBuilder) -> None:
             "Allowed values: CRITICAL, ERROR, WARNING, INFO, DEBUG. "
             "If not set python default or one set by vdk -v LEVEL is used. ",
         )
+        config_builder.add(
+            LOG_STACK_TRACE_ON_EXIT,
+            False,
+            True,
+            "Controls whether the full stack trace is displayed again on exit code 1. "
+            "False by default, shoud be set to true in production environments for more debug output. ",
+        )
         config_builder.add(JOB_GITHASH, "unknown")
         config_builder.add(
             OP_ID,

projects/vdk-core/src/vdk/internal/cli_entry.py

@@ -153,7 +153,8 @@ def vdk_main(
             # if at least one hook implementation returned handled, means we do
             # not need to log the exception
             if not (True in handled):
-                log.exception("Exiting with exception.")
+                if core_context.configuration.get_value("LOG_STACK_TRACE_ON_EXIT"):
+                    log.exception("Exiting with exception.")
                 exit_code = 1
             else:
                 exit_code = 0