[SPARK-52345][SQL] Fix NULL behavior in scripting conditions

### What changes were proposed in this pull request?

`NULL` is a valid Boolean "value" in SQL. Scripting engine is not properly handling cases when conditions (in if-else, case, while, repeat statements) returned NULL.

Scripting engine throws an exception in such cases, stating that the NULL is an invalid value.
Scripting engine should consider such NULLs as a False Boolean value.

### Why are the changes needed?

Fixes the wrong behavior in condition evaluation for scripting statements.

### Does this PR introduce _any_ user-facing change?

No.

### How was this patch tested?

Old and new unit tests.

### Was this patch authored or co-authored using generative AI tooling?

No.

Closes #51047 from davidm-db/scripting_conditions_null_fix.

Authored-by: David Milicevic <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>

Author: davidm-db

common/utils/src/main/resources/error/error-conditions.json

@@ -162,12 +162,6 @@
     ],
     "sqlState" : "22003"
   },
-  "BOOLEAN_STATEMENT_WITH_EMPTY_ROW" : {
-    "message" : [
-      "Boolean statement <invalidStatement> is invalid. Expected single row with a value of the BOOLEAN type, but got an empty row."
-    ],
-    "sqlState" : "21000"
-  },
   "CALL_ON_STREAMING_DATASET_UNSUPPORTED" : {
     "message" : [
       "The method <methodName> can not be called on streaming Dataset/DataFrame."

sql/catalyst/src/main/scala/org/apache/spark/sql/errors/SqlScriptingErrors.scala

@@ -102,16 +102,6 @@ private[sql] object SqlScriptingErrors {
         "sqlScriptingEnabled" -> toSQLConf(SQLConf.SQL_SCRIPTING_ENABLED.key)))
   }
 
-  def booleanStatementWithEmptyRow(
-      origin: Origin,
-      stmt: String): Throwable = {
-    new SqlScriptingException(
-      origin = origin,
-      errorClass = "BOOLEAN_STATEMENT_WITH_EMPTY_ROW",
-      cause = null,
-      messageParameters = Map("invalidStatement" -> toSQLStmt(stmt)))
-  }
-
   def positionalParametersAreNotSupportedWithSqlScripting(): Throwable = {
     new SqlScriptingException(
       origin = null,

sql/core/src/main/scala/org/apache/spark/sql/scripting/SqlScriptingExecutionNode.scala

@@ -84,18 +84,16 @@ trait NonLeafStatementExec extends CompoundStatementExec {
       assert(!statement.isExecuted)
       statement.isExecuted = true
 
-      // DataFrame evaluates to True if it is single row, single column
-      //  of boolean type with value True.
+      // First, it is checked if DataFrame represents a valid Boolean condition - single row,
+      //   single column of Boolean type.
+      // If that is true, the condition evaluates to True only if the Boolean value is True.
+      //   Otherwise, if the Boolean value is False or NULL, the condition evaluates to False.
       val df = statement.buildDataFrame(session)
       df.schema.fields match {
         case Array(field) if field.dataType == BooleanType =>
           df.limit(2).collect() match {
             case Array(row) =>
-              if (row.isNullAt(0)) {
-                throw SqlScriptingErrors.booleanStatementWithEmptyRow(
-                  statement.origin, statement.getText)
-              }
-              row.getBoolean(0)
+              if (row.isNullAt(0)) false else row.getBoolean(0)
             case _ =>
               throw SparkException.internalError(
                 s"Boolean statement ${statement.getText} is invalid. It returns more than one row.")