feat: introduce GenerateRel for lateral view and unnest operations

extensions/functions_table_generic.yaml

@@ -0,0 +1,109 @@
+%YAML 1.2
+---
+urn: extension:io.substrait:functions_table_generic
+table_functions:
+  - name: explode
+    description: >-
+      Produces one row per array element.
+
+      For each input row, this function generates one output row for each element
+      in the input array. The output is a struct with a single field 'value'
+      containing the array element.
+
+      Empty arrays produce zero output rows. For outer semantics (producing a row
+      with NULL when the array is empty), use GenerateRel's preserve_on_empty option.
+    impls:
+      - args:
+          - name: input
+            value: list<any1>
+            description: The array to explode.
+        return: struct<any1>
+
+  - name: explode
+    description: >-
+      Produces one row per map entry.
+
+      For each input row, this function generates one output row for each key-value
+      pair in the input map. The output is a struct with two fields: 'key' and 'value'.
+      Note that map entries may be visited in any order.
+
+      Empty maps produce zero output rows. For outer semantics (producing a row
+      with NULLs when the map is empty), use GenerateRel's preserve_on_empty option.
+
+      Some systems call this unnest.
+    impls:
+      - args:
+          - name: input
+            value: map<any1, any2>
+            description: The map to explode.
+        return: struct<any1, any2>
+
+  - name: posexplode
+    description: >-
+      Produces one row per array element with position.
+
+      For each input row, this function generates one output row for each element
+      in the input array. The output is a struct with two fields:
+      - 'pos': 0-indexed position of the element in the array (i64)
+      - 'value': the array element
+
+      Empty arrays produce zero output rows. For outer semantics (producing a row
+      with NULLs when the array is empty), use GenerateRel's preserve_on_empty option.
+    impls:
+      - args:
+          - name: input
+            value: list<any1>
+            description: The array to explode with positions.
+        return: struct<i64, any1>
+
+  - name: unnest
+    description: >-
+      Expands one or more lists into rows by zipping them in parallel (PostgreSQL semantics).
+
+      When multiple lists are provided, produces rows by expanding the lists in parallel,
+      similar to a zip operation. For example, unnest([1,2], [3,4]) produces rows: (1,3), (2,4).
+
+      If lists have different lengths, shorter lists are padded with NULL values to match
+      the length of the longest list.
+
+      Empty lists (when all inputs are empty) produce zero output rows. For outer semantics
+      (producing a row with NULLs when all lists are empty), use GenerateRel's preserve_on_empty option.
+    impls:
+      - args:
+          - name: input
+            value: list<any1>
+            description: The array to unnest.
+        return: struct<any1>
+      - args:
+          - name: input1
+            value: list<any1>
+            description: First array to unnest.
+          - name: input2
+            value: list<any2>
+            description: Second array to unnest.
+        return: struct<any1, any2>
+      - args:
+          - name: input1
+            value: list<any1>
+            description: First array to unnest.
+          - name: input2
+            value: list<any2>
+            description: Second array to unnest.
+          - name: input3
+            value: list<any3>
+            description: Third array to unnest.
+        return: struct<any1, any2, any3>
+      - args:
+          - name: input1
+            value: list<any1>
+            description: First array to unnest.
+          - name: input2
+            value: list<any2>
+            description: Second array to unnest.
+          - name: input3
+            value: list<any3>
+            description: Third array to unnest.
+          - name: input4
+            value: list<any4>
+            description: Fourth array to unnest.
+        return: struct<any1, any2, any3, any4>

grammar/FuncTestCaseLexer.g4

@@ -11,6 +11,7 @@ Whitespace    : [ \t\n\r]+ -> channel(HIDDEN) ;
 TripleHash: '###';
 SubstraitScalarTest: 'SUBSTRAIT_SCALAR_TEST';
 SubstraitAggregateTest: 'SUBSTRAIT_AGGREGATE_TEST';
+SubstraitTableTest: 'SUBSTRAIT_TABLE_TEST';
 SubstraitInclude: 'SUBSTRAIT_INCLUDE';
 SubstraitDependency: 'SUBSTRAIT_DEPENDENCY';
 

grammar/FuncTestCaseParser.g4

@@ -7,15 +7,33 @@ options {
 }
 
 doc
-    : header testGroup+ EOF
+    : scalarDoc
+    | aggregateDoc
+    | tableDoc
     ;
 
-header
-    : version include dependency*
+scalarDoc
+    : scalarHeader scalarTestGroup+ EOF
     ;
 
-version
-    : TripleHash (SubstraitScalarTest | SubstraitAggregateTest) Colon FormatVersion
+aggregateDoc
+    : aggregateHeader aggregateTestGroup+ EOF
+    ;
+
+tableDoc
+    : tableHeader tableTestGroup+ EOF
+    ;
+
+scalarHeader
+    : TripleHash SubstraitScalarTest Colon FormatVersion include dependency*
+    ;
+
+aggregateHeader
+    : TripleHash SubstraitAggregateTest Colon FormatVersion include dependency*
+    ;
+
+tableHeader
+    : TripleHash SubstraitTableTest Colon FormatVersion include dependency*
     ;
 
 include
@@ -34,9 +52,16 @@ testCase
     : functionName=identifier OParen arguments CParen ( OBracket funcOptions CBracket )? Eq result
     ;
 
-testGroup
-    : testGroupDescription? (testCase)+                          #scalarFuncTestGroup
-    | testGroupDescription? (aggFuncTestCase)+                   #aggregateFuncTestGroup
+scalarTestGroup
+    : testGroupDescription? testCase+
+    ;
+
+aggregateTestGroup
+    : testGroupDescription? aggFuncTestCase+
+    ;
+
+tableTestGroup
+    : testGroupDescription? tableFuncTestCase+
     ;
 
 arguments
@@ -76,6 +101,18 @@ aggFuncTestCase
     : aggFuncCall ( OBracket funcOptions CBracket )? Eq result
     ;
 
+tableFuncTestCase
+    : functionName=identifier OParen arguments CParen ( OBracket funcOptions CBracket )? Eq multiRowResult
+    ;
+
+multiRowResult
+    : OBracket (rowTuple (Comma rowTuple)*)? CBracket DoubleColon structType
+    ;
+
+rowTuple
+    : OParen (literal (Comma literal)*)? CParen
+    ;
+
 aggFuncCall
     : tableData funcName=identifier OParen qualifiedAggregateFuncArgs? CParen           #multiArgAggregateFuncCall
     | tableRows functName=identifier OParen aggregateFuncArgs? CParen                   #compactAggregateFuncCall
@@ -344,12 +381,16 @@ parameterizedType
     | precisionTimestampType
     | precisionTimestampTZType
     | funcType
+    | structType
 // TODO implement the rest of the parameterized types
-//  | Struct isnull='?'? Lt expr (Comma expr)* Gt #struct
 //  | NStruct isnull='?'? Lt Identifier expr (Comma Identifier expr)* Gt #nStruct
 //  | Map isnull='?'? Lt key=expr Comma value=expr Gt #map
   ;
 
+structType
+    : Struct isnull=QMark? OAngleBracket dataType (Comma dataType)* CAngleBracket
+  ;
+
 numericParameter
   : IntegerLiteral #integerLiteral
   ;

proto/substrait/algebra.proto

@@ -556,6 +556,41 @@ message ExpandRel {
   }
 }
 
+// The Generate operator applies a table function to produce zero or more output
+// rows for each input row. This operation is used to flatten or "explode" nested
+// data structures like arrays and maps into separate rows, similar to SQL's
+// LATERAL VIEW EXPLODE or UNNEST operations.
+message GenerateRel {
+  RelCommon common = 1;
+
+  // The input relation to generate from. Each input row will produce zero or
+  // more output rows based on the generator table function.
+  Rel input = 2;
+
+  // The table expression to apply to each input row. This produces zero
+  // or more rows per input row. The output_type defines the schema
+  // of the generated columns, which are appended to the input schema.
+  //
+  // Required.
+  TableExpression generator = 3;
+
+  // Controls behavior when the generator produces zero rows for an input record.
+  //
+  // When false (default):
+  //   Input rows that produce zero output rows are excluded from the result.
+  //
+  // When true:
+  //   Input rows that produce zero output rows still generate one output row
+  //   with the input fields populated and the generator output fields set to NULL.
+  //   This is equivalent to Spark's "LATERAL VIEW OUTER" or can be achieved in
+  //   standard SQL via LEFT JOIN LATERAL.
+  //
+  // Optional; defaults to false.
+  bool preserve_on_empty = 4;
+
+  substrait.extensions.AdvancedExtension advanced_extension = 10;
+}
+
 // A relation with output field names.
 //
 // This is for use at the root of a `Rel` tree.
@@ -585,6 +620,7 @@ message Rel {
     WriteRel write = 19;
     DdlRel ddl = 20;
     UpdateRel update = 22;
+    GenerateRel generate = 23;
     // Physical relations
     HashJoinRel hash_join = 13;
     MergeJoinRel merge_join = 14;
@@ -1690,6 +1726,54 @@ message Expression {
   }
 }
 
+// TableExpression represents expressions that produce zero or more rows,
+// as opposed to Expression which produces a single scalar value.
+// Table expressions are used in relational contexts where multi-row output
+// is expected (e.g., GenerateRel for lateral views and unnest operations).
+//
+// Since they have different cardinality than Expression expressions,
+// TableExpression expressions cannot be composed with other expressions.
+message TableExpression {
+  oneof table_expr_type {
+    TableFunction table_function = 1;
+  }
+
+  // A table function call that produces zero or more rows per invocation.
+  // Table functions are similar to scalar functions but produce multiple output rows
+  // instead of a single scalar value. They are commonly used with the GenerateRel
+  // operator to flatten nested data structures like arrays and maps.
+  message TableFunction {
+    // Points to a function_anchor defined in this plan, which must refer
+    // to a table function in the associated YAML file. Table functions are
+    // distinguished from scalar functions by their ability to produce multiple
+    // output rows. Required; 0 is a valid anchor/reference.
+    uint32 function_reference = 1;
+
+    // The arguments to be bound to the table function. This must have exactly the
+    // number of arguments specified in the function definition, and the argument
+    // types must match:
+    //
+    //  - Value arguments must be bound using FunctionArgument.value
+    //  - Type arguments must be bound using FunctionArgument.type
+    //  - Enum arguments must be bound using FunctionArgument.enum
+    //
+    // For table functions, arguments typically reference fields from the input
+    // relation that contain nested data (arrays, maps, etc.) to be exploded.
+    repeated FunctionArgument arguments = 2;
+
+    // Options to specify behavior for corner cases (e.g., handling malformed data,
+    // ordering within generated rows, deduplication).
+    repeated FunctionOption options = 3;
+
+    // Must be set to the output type of the table function. For table functions,
+    // this must be a Struct type where each field represents a column in the
+    // generated output rows.
+    //
+    // Required.
+    Type output_type = 4;
+  }
+}
+
 // Expression that represents a dynamic parameter.
 // Dynamic parameters are identified by a surrogate key within a plan.
 message DynamicParameter {