diff --git a/extensions/functions_table.yaml b/extensions/functions_table.yaml new file mode 100644 index 000000000..508980391 --- /dev/null +++ b/extensions/functions_table.yaml @@ -0,0 +1,93 @@ +%YAML 1.2 +--- +# Table functions: Functions that produce relations (zero or more records). +urn: extension:io.substrait:functions_table +table_functions: + - name: "generate_series" + description: >- + Generates a series of integer values from start to stop, incrementing by step. + + Produces zero or more records containing a single integer value. The series + includes both the start and stop values if they fall on a step boundary. + If step is positive, stops when the value exceeds stop. If step is negative, + stops when the value is less than stop. Returns empty if step is zero or if + the step direction doesn't allow reaching stop from start. + impls: + - args: + - name: start + value: i64 + description: The starting value of the series + - name: stop + value: i64 + description: The ending value of the series (inclusive) + - name: step + value: i64 + description: The increment between values + return: + names: + - value + struct: + types: + - i64 + - args: + - name: start + value: i32 + description: The starting value of the series + - name: stop + value: i32 + description: The ending value of the series (inclusive) + - name: step + value: i32 + description: The increment between values + return: + names: + - value + struct: + types: + - i32 + - args: + - name: start + value: i64 + description: The starting value of the series + - name: stop + value: i64 + description: The ending value of the series (inclusive) + return: + names: + - value + struct: + types: + - i64 + - args: + - name: start + value: i32 + description: The starting value of the series + - name: stop + value: i32 + description: The ending value of the series (inclusive) + return: + names: + - value + struct: + types: + - i32 + - name: "unnest" + description: >- + Expands one or more list expressions into a set of rows. + + When multiple lists are provided, produces rows by expanding the lists in parallel (like a zip operation). + For example, unnest([1,2], [3,4]) produces rows: (1,3), (2,4). + If lists have different lengths, shorter lists are padded with nulls. + impls: + - args: + - name: input + value: list + description: The list(s) to unnest + variadic: + min: 1 + return: + names: + - element + struct: + types: + - any1 diff --git a/proto/substrait/algebra.proto b/proto/substrait/algebra.proto index 3c15f7931..e07b68651 100644 --- a/proto/substrait/algebra.proto +++ b/proto/substrait/algebra.proto @@ -552,6 +552,45 @@ message ExpandRel { } } +// Invokes a table-valued function that produces a relation (zero or more records). +// +// +// Table functions produce a table with a schema that can be: +// - Statically defined (concrete types in YAML) +// - Type-parameterized (derived from argument types, e.g., unnest(list) -> {element: T}) +// +// Future extensions may add an optional input field to support transformation +// table functions that operate on input relations. +message TableFunctionRel { + RelCommon common = 1; + + // Points to a function_anchor defined in this plan, which must refer + // to a table function in the associated YAML file. Avoid using + // anchor/reference zero. + uint32 function_reference = 2; + + // The arguments to be bound to the function. This must have exactly the + // number of arguments specified in the function definition from the YAML file, + // and the argument types must also match exactly: + // - Value arguments must be bound using FunctionArgument.value. + // - Type arguments must be bound using FunctionArgument.type. + // - Enum arguments must be bound using FunctionArgument.enum with a + // string that case-insensitively matches one of the allowed options. + repeated FunctionArgument arguments = 3; + + // Options to specify behavior for corner cases, or leave behavior + // unspecified if the consumer does not need specific behavior in these + // cases. + repeated FunctionOption options = 5; + + // The concrete output schema of the relation. For schemas which can be derived + // from their YAML specification this must match the schema computed by evaluating the + // YAML ExpressionNamedStruct with the bound argument types. + NamedStruct table_schema = 6; + + substrait.extensions.AdvancedExtension advanced_extension = 7; +} + // A relation with output field names. // // This is for use at the root of a `Rel` tree. @@ -581,6 +620,7 @@ message Rel { WriteRel write = 19; DdlRel ddl = 20; UpdateRel update = 22; + TableFunctionRel table_function = 23; // Physical relations HashJoinRel hash_join = 13; MergeJoinRel merge_join = 14; diff --git a/proto/substrait/function.proto b/proto/substrait/function.proto index 4377d1d71..11c4c72df 100644 --- a/proto/substrait/function.proto +++ b/proto/substrait/function.proto @@ -106,6 +106,26 @@ message FunctionSignature { } } + message Table { + repeated Argument arguments = 2; + repeated string name = 3; + Description description = 4; + + // The schema expression that defines the output relation structure. + // This must be an ExpressionNamedStruct since table functions always + // produce relations with named fields. The struct can be: + // - Static (concrete field types) + // - Type-parameterized (field types derived from argument types) + DerivationExpression.ExpressionNamedStruct schema = 5; + + oneof final_variable_behavior { + FinalArgVariadic variadic = 6; + FinalArgNormal normal = 7; + } + + repeated Implementation implementations = 8; + } + message Description { string language = 1; string body = 2; diff --git a/site/docs/expressions/table_functions.md b/site/docs/expressions/table_functions.md index b5ab3c5fa..a320cb5f6 100644 --- a/site/docs/expressions/table_functions.md +++ b/site/docs/expressions/table_functions.md @@ -1,8 +1,286 @@ # Table Functions -Table functions produce zero or more records for each input record. Table functions use a signature similar to scalar functions. However, they are not allowed in the same contexts. +Table functions are functions that produce a relation (zero or more records) as output. Table functions are similar to scalar and aggregate functions but produce relations instead of individual values. +!!! warning "Partial Implementation" + **Currently implemented:** 0-input table functions - leaf operators that take constant arguments and produce relations. + **Not yet implemented:** Transformation table functions that accept input relations. -to be completed... +## Function Signature Properties +Table function signatures contain the properties defined below: + +| Property | Description | Required | +| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------- | +| Name | One or more user-friendly, case-sensitive UTF-8 strings that are used to reference this function. | At least one value is required. | +| List of arguments | Argument properties follow the same pattern as [scalar functions](scalar_functions.md#argument-types): value arguments, type arguments, and enumerations. Arguments can be fully defined or calculated with a type expression. | Optional, defaults to niladic. | +| Variadic Behavior | Whether the last argument of the function is variadic or a single argument. If variadic, the argument can optionally have a lower bound (minimum number of instances) and an upper bound (maximum number of instances). | Optional, defaults to single value. | +| Description | Additional description of function for implementers or users. Should be written human-readable to allow exposure to end users. Presented as a map with language => description mappings. | Optional | +| Return Schema | The output schema of the relation, expressed as an `ExpressionNamedStruct`. Can be concrete types or type-parameterized (derived from argument types). May be omitted if schema depends on runtime data content (not determinable from function signature). | Optional (see [Schema Determination](#schema-determination)) | +| Implementation Map | A map of implementation locations for one or more implementations of the given function. Each key is a function implementation type with properties associated with retrieval of that implementation. | Optional | + +Table functions use the same argument types as scalar functions (value arguments, type arguments, and enumerations). See [Scalar Functions - Argument Types](scalar_functions.md#argument-types) for details. + +## Schema Determination + +Table function schemas can be specified in two ways, depending on whether the YAML definition includes a `return` field: + +### Schemas Derivable from YAML + +When a table function's YAML definition includes a `return` field, the schema can be deterministically derived from the function signature and the types of the bound arguments. The plan must include this derived schema in the `table_schema` field with any type parameters resolved. + +**Example YAML definitions** (from `functions_table.yaml`): + +Concrete type example - `generate_series` always produces `{value: i64}`: + +```yaml +- name: "generate_series" + impls: + - args: + - name: start + value: i64 + - name: stop + value: i64 + - name: step + value: i64 + return: + names: + - value + struct: + types: + - i64 +``` + +Type-parameterized and variadic example - `unnest` can take 1 or more lists: + +```yaml +- name: "unnest" + impls: + - args: + - name: input + value: list + variadic: + min: 1 + return: + names: + - element + struct: + types: + - any1 +``` + +### Schemas Determined by Plan Producer + +When a table function's YAML definition omits the `return` field, the schema cannot be determined from type information alone. In these cases, the plan producer must provide the schema in `table_schema`. + +**Example:** A function like `read_parquet(path)` where the schema depends on the actual Parquet file's structure - the YAML would omit the `return` field, and the plan producer would provide the concrete schema in `table_schema`. + +!!! note "Required Constraint" + **If a table function's YAML definition includes a `return` field, the `table_schema` field in the plan MUST match the YAML definition (with any type parameters resolved based on the bound argument types).** + +## Variadic Table Functions + +Table functions can be variadic, meaning the last parameter can be repeated one or more times. This is specified using the `variadic` field in the YAML definition with a `min` value indicating the minimum number of times the parameter must appear. + +**Example:** The `unnest` function is variadic with `min: 1`, allowing it to accept one or more list arguments: + +```yaml +- name: "unnest" + impls: + - args: + - name: input + value: list + variadic: + min: 1 + return: + names: + - element + struct: + types: + - any1 +``` + +When multiple arguments are provided to a variadic table function, the behavior depends on the function's semantics. For example, `unnest` expands multiple lists in parallel (like a zip operation). + +## Usage in Plans + +Table functions are invoked in plans using the `TableFunctionRel` relation type. Table functions can be used anywhere a relation is expected - as a leaf node, or as input to other relational operators like `FilterRel`, `ProjectRel`, etc. + +For details on the `TableFunctionRel` message structure and properties, see the [Table Function](../relations/logical_relations.md#table-function) section in Logical Relations. + +## Examples + +### Example 1: Generating a Sequence + +Generate integers from 1 to 100: + +``` +TableFunctionRel { + function_reference: + arguments: [ + { value: { literal: { i64: 1 } } }, + { value: { literal: { i64: 100 } } } + ] + table_schema: { + names: ["value"] + struct: { + types: [{ i64: {} }] + } + } +} +``` + +**SQL equivalent:** `SELECT * FROM generate_series(1, 100)` + +**Output:** +``` +value +----- +1 +2 +3 +... +100 +``` + +### Example 2: Unnest a Literal Array + +Unnest a literal list into rows: + +``` +TableFunctionRel { + function_reference: + arguments: [ + { value: { literal: { + list: { + values: [ + { string: { value: "apple" } }, + { string: { value: "banana" } }, + { string: { value: "cherry" } } + ] + } + } } } // Type is list + ] + table_schema: { + names: ["element"] + struct: { + types: [{ string: {} }] // T resolved to string from list argument + } + } +} +``` + +**SQL equivalent:** `SELECT * FROM UNNEST(['apple', 'banana', 'cherry'])` + +**Output:** +``` +element +-------- +apple +banana +cherry +``` + +### Example 2b: Variadic Unnest (Multiple Lists) + +Unnest multiple lists to produce their cross product: + +``` +TableFunctionRel { + function_reference: + arguments: [ + { value: { literal: { + list: { + values: [ + { i32: { value: 1 } }, + { i32: { value: 2 } } + ] + } + } } }, // First list: [1, 2] + { value: { literal: { + list: { + values: [ + { i32: { value: 3 } }, + { i32: { value: 4 } } + ] + } + } } } // Second list: [3, 4] + ] + table_schema: { + names: ["element"] // Note: With multiple lists, schema may have multiple fields + struct: { + types: [{ i32: {} }] + } + } +} +``` + +**SQL equivalent:** `SELECT * FROM UNNEST([1, 2], [3, 4])` + +**Output (parallel expansion / zip):** +``` +element +------- +1, 3 +2, 4 +``` + +!!! note "Limitation: Correlated Table Functions" + **The more sophisticated use case - unnesting a column from an existing table - cannot currently be represented.** For example, the SQL query `SELECT element FROM my_table, UNNEST(my_table.array_column)` would require applying the table function once per row of the input table. + + This requires **lateral joins** (correlated subqueries where a table function references columns from an outer relation), which are not yet specified in Substrait. Since TableFunctionRel is currently a leaf operator with no input relation, you cannot use field references in the function arguments. + + Future extensions will add support for transformation table functions and/or lateral join semantics to handle these cases. + + +### Example 3: Composing with Other Operators + +Table functions can be composed with other relational operators. For example, filtering the generated series to get only even numbers: + +``` +FilterRel { + input: { + TableFunctionRel { + function_reference: + arguments: [ + { value: { literal: { i64: 1 } } }, + { value: { literal: { i64: 100 } } } + ] + table_schema: { + names: ["value"] + struct: { + types: [{ i64: {} }] + } + } + } + } + condition: { + scalar_function: { + function_reference: + arguments: [ + { + value: { + scalar_function: { + function_reference: + arguments: [ + { value: { selection: { direct_reference: { struct_field: { field: 0 } } } } }, + { value: { literal: { i64: 2 } } } + ] + } + } + }, + { value: { literal: { i64: 0 } } } + ] + } + } +} +``` + +## Future Extensions + +The current specification focuses on 0-input (generator/leaf) table functions. Future versions may support: + +- **Transformation table functions**: Functions that take an input relation and transform it (by adding an optional `input` field to `TableFunctionRel`) +- **Set-returning functions**: Functions that process input records and produce multiple output records per input +- **Lateral joins**: Applying table functions to each row of an input relation diff --git a/site/docs/extensions/generate_function_docs.py b/site/docs/extensions/generate_function_docs.py index 5bb2171d7..348e2f2c5 100644 --- a/site/docs/extensions/generate_function_docs.py +++ b/site/docs/extensions/generate_function_docs.py @@ -125,19 +125,29 @@ def write_markdown(file_obj: dict, file_name: str) -> None: # If the return value for the function implementation is multiple lines long, # print each line separately. This is the case for some functions in # functions_arithmetic_decimal.yaml - if "\n" in impl["return"]: - mdFile.new_line( - f"{count}. {function_name}({func_concat_arg_input_values}): -> " - ) - multiline_return_str = "\t" + impl["return"] - multiline_return_str = multiline_return_str.replace("\n", "\n\t") - mdFile.new_line("\t```") - mdFile.new_line(f"{multiline_return_str}") - mdFile.new_line("\t```") + # Table functions may omit the return field if schema depends on runtime data + if "return" in impl: + if "\n" in impl["return"]: + mdFile.new_line( + f"{count}. {function_name}({func_concat_arg_input_values}): -> " + ) + multiline_return_str = "\t" + impl["return"] + multiline_return_str = multiline_return_str.replace( + "\n", "\n\t" + ) + mdFile.new_line("\t```") + mdFile.new_line(f"{multiline_return_str}") + mdFile.new_line("\t```") + else: + mdFile.new_line( + f"{count}. {function_name}({func_concat_arg_input_values}): -> " + f"`{impl['return']}`" + ) else: + # Return type not specified (e.g., table functions with runtime-dependent schemas) mdFile.new_line( f"{count}. {function_name}({func_concat_arg_input_values}): -> " - f"`{impl['return']}`" + f"`schema determined at runtime`" ) if "description" in function_spec: diff --git a/site/docs/relations/logical_relations.md b/site/docs/relations/logical_relations.md index c2317a0ca..b10cad679 100644 --- a/site/docs/relations/logical_relations.md +++ b/site/docs/relations/logical_relations.md @@ -97,7 +97,7 @@ possible approach is that a chunk should only be read if the midpoint of the chu #### Iceberg Table Type -A Iceberg Table is a table built on [Apache Iceberg](https://iceberg.apache.org/). Iceberg tables can be read by either directly reading a [metadata file](https://iceberg.apache.org/spec/#table-metadata) or by consulting a [catalog](https://iceberg.apache.org/concepts/catalog/). +A Iceberg Table is a table built on [Apache Iceberg](https://iceberg.apache.org/). Iceberg tables can be read by either directly reading a [metadata file](https://iceberg.apache.org/spec/#table-metadata) or by consulting a [catalog](https://iceberg.apache.org/concepts/catalog/). ##### Metadata File Reading @@ -110,6 +110,62 @@ Points to an [Iceberg metadata file](https://iceberg.apache.org/spec/#table-meta | snapshot_timestamp | The snapshot that should be read using timestamp. If not provided, the current snapshot is read. | Optional | +## Table Function + +The table function operator invokes a function that produces a relation (zero or more records). These are leaf operators that take constant (non-relational) arguments and generate data. + +Like scalar function return types, table function schemas can be concrete types or reference type parameters from arguments. The schema is expressed as an `ExpressionNamedStruct` and is either derived from the function signature or must be explicitly provided when it depends on runtime data content. It is preferred to explicitly provide a schema derivation in the YAML file when possible. + +Table functions can be **variadic**, meaning the last parameter can be repeated one or more times (specified via the `variadic` field in YAML). + +| Signature | Value | +| -------------------- | ------------------------------------------- | +| Inputs | 0 (leaf operator) | +| Outputs | 1 | + +### Table Function Properties + +| Property | Description | Required | +| ------------------ | ------------------------------------------------------------ | -------- | +| Function Reference | Points to a function_anchor defined in the plan, referencing a table function in the extension YAML files | Required | +| Arguments | Constant expressions to pass as arguments to the function. Must match the function signature exactly. Must be literals or expressions that can be evaluated without input data. | Required | +| Table Schema | The output schema (NamedStruct). Always present. **Must match YAML definition (with type parameters resolved) when YAML defines a `return` field.** | Required | + +### Use Cases + +Common examples of table functions include: +- `generate_series`: Generate sequences of numbers +- `unnest`: Expand arrays or lists into rows (variadic - can accept multiple lists) + +### Example + +Generate a series of integers from 1 to 100: + +``` +TableFunctionRel { + function_reference: + arguments: [ + { value: { literal: { i64: 1 } } }, + { value: { literal: { i64: 100 } } } + ] + table_schema: { + names: ["value"] + struct: { + types: [{ i64: {} }] + } + } +} +``` + +See [Table Functions](../expressions/table_functions.md) for detailed documentation. + +=== "TableFunctionRel Message" + + ```proto +%%% proto.algebra.TableFunctionRel %%% + ``` + + ## Filter Operation The filter operator eliminates one or more records from the input data based on a boolean filter expression. diff --git a/site/docs/spec/specification.md b/site/docs/spec/specification.md index d19d10451..2fd18f2b6 100644 --- a/site/docs/spec/specification.md +++ b/site/docs/spec/specification.md @@ -29,11 +29,17 @@ The specification has passed the initial design phase and is now in the final st | [Binary Serialization](../serialization/binary_serialization.md) | A high performance & compact binary representation of the plan specification. | +## Components (Partially Implemented) + +| Section | Description | +| ------------------------------------------------------------ | ------------------------------------------------------------ | +| [Table Functions](../expressions/table_functions.md) | **Partial implementation:** Functions that produce relations (0..N records). Table functions can accept 0 or more input relations. **Currently, only 0-input functions are implemented** - these are leaf operators that take constant arguments and generate data. Examples include sequence generation (`generate_series`) and expanding collections (`unnest`). Transformation table functions that accept input relations are not yet implemented. | + + ## Components (Designed but not Implemented) | Section | Description | | ------------------------------------------------------------ | ------------------------------------------------------------ | -| [Table Functions](../expressions/table_functions.md) | Functions that convert one or more values from an input record into 0..N output records. Example include operations such as explode, pos-explode, etc. | | [User Defined Relations](../relations/user_defined_relations.md) | Installed and reusable relational operations customized to a particular platform. | | [Embedded Relations](../relations/embedded_relations.md) | Relational operations where plans contain the "machine code" to directly execute the necessary operations. | | [Physical Relations](../relations/physical_relations.md) | Specific execution sub-variations of common relational operations that describe have multiple unique physical variants associated with a single logical operation. Examples include hash join, merge join, nested loop join, etc. | diff --git a/text/simple_extensions_schema.yaml b/text/simple_extensions_schema.yaml index f6bef9862..3f8fb2db1 100644 --- a/text/simple_extensions_schema.yaml +++ b/text/simple_extensions_schema.yaml @@ -64,6 +64,10 @@ properties: type: array items: $ref: "#/$defs/windowFunction" + table_functions: + type: array + items: + $ref: "#/$defs/tableFunction" $defs: type: @@ -307,3 +311,33 @@ $defs: window_type: type: string enum: [STREAMING, PARTITION] + + tableFunction: + type: object + additionalProperties: false + required: [name, impls] + properties: + name: + type: string + description: + type: string + impls: + type: array + minItems: 1 + items: + type: object + additionalProperties: false + properties: + args: + $ref: "#/$defs/arguments" + variadic: + $ref: "#/$defs/variadicBehavior" + return: + # The output schema (DerivationExpression, typically ExpressionNamedStruct). + # This can be a static struct definition or a dynamic derivation based on + # argument types (e.g., type-parameterized functions). + # Omit this field only if the schema depends on runtime data content + # (not determinable from function signature). + $ref: "#/$defs/type" + implementation: + $ref: "#/$defs/implementation"