From 1635a4df5cf8841c46f9df88277500f2f01fbb65 Mon Sep 17 00:00:00 2001 From: Ben Bellick Date: Tue, 21 Oct 2025 17:25:36 -0400 Subject: [PATCH 01/10] feat: introduction of simple table functions introduces a notion of YAML-defined functions which are represented as a new `TableFunctionRel`. Closes #823 --- extensions/functions_table.yaml | 102 +++++++++ proto/substrait/algebra.proto | 46 ++++ proto/substrait/function.proto | 13 ++ site/docs/expressions/table_functions.md | 261 ++++++++++++++++++++++- site/docs/relations/logical_relations.md | 58 ++++- site/docs/spec/specification.md | 8 +- text/simple_extensions_schema.yaml | 34 +++ 7 files changed, 518 insertions(+), 4 deletions(-) create mode 100644 extensions/functions_table.yaml diff --git a/extensions/functions_table.yaml b/extensions/functions_table.yaml new file mode 100644 index 000000000..a1d97e363 --- /dev/null +++ b/extensions/functions_table.yaml @@ -0,0 +1,102 @@ +%YAML 1.2 +--- +# Table functions: Functions that produce relations (zero or more records). +# Currently, only 0-input functions are supported - these take constant arguments +# and generate data as leaf operators. +urn: extension:io.substrait:functions_table +table_functions: + - name: "generate_series" + description: >- + Generates a series of integer values from start to stop, incrementing by step. + + Takes constant arguments and produces zero or more records containing a single + integer value. The series includes both the start and stop values if they fall + on a step boundary. If step is positive, stops when the value exceeds stop. + If step is negative, stops when the value is less than stop. Returns empty if + step is zero or if the step direction doesn't allow reaching stop from start. + impls: + - args: + - name: start + value: i64 + description: The starting value of the series + - name: stop + value: i64 + description: The ending value of the series (inclusive) + - name: step + value: i64 + description: The increment between values + constant: true + deterministic: true + sessionDependent: false + schema: + names: + - value + struct: + types: + - i64 + - args: + - name: start + value: i32 + description: The starting value of the series + - name: stop + value: i32 + description: The ending value of the series (inclusive) + - name: step + value: i32 + description: The increment between values + constant: true + deterministic: true + sessionDependent: false + schema: + names: + - value + struct: + types: + - i32 + - args: + - name: start + value: i64 + description: The starting value of the series + - name: stop + value: i64 + description: The ending value of the series (inclusive) + deterministic: true + sessionDependent: false + schema: + names: + - value + struct: + types: + - i64 + - args: + - name: start + value: i32 + description: The starting value of the series + - name: stop + value: i32 + description: The ending value of the series (inclusive) + deterministic: true + sessionDependent: false + schema: + names: + - value + struct: + types: + - i32 + - name: "unnest" + description: Expands a list literal into a set of rows, one row per element. + impls: + - args: + - name: input + value: "list" + description: The list to unnest + deterministic: true + sessionDependent: false + # Schema references type parameter T from list + # The field type is derived from the list element type + schema: + names: + - element + struct: + types: + - T diff --git a/proto/substrait/algebra.proto b/proto/substrait/algebra.proto index 3c15f7931..4b57a084c 100644 --- a/proto/substrait/algebra.proto +++ b/proto/substrait/algebra.proto @@ -552,6 +552,51 @@ message ExpandRel { } } +// Invokes a table-valued function that produces a relation (zero or more records). +// +// +// Table functions produce a table with either: +// - A schema that can be derived based on argument types (type-parameterized functions) +// - A schema that depends on runtime data (use derived: false) +// +// Future extensions may add an optional input field to support transformation +// table functions that operate on input relations. +message TableFunctionRel { + RelCommon common = 1; + + // Points to a function_anchor defined in this plan, which must refer + // to a table function in the associated YAML file. Avoid using + // anchor/reference zero. + uint32 function_reference = 2; + + // The arguments to be bound to the function. This must have exactly the + // number of arguments specified in the function definition from the YAML file, + // and the argument types must also match exactly: + // + // - Value arguments must be bound using FunctionArgument.value. + // Currently (0-input functions only), expressions must be constants + // (literals or expressions evaluable without input data). + // - Type arguments must be bound using FunctionArgument.type. + // - Enum arguments must be bound using FunctionArgument.enum with a + // string that case-insensitively matches one of the allowed options. + repeated FunctionArgument arguments = 3; + + // The derived fields indicates whether or not the YAML file produced the schema: + // - If true, the table_schema was produced purely from the type expressions in the + // YAML file + the types of the provided arguments + // - If false, the table_schema was produced by the plan producer + // + // This value is required to be true if and only if a schema is provided in the YAML + // definition of this function. + bool derived = 4; + + // The schema of the output relation. This schema is required to match the implied schema + // by the YAML definition, if a schema is present in the definition. + NamedStruct table_schema = 5; + + substrait.extensions.AdvancedExtension advanced_extension = 10; +} + // A relation with output field names. // // This is for use at the root of a `Rel` tree. @@ -581,6 +626,7 @@ message Rel { WriteRel write = 19; DdlRel ddl = 20; UpdateRel update = 22; + TableFunctionRel table_function = 23; // Physical relations HashJoinRel hash_join = 13; MergeJoinRel merge_join = 14; diff --git a/proto/substrait/function.proto b/proto/substrait/function.proto index 4377d1d71..befd1f698 100644 --- a/proto/substrait/function.proto +++ b/proto/substrait/function.proto @@ -106,6 +106,19 @@ message FunctionSignature { } } + message Table { + repeated Argument arguments = 2; + repeated string name = 3; + Description description = 4; + + bool deterministic = 7; + bool session_dependent = 8; + + NamedStruct schema = 9; + + repeated Implementation implementations = 10; + } + message Description { string language = 1; string body = 2; diff --git a/site/docs/expressions/table_functions.md b/site/docs/expressions/table_functions.md index b5ab3c5fa..cc86b6a55 100644 --- a/site/docs/expressions/table_functions.md +++ b/site/docs/expressions/table_functions.md @@ -1,8 +1,265 @@ # Table Functions -Table functions produce zero or more records for each input record. Table functions use a signature similar to scalar functions. However, they are not allowed in the same contexts. +!!! warning "Partial Implementation" + **Currently implemented:** 0-input table functions - leaf operators that take constant arguments and produce relations. + **Not yet implemented:** Transformation table functions that accept input relations. +## Definition -to be completed... +Table functions (0-input, currently supported) are **leaf operators** in the query tree that: +- Take a **fixed number of constant arguments** (literals or expressions that can be evaluated without input data) +- Produce **zero or more records** as output (a relation/table) +- Do **not consume an input relation** - they generate data from constants +- Have either a **derived schema** (determinable from function signature) or an **explicit schema** (depends on runtime data) + +Future extensions may add support for transformation table functions that consume and transform input relations by adding an optional input field to `TableFunctionRel`. + +## Function Signatures + +Table functions are defined in YAML extension files, similar to scalar, aggregate, and window functions. A table function signature specifies: + +- **Arguments**: The parameters the function accepts (must be constant expressions) +- **Schema**: The output schema of the generated relation +- **Determinism**: Whether the function produces the same output for the same inputs +- **Session Dependency**: Whether the function depends on session state + +## Schema Determination + +Like scalar functions' return types, table function schemas follow a clear pattern: + +!!! note "Required Constraint" + **If a table function's YAML definition includes an output schema, the `derived` field MUST be set to `true` in the plan, and the `table_schema` field MUST match the YAML definition (with any type parameters resolved based on the bound argument types).** + +**Derived schemas (`derived: true`)** - The schema can be **deterministically derived from the function signature**, including: +- **Static schemas**: Fixed output regardless of argument values (e.g., `generate_series` always produces `{value: i64}`) +- **Type-parameterized schemas**: Schema depends on argument types (e.g., `unnest(list)` produces `{element: T}`) + +Both cases use `derived: true` because the schema is fully determinable from the function signature and bound argument types. + +**Explicit schemas (`derived: false`)** - The schema **depends on runtime data content** and cannot be determined from the function signature alone. + +### Derived Schema Examples + +For functions where the schema is determinable from the function signature (either concrete or type-parameterized), set `derived: true`. The `table_schema` field contains the schema derived from the YAML definition: + +**Static schema example:** +``` +TableFunctionRel { + function_reference: + arguments: [ + { value: { literal: { i64: 1 } } }, + { value: { literal: { i64: 100 } } } + ] + derived: true // Schema came from YAML definition + table_schema: { + names: ["value"] + struct: { + types: [{ i64: {} }] + } + } +} +``` + +**Type-parameterized schema example:** +``` +TableFunctionRel { + function_reference: + arguments: [ + { value: { literal: { list: [...] } } } // list + ] + derived: true // Schema from YAML with T resolved to string + table_schema: { + names: ["element"] + struct: { + types: [{ string: {} }] // T resolved to string from list + } + } +} +``` + +### Explicit Schema Examples + +For functions where the schema depends on runtime data content, set `derived: false` and provide the schema in `table_schema`: + +``` +TableFunctionRel { + common: { ... } + function_reference: + arguments: [ + // Function arguments + ] + derived: false // Schema was determined by the plan producer + table_schema: { + names: ["id", "name", "age"] + struct: { + types: [ + { i32: {} }, + { string: {} }, + { i32: {} } + ] + } + } +} +``` + +## Usage in Plans + +Table functions are represented as their own relation type, `TableFunctionRel`. + +### TableFunctionRel Components + +- **function_reference**: Points to a function anchor referencing the table function definition +- **arguments**: Must be constant expressions (currently; literals or expressions evaluable without input data) +- **derived**: Boolean flag indicating schema source: + - `true` - Schema determinable from function signature (concrete types or type parameters). **Required when the YAML definition includes a schema.** + - `false` - Schema depends on runtime data content. **Only allowed when the YAML definition omits the schema field.** +- **table_schema**: The output schema (always present). Must match the YAML definition if derived is true (with type parameters resolved). Contains the actual schema whether derived from YAML or provided by the producer. +- **common**: Standard relation properties (emit, hints, etc.) + +**The key distinction:** Set `derived: true` if the schema can be determined by looking at the function signature and argument types in the YAML definition. Set `derived: false` only if the YAML definition omits the schema field because it requires inspecting runtime data content. + +Table functions can be used anywhere a relation is expected - as a leaf node, or as input to other relational operators like `FilterRel`, `ProjectRel`, etc. + +## Examples + +### Example 1: Generating a Sequence + +Generate integers from 1 to 100: + +``` +TableFunctionRel { + function_reference: + arguments: [ + { value: { literal: { i64: 1 } } }, + { value: { literal: { i64: 100 } } } + ] + derived: true + table_schema: { + names: ["value"] + struct: { + types: [{ i64: {} }] + } + } +} +``` + +**SQL equivalent:** `SELECT * FROM generate_series(1, 100)` + +**Output:** +``` +value +----- +1 +2 +3 +... +100 +``` + +### Example 2: Unnest a Literal Array + +Unnest a literal list into rows: + +``` +TableFunctionRel { + function_reference: + arguments: [ + { value: { literal: { + list: { + values: [ + { string: { value: "apple" } }, + { string: { value: "banana" } }, + { string: { value: "cherry" } } + ] + } + } } } // Type is list + ] + derived: true // Schema from YAML with T resolved to string + table_schema: { + names: ["element"] + struct: { + types: [{ string: {} }] + } + } +} +``` + +**SQL equivalent:** `SELECT * FROM UNNEST(['apple', 'banana', 'cherry'])` + +**Output:** +``` +element +-------- +apple +banana +cherry +``` + +!!! note "Limitation: Correlated Table Functions" + **The more sophisticated use case - unnesting a column from an existing table - cannot currently be represented.** For example, the SQL query `SELECT element FROM my_table, UNNEST(my_table.array_column)` would require applying the table function once per row of the input table. + + This requires **lateral joins** (correlated subqueries where a table function references columns from an outer relation), which are not yet specified in Substrait. Since TableFunctionRel is currently a leaf operator with no input relation, you cannot use field references in the function arguments. + + Future extensions will add support for transformation table functions and/or lateral join semantics to handle these cases. + + +### Example 3: Composing with Other Operators + +Table functions can be composed with other relational operators. For example, filtering the generated series to get only even numbers: + +``` +FilterRel { + input: { + TableFunctionRel { + function_reference: + arguments: [ + { value: { literal: { i64: 1 } } }, + { value: { literal: { i64: 100 } } } + ] + derived: true + table_schema: { + names: ["value"] + struct: { + types: [{ i64: {} }] + } + } + } + } + condition: { + scalar_function: { + function_reference: + arguments: [ + { + value: { + scalar_function: { + function_reference: + arguments: [ + { value: { selection: { direct_reference: { struct_field: { field: 0 } } } } }, + { value: { literal: { i64: 2 } } } + ] + } + } + }, + { value: { literal: { i64: 0 } } } + ] + } + } +} +``` + +## Future Extensions + +The current specification focuses on 0-input (generator/leaf) table functions. Future versions may support: + +- **Transformation table functions**: Functions that take an input relation and transform it (by adding an optional `input` field to `TableFunctionRel`) +- **Set-returning functions**: Functions that process input records and produce multiple output records per input +- **Lateral joins**: Applying table functions to each row of an input relation + + +=== "TableFunctionRel Message" + + ```proto +%%% proto.algebra.TableFunctionRel %%% + ``` diff --git a/site/docs/relations/logical_relations.md b/site/docs/relations/logical_relations.md index c2317a0ca..7045c55dd 100644 --- a/site/docs/relations/logical_relations.md +++ b/site/docs/relations/logical_relations.md @@ -97,7 +97,7 @@ possible approach is that a chunk should only be read if the midpoint of the chu #### Iceberg Table Type -A Iceberg Table is a table built on [Apache Iceberg](https://iceberg.apache.org/). Iceberg tables can be read by either directly reading a [metadata file](https://iceberg.apache.org/spec/#table-metadata) or by consulting a [catalog](https://iceberg.apache.org/concepts/catalog/). +A Iceberg Table is a table built on [Apache Iceberg](https://iceberg.apache.org/). Iceberg tables can be read by either directly reading a [metadata file](https://iceberg.apache.org/spec/#table-metadata) or by consulting a [catalog](https://iceberg.apache.org/concepts/catalog/). ##### Metadata File Reading @@ -110,6 +110,62 @@ Points to an [Iceberg metadata file](https://iceberg.apache.org/spec/#table-meta | snapshot_timestamp | The snapshot that should be read using timestamp. If not provided, the current snapshot is read. | Optional | +## Table Function + +The table function operator invokes a function that produces a relation (zero or more records). These are leaf operators that take constant (non-relational) arguments and generate data. + +Like scalar function return types, table function schemas can be concrete types or reference type parameters from arguments. The schema is either derived from the function signature or must be explicitly provided when it depends on runtime data content. It is preferred to explicitly provide a schema derivation in the YAML file when possible. + +| Signature | Value | +| -------------------- | ------------------------------------------- | +| Inputs | 0 (leaf operator) | +| Outputs | 1 | + +### Table Function Properties + +| Property | Description | Required | +| ------------------ | ------------------------------------------------------------ | -------- | +| Function Reference | Points to a function_anchor defined in the plan, referencing a table function in the extension YAML files | Required | +| Arguments | Constant expressions to pass as arguments to the function. Must match the function signature exactly. Must be literals or expressions that can be evaluated without input data. | Required | +| Derived | Boolean flag indicating schema source:
• `true` - Schema determinable from function signature (concrete or type-parameterized). **Must be true if YAML defines a schema.**
• `false` - Schema depends on runtime data content. **Only allowed if YAML omits schema.** | Required | +| Table Schema | The output schema (NamedStruct). Always present. **Must match YAML definition (with type parameters resolved) when derived is true.** | Required | + +### Use Cases + +Common examples of table functions include: +- `generate_series`: Generate sequences of numbers +- `unnest`: Expand arrays or lists into rows + +### Example + +Generate a series of integers from 1 to 100: + +``` +TableFunctionRel { + function_reference: + arguments: [ + { value: { literal: { i64: 1 } } }, + { value: { literal: { i64: 100 } } } + ] + derived: true + table_schema: { + names: ["value"] + struct: { + types: [{ i64: {} }] + } + } +} +``` + +See [Table Functions](../expressions/table_functions.md) for detailed documentation. + +=== "TableFunctionRel Message" + + ```proto +%%% proto.algebra.TableFunctionRel %%% + ``` + + ## Filter Operation The filter operator eliminates one or more records from the input data based on a boolean filter expression. diff --git a/site/docs/spec/specification.md b/site/docs/spec/specification.md index d19d10451..2fd18f2b6 100644 --- a/site/docs/spec/specification.md +++ b/site/docs/spec/specification.md @@ -29,11 +29,17 @@ The specification has passed the initial design phase and is now in the final st | [Binary Serialization](../serialization/binary_serialization.md) | A high performance & compact binary representation of the plan specification. | +## Components (Partially Implemented) + +| Section | Description | +| ------------------------------------------------------------ | ------------------------------------------------------------ | +| [Table Functions](../expressions/table_functions.md) | **Partial implementation:** Functions that produce relations (0..N records). Table functions can accept 0 or more input relations. **Currently, only 0-input functions are implemented** - these are leaf operators that take constant arguments and generate data. Examples include sequence generation (`generate_series`) and expanding collections (`unnest`). Transformation table functions that accept input relations are not yet implemented. | + + ## Components (Designed but not Implemented) | Section | Description | | ------------------------------------------------------------ | ------------------------------------------------------------ | -| [Table Functions](../expressions/table_functions.md) | Functions that convert one or more values from an input record into 0..N output records. Example include operations such as explode, pos-explode, etc. | | [User Defined Relations](../relations/user_defined_relations.md) | Installed and reusable relational operations customized to a particular platform. | | [Embedded Relations](../relations/embedded_relations.md) | Relational operations where plans contain the "machine code" to directly execute the necessary operations. | | [Physical Relations](../relations/physical_relations.md) | Specific execution sub-variations of common relational operations that describe have multiple unique physical variants associated with a single logical operation. Examples include hash join, merge join, nested loop join, etc. | diff --git a/text/simple_extensions_schema.yaml b/text/simple_extensions_schema.yaml index f6bef9862..9d9bf4e33 100644 --- a/text/simple_extensions_schema.yaml +++ b/text/simple_extensions_schema.yaml @@ -64,6 +64,10 @@ properties: type: array items: $ref: "#/$defs/windowFunction" + table_functions: + type: array + items: + $ref: "#/$defs/tableFunction" $defs: type: @@ -307,3 +311,33 @@ $defs: window_type: type: string enum: [STREAMING, PARTITION] + + tableFunction: + type: object + additionalProperties: false + required: [name, impls] + properties: + name: + type: string + description: + type: string + impls: + type: array + minItems: 1 + items: + type: object + additionalProperties: false + properties: + args: + $ref: "#/$defs/arguments" + sessionDependent: + $ref: "#/$defs/sessionDependent" + deterministic: + $ref: "#/$defs/deterministic" + schema: + # The output schema. + # Omit this field only if the schema depends on runtime data content + # (not determinable from function signature). + $ref: "#/$defs/type" + implementation: + $ref: "#/$defs/implementation" From 092ba9d97d335f180583764391c4829d7878a9fe Mon Sep 17 00:00:00 2001 From: Ben Bellick Date: Sat, 25 Oct 2025 16:46:08 -0400 Subject: [PATCH 02/10] fix: table functions output as return instead of schema This is for consistency with other functions --- extensions/functions_table.yaml | 10 +++++----- site/docs/expressions/table_functions.md | 8 ++++---- site/docs/relations/logical_relations.md | 2 +- text/simple_extensions_schema.yaml | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/extensions/functions_table.yaml b/extensions/functions_table.yaml index a1d97e363..67a2451a5 100644 --- a/extensions/functions_table.yaml +++ b/extensions/functions_table.yaml @@ -28,7 +28,7 @@ table_functions: constant: true deterministic: true sessionDependent: false - schema: + return: names: - value struct: @@ -47,7 +47,7 @@ table_functions: constant: true deterministic: true sessionDependent: false - schema: + return: names: - value struct: @@ -62,7 +62,7 @@ table_functions: description: The ending value of the series (inclusive) deterministic: true sessionDependent: false - schema: + return: names: - value struct: @@ -77,7 +77,7 @@ table_functions: description: The ending value of the series (inclusive) deterministic: true sessionDependent: false - schema: + return: names: - value struct: @@ -94,7 +94,7 @@ table_functions: sessionDependent: false # Schema references type parameter T from list # The field type is derived from the list element type - schema: + return: names: - element struct: diff --git a/site/docs/expressions/table_functions.md b/site/docs/expressions/table_functions.md index cc86b6a55..174dc2401 100644 --- a/site/docs/expressions/table_functions.md +++ b/site/docs/expressions/table_functions.md @@ -30,7 +30,7 @@ Table functions are defined in YAML extension files, similar to scalar, aggregat Like scalar functions' return types, table function schemas follow a clear pattern: !!! note "Required Constraint" - **If a table function's YAML definition includes an output schema, the `derived` field MUST be set to `true` in the plan, and the `table_schema` field MUST match the YAML definition (with any type parameters resolved based on the bound argument types).** + **If a table function's YAML definition includes a `return` field, the `derived` field MUST be set to `true` in the plan, and the `table_schema` field MUST match the YAML definition (with any type parameters resolved based on the bound argument types).** **Derived schemas (`derived: true`)** - The schema can be **deterministically derived from the function signature**, including: - **Static schemas**: Fixed output regardless of argument values (e.g., `generate_series` always produces `{value: i64}`) @@ -113,12 +113,12 @@ Table functions are represented as their own relation type, `TableFunctionRel`. - **function_reference**: Points to a function anchor referencing the table function definition - **arguments**: Must be constant expressions (currently; literals or expressions evaluable without input data) - **derived**: Boolean flag indicating schema source: - - `true` - Schema determinable from function signature (concrete types or type parameters). **Required when the YAML definition includes a schema.** - - `false` - Schema depends on runtime data content. **Only allowed when the YAML definition omits the schema field.** + - `true` - Schema determinable from function signature (concrete types or type parameters). **Required when the YAML definition includes a `return` field.** + - `false` - Schema depends on runtime data content. **Only allowed when the YAML definition omits the `return` field.** - **table_schema**: The output schema (always present). Must match the YAML definition if derived is true (with type parameters resolved). Contains the actual schema whether derived from YAML or provided by the producer. - **common**: Standard relation properties (emit, hints, etc.) -**The key distinction:** Set `derived: true` if the schema can be determined by looking at the function signature and argument types in the YAML definition. Set `derived: false` only if the YAML definition omits the schema field because it requires inspecting runtime data content. +**The key distinction:** Set `derived: true` if the schema can be determined by looking at the function signature and argument types in the YAML definition. Set `derived: false` only if the YAML definition omits the `return` field because it requires inspecting runtime data content. Table functions can be used anywhere a relation is expected - as a leaf node, or as input to other relational operators like `FilterRel`, `ProjectRel`, etc. diff --git a/site/docs/relations/logical_relations.md b/site/docs/relations/logical_relations.md index 7045c55dd..b3144417b 100644 --- a/site/docs/relations/logical_relations.md +++ b/site/docs/relations/logical_relations.md @@ -127,7 +127,7 @@ Like scalar function return types, table function schemas can be concrete types | ------------------ | ------------------------------------------------------------ | -------- | | Function Reference | Points to a function_anchor defined in the plan, referencing a table function in the extension YAML files | Required | | Arguments | Constant expressions to pass as arguments to the function. Must match the function signature exactly. Must be literals or expressions that can be evaluated without input data. | Required | -| Derived | Boolean flag indicating schema source:
• `true` - Schema determinable from function signature (concrete or type-parameterized). **Must be true if YAML defines a schema.**
• `false` - Schema depends on runtime data content. **Only allowed if YAML omits schema.** | Required | +| Derived | Boolean flag indicating schema source:
• `true` - Schema determinable from function signature (concrete or type-parameterized). **Must be true if YAML defines a `return` field.**
• `false` - Schema depends on runtime data content. **Only allowed if YAML omits `return` field.** | Required | | Table Schema | The output schema (NamedStruct). Always present. **Must match YAML definition (with type parameters resolved) when derived is true.** | Required | ### Use Cases diff --git a/text/simple_extensions_schema.yaml b/text/simple_extensions_schema.yaml index 9d9bf4e33..79aefa4a6 100644 --- a/text/simple_extensions_schema.yaml +++ b/text/simple_extensions_schema.yaml @@ -334,8 +334,8 @@ $defs: $ref: "#/$defs/sessionDependent" deterministic: $ref: "#/$defs/deterministic" - schema: - # The output schema. + return: + # The output schema (NamedStruct). # Omit this field only if the schema depends on runtime data content # (not determinable from function signature). $ref: "#/$defs/type" From 26fec6b862f41d3e154594ec2311d0635a729cbb Mon Sep 17 00:00:00 2001 From: Ben Bellick Date: Sat, 25 Oct 2025 16:46:40 -0400 Subject: [PATCH 03/10] wip: fix doc generation for absent return type (table functions) --- .../docs/extensions/generate_function_docs.py | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/site/docs/extensions/generate_function_docs.py b/site/docs/extensions/generate_function_docs.py index 5bb2171d7..348e2f2c5 100644 --- a/site/docs/extensions/generate_function_docs.py +++ b/site/docs/extensions/generate_function_docs.py @@ -125,19 +125,29 @@ def write_markdown(file_obj: dict, file_name: str) -> None: # If the return value for the function implementation is multiple lines long, # print each line separately. This is the case for some functions in # functions_arithmetic_decimal.yaml - if "\n" in impl["return"]: - mdFile.new_line( - f"{count}. {function_name}({func_concat_arg_input_values}): -> " - ) - multiline_return_str = "\t" + impl["return"] - multiline_return_str = multiline_return_str.replace("\n", "\n\t") - mdFile.new_line("\t```") - mdFile.new_line(f"{multiline_return_str}") - mdFile.new_line("\t```") + # Table functions may omit the return field if schema depends on runtime data + if "return" in impl: + if "\n" in impl["return"]: + mdFile.new_line( + f"{count}. {function_name}({func_concat_arg_input_values}): -> " + ) + multiline_return_str = "\t" + impl["return"] + multiline_return_str = multiline_return_str.replace( + "\n", "\n\t" + ) + mdFile.new_line("\t```") + mdFile.new_line(f"{multiline_return_str}") + mdFile.new_line("\t```") + else: + mdFile.new_line( + f"{count}. {function_name}({func_concat_arg_input_values}): -> " + f"`{impl['return']}`" + ) else: + # Return type not specified (e.g., table functions with runtime-dependent schemas) mdFile.new_line( f"{count}. {function_name}({func_concat_arg_input_values}): -> " - f"`{impl['return']}`" + f"`schema determined at runtime`" ) if "description" in function_spec: From b090c4ee1b6cada41198c8b438acd394c136c88d Mon Sep 17 00:00:00 2001 From: Ben Bellick Date: Tue, 28 Oct 2025 15:46:27 -0400 Subject: [PATCH 04/10] docs: small improvements to docs for table functions --- site/docs/expressions/table_functions.md | 114 +++++++++++++++++------ 1 file changed, 84 insertions(+), 30 deletions(-) diff --git a/site/docs/expressions/table_functions.md b/site/docs/expressions/table_functions.md index 174dc2401..d4510dcc5 100644 --- a/site/docs/expressions/table_functions.md +++ b/site/docs/expressions/table_functions.md @@ -12,7 +12,9 @@ Table functions (0-input, currently supported) are **leaf operators** in the que - Take a **fixed number of constant arguments** (literals or expressions that can be evaluated without input data) - Produce **zero or more records** as output (a relation/table) - Do **not consume an input relation** - they generate data from constants -- Have either a **derived schema** (determinable from function signature) or an **explicit schema** (depends on runtime data) +- Have either a **derived schema** (determinable from the YAML `return` field and argument types) or an **explicit schema** (determined at runtime when YAML omits the `return` field) + +See [Schema Determination](#schema-determination) for details on how schemas are specified. Future extensions may add support for transformation table functions that consume and transform input relations by adding an optional input field to `TableFunctionRel`. @@ -21,76 +23,126 @@ Future extensions may add support for transformation table functions that consum Table functions are defined in YAML extension files, similar to scalar, aggregate, and window functions. A table function signature specifies: - **Arguments**: The parameters the function accepts (must be constant expressions) -- **Schema**: The output schema of the generated relation +- **Schema**: The output schema of the generated relation (may or may not be specified in YAML) - **Determinism**: Whether the function produces the same output for the same inputs - **Session Dependency**: Whether the function depends on session state ## Schema Determination -Like scalar functions' return types, table function schemas follow a clear pattern: +Table function schemas can be specified in two ways, depending on whether the YAML definition includes a `return` field: + +### Derived Schemas (`derived: true`) + +When a table function's YAML definition **includes a `return` field**, the schema can be deterministically derived from the function signature and the types of the bound arguments. + +- In the plan: Set `derived: true` and include the schema from YAML in `table_schema` with any type parameters resolved +- The schema is fully determinable from type information alone + +This includes both: +- **Concrete types**: Schema is fixed (e.g., `generate_series` always produces `{value: i64}`) +- **Type-parameterized**: Schema depends on argument types (e.g., `unnest(list)` produces `{element: T}` where `T` is resolved from the argument) + +**Example YAML definitions** (from `functions_table.yaml`): + +```yaml +# Concrete type example - schema is always {value: i64} +- name: "generate_series" + impls: + - args: + - name: start + value: i64 + - name: stop + value: i64 + - name: step + value: i64 + return: + names: + - value + struct: + types: + - i64 + +# Type-parameterized example - schema is {element: T} where T comes from list +- name: "unnest" + impls: + - args: + - name: input + value: "list" + return: + names: + - element + struct: + types: + - T +``` + +### Explicit Schemas (`derived: false`) + +When a table function's YAML definition **omits the `return` field**, the schema depends on runtime data content and cannot be determined from type information alone. + +- In the plan: Set `derived: false` and provide the schema in `table_schema` +- The plan producer determines the schema (e.g., by inspecting file contents, database metadata, etc.) + +**Example scenario**: A function like `read_parquet(path)` where the schema depends on the actual Parquet file's structure. !!! note "Required Constraint" **If a table function's YAML definition includes a `return` field, the `derived` field MUST be set to `true` in the plan, and the `table_schema` field MUST match the YAML definition (with any type parameters resolved based on the bound argument types).** -**Derived schemas (`derived: true`)** - The schema can be **deterministically derived from the function signature**, including: -- **Static schemas**: Fixed output regardless of argument values (e.g., `generate_series` always produces `{value: i64}`) -- **Type-parameterized schemas**: Schema depends on argument types (e.g., `unnest(list)` produces `{element: T}`) - -Both cases use `derived: true` because the schema is fully determinable from the function signature and bound argument types. +### Plan Examples -**Explicit schemas (`derived: false`)** - The schema **depends on runtime data content** and cannot be determined from the function signature alone. +Now let's see how these two cases appear in actual Substrait plans: -### Derived Schema Examples +#### Derived Schema Examples -For functions where the schema is determinable from the function signature (either concrete or type-parameterized), set `derived: true`. The `table_schema` field contains the schema derived from the YAML definition: +For functions where the YAML includes a `return` field, set `derived: true`. The schema is derived from the YAML definition, with any type parameters resolved based on argument types. -**Static schema example:** +**Concrete type example** (`generate_series`): ``` TableFunctionRel { function_reference: arguments: [ { value: { literal: { i64: 1 } } }, - { value: { literal: { i64: 100 } } } + { value: { literal: { i64: 100 } } }, + { value: { literal: { i64: 1 } } } ] - derived: true // Schema came from YAML definition + derived: true // Schema from YAML return field table_schema: { names: ["value"] struct: { - types: [{ i64: {} }] + types: [{ i64: {} }] // Matches YAML definition exactly } } } ``` -**Type-parameterized schema example:** +**Type-parameterized example** (`unnest`): ``` TableFunctionRel { function_reference: arguments: [ { value: { literal: { list: [...] } } } // list ] - derived: true // Schema from YAML with T resolved to string + derived: true // Schema from YAML with T resolved table_schema: { names: ["element"] struct: { - types: [{ string: {} }] // T resolved to string from list + types: [{ string: {} }] // T resolved to string from list argument } } } ``` -### Explicit Schema Examples +#### Explicit Schema Example -For functions where the schema depends on runtime data content, set `derived: false` and provide the schema in `table_schema`: +For functions where the YAML omits the `return` field, set `derived: false` and provide the schema determined by the plan producer: ``` TableFunctionRel { - common: { ... } - function_reference: + function_reference: arguments: [ - // Function arguments + { value: { literal: { string: "data.parquet" } } } ] - derived: false // Schema was determined by the plan producer + derived: false // No return field in YAML - schema from runtime inspection table_schema: { names: ["id", "name", "age"] struct: { @@ -113,18 +165,20 @@ Table functions are represented as their own relation type, `TableFunctionRel`. - **function_reference**: Points to a function anchor referencing the table function definition - **arguments**: Must be constant expressions (currently; literals or expressions evaluable without input data) - **derived**: Boolean flag indicating schema source: - - `true` - Schema determinable from function signature (concrete types or type parameters). **Required when the YAML definition includes a `return` field.** - - `false` - Schema depends on runtime data content. **Only allowed when the YAML definition omits the `return` field.** -- **table_schema**: The output schema (always present). Must match the YAML definition if derived is true (with type parameters resolved). Contains the actual schema whether derived from YAML or provided by the producer. + - `true` - Schema is determinable from the YAML `return` field and argument types (includes both concrete and type-parameterized schemas) + - `false` - Schema depends on runtime data content (no `return` field in YAML) +- **table_schema**: The output schema (always present). For `derived: true`, must match the YAML `return` field (with type parameters resolved). For `derived: false`, provided by the plan producer. - **common**: Standard relation properties (emit, hints, etc.) -**The key distinction:** Set `derived: true` if the schema can be determined by looking at the function signature and argument types in the YAML definition. Set `derived: false` only if the YAML definition omits the `return` field because it requires inspecting runtime data content. +**Quick reference for setting `derived`:** +- YAML has `return` field → `derived: true` +- YAML omits `return` field → `derived: false` Table functions can be used anywhere a relation is expected - as a leaf node, or as input to other relational operators like `FilterRel`, `ProjectRel`, etc. ## Examples -### Example 1: Generating a Sequence +### Example 1: Generating a Sequence (Derived Schema - Concrete Types) Generate integers from 1 to 100: @@ -158,7 +212,7 @@ value 100 ``` -### Example 2: Unnest a Literal Array +### Example 2: Unnest a Literal Array (Derived Schema - Type-Parameterized) Unnest a literal list into rows: From a4ab6e906bf3974d2f0c6d25bc0618457a76d1d5 Mon Sep 17 00:00:00 2001 From: Ben Bellick Date: Mon, 10 Nov 2025 18:13:44 -0500 Subject: [PATCH 05/10] fix: fix some of @yongchul's comments --- extensions/functions_table.yaml | 4 +-- proto/substrait/algebra.proto | 38 +++++++++++++++--------- proto/substrait/function.proto | 7 ++++- site/docs/expressions/table_functions.md | 2 +- site/docs/relations/logical_relations.md | 2 +- text/simple_extensions_schema.yaml | 4 ++- 6 files changed, 37 insertions(+), 20 deletions(-) diff --git a/extensions/functions_table.yaml b/extensions/functions_table.yaml index 67a2451a5..e06865c6b 100644 --- a/extensions/functions_table.yaml +++ b/extensions/functions_table.yaml @@ -84,11 +84,11 @@ table_functions: types: - i32 - name: "unnest" - description: Expands a list literal into a set of rows, one row per element. + description: Expands a list expression into a set of rows, one row per element. impls: - args: - name: input - value: "list" + value: list description: The list to unnest deterministic: true sessionDependent: false diff --git a/proto/substrait/algebra.proto b/proto/substrait/algebra.proto index 4b57a084c..b6543fe9e 100644 --- a/proto/substrait/algebra.proto +++ b/proto/substrait/algebra.proto @@ -555,9 +555,10 @@ message ExpandRel { // Invokes a table-valued function that produces a relation (zero or more records). // // -// Table functions produce a table with either: -// - A schema that can be derived based on argument types (type-parameterized functions) -// - A schema that depends on runtime data (use derived: false) +// Table functions produce a table with a schema that can be: +// - Statically defined (concrete types in YAML) +// - Type-parameterized (derived from argument types, e.g., unnest(list) -> {element: T}) +// - Runtime-dependent (determined by inspecting data content, use derived: false) // // Future extensions may add an optional input field to support transformation // table functions that operate on input relations. @@ -572,7 +573,6 @@ message TableFunctionRel { // The arguments to be bound to the function. This must have exactly the // number of arguments specified in the function definition from the YAML file, // and the argument types must also match exactly: - // // - Value arguments must be bound using FunctionArgument.value. // Currently (0-input functions only), expressions must be constants // (literals or expressions evaluable without input data). @@ -581,20 +581,30 @@ message TableFunctionRel { // string that case-insensitively matches one of the allowed options. repeated FunctionArgument arguments = 3; - // The derived fields indicates whether or not the YAML file produced the schema: - // - If true, the table_schema was produced purely from the type expressions in the - // YAML file + the types of the provided arguments - // - If false, the table_schema was produced by the plan producer + // Options to specify behavior for corner cases, or leave behavior + // unspecified if the consumer does not need specific behavior in these + // cases. + repeated FunctionOption options = 5; + + // Indicates whether the schema was derived from the YAML function definition: + // - If true: the table_schema was computed from the ExpressionNamedStruct in the + // YAML file by evaluating it with the bound argument types. This includes: + // * Static schemas (concrete field types) + // * Type-parameterized schemas (e.g., T from list) + // - If false: the table_schema was determined by the plan producer based on runtime + // data inspection (YAML omits the return field) // - // This value is required to be true if and only if a schema is provided in the YAML + // This value must be true if and only if a return field is provided in the YAML // definition of this function. - bool derived = 4; + bool derived = 6; - // The schema of the output relation. This schema is required to match the implied schema - // by the YAML definition, if a schema is present in the definition. - NamedStruct table_schema = 5; + // The concrete output schema of the relation. For derived schemas (derived=true), + // this must match the schema computed by evaluating the YAML ExpressionNamedStruct + // with the bound argument types. For runtime-dependent schemas (derived=false), + // this is provided by the plan producer. + NamedStruct table_schema = 7; - substrait.extensions.AdvancedExtension advanced_extension = 10; + substrait.extensions.AdvancedExtension advanced_extension = 8; } // A relation with output field names. diff --git a/proto/substrait/function.proto b/proto/substrait/function.proto index befd1f698..c977c4ea8 100644 --- a/proto/substrait/function.proto +++ b/proto/substrait/function.proto @@ -114,7 +114,12 @@ message FunctionSignature { bool deterministic = 7; bool session_dependent = 8; - NamedStruct schema = 9; + // The schema expression that defines the output relation structure. + // This must be an ExpressionNamedStruct since table functions always + // produce relations with named fields. The struct can be: + // - Static (concrete field types) + // - Type-parameterized (field types derived from argument types) + DerivationExpression.ExpressionNamedStruct schema = 9; repeated Implementation implementations = 10; } diff --git a/site/docs/expressions/table_functions.md b/site/docs/expressions/table_functions.md index d4510dcc5..349c8ffb7 100644 --- a/site/docs/expressions/table_functions.md +++ b/site/docs/expressions/table_functions.md @@ -23,7 +23,7 @@ Future extensions may add support for transformation table functions that consum Table functions are defined in YAML extension files, similar to scalar, aggregate, and window functions. A table function signature specifies: - **Arguments**: The parameters the function accepts (must be constant expressions) -- **Schema**: The output schema of the generated relation (may or may not be specified in YAML) +- **Schema**: The output schema of the generated relation, expressed as an `ExpressionNamedStruct` that can be static or type-parameterized (may or may not be specified in the YAML definition) - **Determinism**: Whether the function produces the same output for the same inputs - **Session Dependency**: Whether the function depends on session state diff --git a/site/docs/relations/logical_relations.md b/site/docs/relations/logical_relations.md index b3144417b..9bd970720 100644 --- a/site/docs/relations/logical_relations.md +++ b/site/docs/relations/logical_relations.md @@ -114,7 +114,7 @@ Points to an [Iceberg metadata file](https://iceberg.apache.org/spec/#table-meta The table function operator invokes a function that produces a relation (zero or more records). These are leaf operators that take constant (non-relational) arguments and generate data. -Like scalar function return types, table function schemas can be concrete types or reference type parameters from arguments. The schema is either derived from the function signature or must be explicitly provided when it depends on runtime data content. It is preferred to explicitly provide a schema derivation in the YAML file when possible. +Like scalar function return types, table function schemas can be concrete types or reference type parameters from arguments. The schema is expressed as an `ExpressionNamedStruct` and is either derived from the function signature or must be explicitly provided when it depends on runtime data content. It is preferred to explicitly provide a schema derivation in the YAML file when possible. | Signature | Value | | -------------------- | ------------------------------------------- | diff --git a/text/simple_extensions_schema.yaml b/text/simple_extensions_schema.yaml index 79aefa4a6..63e522539 100644 --- a/text/simple_extensions_schema.yaml +++ b/text/simple_extensions_schema.yaml @@ -335,7 +335,9 @@ $defs: deterministic: $ref: "#/$defs/deterministic" return: - # The output schema (NamedStruct). + # The output schema (DerivationExpression, typically ExpressionNamedStruct). + # This can be a static struct definition or a dynamic derivation based on + # argument types (e.g., type-parameterized functions). # Omit this field only if the schema depends on runtime data content # (not determinable from function signature). $ref: "#/$defs/type" From 15f6897d7284e8e1ef6117629ffc2b909dc422cb Mon Sep 17 00:00:00 2001 From: Ben Bellick Date: Tue, 11 Nov 2025 10:12:43 -0500 Subject: [PATCH 06/10] tweak: drop session dependence and deterministic fields from rel --- extensions/functions_table.yaml | 10 ---------- proto/substrait/function.proto | 7 ++----- site/docs/expressions/table_functions.md | 2 -- text/simple_extensions_schema.yaml | 4 ---- 4 files changed, 2 insertions(+), 21 deletions(-) diff --git a/extensions/functions_table.yaml b/extensions/functions_table.yaml index e06865c6b..2c54b38ad 100644 --- a/extensions/functions_table.yaml +++ b/extensions/functions_table.yaml @@ -26,8 +26,6 @@ table_functions: value: i64 description: The increment between values constant: true - deterministic: true - sessionDependent: false return: names: - value @@ -45,8 +43,6 @@ table_functions: value: i32 description: The increment between values constant: true - deterministic: true - sessionDependent: false return: names: - value @@ -60,8 +56,6 @@ table_functions: - name: stop value: i64 description: The ending value of the series (inclusive) - deterministic: true - sessionDependent: false return: names: - value @@ -75,8 +69,6 @@ table_functions: - name: stop value: i32 description: The ending value of the series (inclusive) - deterministic: true - sessionDependent: false return: names: - value @@ -90,8 +82,6 @@ table_functions: - name: input value: list description: The list to unnest - deterministic: true - sessionDependent: false # Schema references type parameter T from list # The field type is derived from the list element type return: diff --git a/proto/substrait/function.proto b/proto/substrait/function.proto index c977c4ea8..57528eb14 100644 --- a/proto/substrait/function.proto +++ b/proto/substrait/function.proto @@ -111,17 +111,14 @@ message FunctionSignature { repeated string name = 3; Description description = 4; - bool deterministic = 7; - bool session_dependent = 8; - // The schema expression that defines the output relation structure. // This must be an ExpressionNamedStruct since table functions always // produce relations with named fields. The struct can be: // - Static (concrete field types) // - Type-parameterized (field types derived from argument types) - DerivationExpression.ExpressionNamedStruct schema = 9; + DerivationExpression.ExpressionNamedStruct schema = 5; - repeated Implementation implementations = 10; + repeated Implementation implementations = 6; } message Description { diff --git a/site/docs/expressions/table_functions.md b/site/docs/expressions/table_functions.md index 349c8ffb7..0cf26cc25 100644 --- a/site/docs/expressions/table_functions.md +++ b/site/docs/expressions/table_functions.md @@ -24,8 +24,6 @@ Table functions are defined in YAML extension files, similar to scalar, aggregat - **Arguments**: The parameters the function accepts (must be constant expressions) - **Schema**: The output schema of the generated relation, expressed as an `ExpressionNamedStruct` that can be static or type-parameterized (may or may not be specified in the YAML definition) -- **Determinism**: Whether the function produces the same output for the same inputs -- **Session Dependency**: Whether the function depends on session state ## Schema Determination diff --git a/text/simple_extensions_schema.yaml b/text/simple_extensions_schema.yaml index 63e522539..fda1ba523 100644 --- a/text/simple_extensions_schema.yaml +++ b/text/simple_extensions_schema.yaml @@ -330,10 +330,6 @@ $defs: properties: args: $ref: "#/$defs/arguments" - sessionDependent: - $ref: "#/$defs/sessionDependent" - deterministic: - $ref: "#/$defs/deterministic" return: # The output schema (DerivationExpression, typically ExpressionNamedStruct). # This can be a static struct definition or a dynamic derivation based on From 96860da06690d4b0ade5aec47bbe5db4acddb177 Mon Sep 17 00:00:00 2001 From: Ben Bellick Date: Sat, 15 Nov 2025 13:33:54 -0500 Subject: [PATCH 07/10] fix: address @yongchul's comments - drop restriction to constant args in table functions - drop mentions of "runtime" dependency - drop derived field from proto message for table functions --- extensions/functions_table.yaml | 14 +++---- proto/substrait/algebra.proto | 26 +++--------- site/docs/expressions/table_functions.md | 53 +++++++++--------------- 3 files changed, 30 insertions(+), 63 deletions(-) diff --git a/extensions/functions_table.yaml b/extensions/functions_table.yaml index 2c54b38ad..0034fdf56 100644 --- a/extensions/functions_table.yaml +++ b/extensions/functions_table.yaml @@ -1,19 +1,17 @@ %YAML 1.2 --- # Table functions: Functions that produce relations (zero or more records). -# Currently, only 0-input functions are supported - these take constant arguments -# and generate data as leaf operators. urn: extension:io.substrait:functions_table table_functions: - name: "generate_series" description: >- Generates a series of integer values from start to stop, incrementing by step. - Takes constant arguments and produces zero or more records containing a single - integer value. The series includes both the start and stop values if they fall - on a step boundary. If step is positive, stops when the value exceeds stop. - If step is negative, stops when the value is less than stop. Returns empty if - step is zero or if the step direction doesn't allow reaching stop from start. + Produces zero or more records containing a single integer value. The series + includes both the start and stop values if they fall on a step boundary. + If step is positive, stops when the value exceeds stop. If step is negative, + stops when the value is less than stop. Returns empty if step is zero or if + the step direction doesn't allow reaching stop from start. impls: - args: - name: start @@ -25,7 +23,6 @@ table_functions: - name: step value: i64 description: The increment between values - constant: true return: names: - value @@ -42,7 +39,6 @@ table_functions: - name: step value: i32 description: The increment between values - constant: true return: names: - value diff --git a/proto/substrait/algebra.proto b/proto/substrait/algebra.proto index b6543fe9e..e07b68651 100644 --- a/proto/substrait/algebra.proto +++ b/proto/substrait/algebra.proto @@ -558,7 +558,6 @@ message ExpandRel { // Table functions produce a table with a schema that can be: // - Statically defined (concrete types in YAML) // - Type-parameterized (derived from argument types, e.g., unnest(list) -> {element: T}) -// - Runtime-dependent (determined by inspecting data content, use derived: false) // // Future extensions may add an optional input field to support transformation // table functions that operate on input relations. @@ -574,8 +573,6 @@ message TableFunctionRel { // number of arguments specified in the function definition from the YAML file, // and the argument types must also match exactly: // - Value arguments must be bound using FunctionArgument.value. - // Currently (0-input functions only), expressions must be constants - // (literals or expressions evaluable without input data). // - Type arguments must be bound using FunctionArgument.type. // - Enum arguments must be bound using FunctionArgument.enum with a // string that case-insensitively matches one of the allowed options. @@ -586,25 +583,12 @@ message TableFunctionRel { // cases. repeated FunctionOption options = 5; - // Indicates whether the schema was derived from the YAML function definition: - // - If true: the table_schema was computed from the ExpressionNamedStruct in the - // YAML file by evaluating it with the bound argument types. This includes: - // * Static schemas (concrete field types) - // * Type-parameterized schemas (e.g., T from list) - // - If false: the table_schema was determined by the plan producer based on runtime - // data inspection (YAML omits the return field) - // - // This value must be true if and only if a return field is provided in the YAML - // definition of this function. - bool derived = 6; - - // The concrete output schema of the relation. For derived schemas (derived=true), - // this must match the schema computed by evaluating the YAML ExpressionNamedStruct - // with the bound argument types. For runtime-dependent schemas (derived=false), - // this is provided by the plan producer. - NamedStruct table_schema = 7; + // The concrete output schema of the relation. For schemas which can be derived + // from their YAML specification this must match the schema computed by evaluating the + // YAML ExpressionNamedStruct with the bound argument types. + NamedStruct table_schema = 6; - substrait.extensions.AdvancedExtension advanced_extension = 8; + substrait.extensions.AdvancedExtension advanced_extension = 7; } // A relation with output field names. diff --git a/site/docs/expressions/table_functions.md b/site/docs/expressions/table_functions.md index 0cf26cc25..b68fe6c00 100644 --- a/site/docs/expressions/table_functions.md +++ b/site/docs/expressions/table_functions.md @@ -1,7 +1,7 @@ # Table Functions !!! warning "Partial Implementation" - **Currently implemented:** 0-input table functions - leaf operators that take constant arguments and produce relations. + **Currently implemented:** 0-input table functions - leaf operators that take arguments and produce relations. **Not yet implemented:** Transformation table functions that accept input relations. @@ -9,10 +9,10 @@ Table functions (0-input, currently supported) are **leaf operators** in the query tree that: -- Take a **fixed number of constant arguments** (literals or expressions that can be evaluated without input data) +- Take a **fixed number of arguments** - Produce **zero or more records** as output (a relation/table) -- Do **not consume an input relation** - they generate data from constants -- Have either a **derived schema** (determinable from the YAML `return` field and argument types) or an **explicit schema** (determined at runtime when YAML omits the `return` field) +- Do **not consume an input relation as an argument** +- Have a schema that is either **determinable from the YAML definition** (when `return` field is present) or **determined by the plan producer** (when YAML omits the `return` field) See [Schema Determination](#schema-determination) for details on how schemas are specified. @@ -22,18 +22,18 @@ Future extensions may add support for transformation table functions that consum Table functions are defined in YAML extension files, similar to scalar, aggregate, and window functions. A table function signature specifies: -- **Arguments**: The parameters the function accepts (must be constant expressions) +- **Arguments**: The parameters the function accepts - **Schema**: The output schema of the generated relation, expressed as an `ExpressionNamedStruct` that can be static or type-parameterized (may or may not be specified in the YAML definition) ## Schema Determination Table function schemas can be specified in two ways, depending on whether the YAML definition includes a `return` field: -### Derived Schemas (`derived: true`) +### Schemas Derivable from YAML When a table function's YAML definition **includes a `return` field**, the schema can be deterministically derived from the function signature and the types of the bound arguments. -- In the plan: Set `derived: true` and include the schema from YAML in `table_schema` with any type parameters resolved +- In the plan: Include the schema from YAML in `table_schema` with any type parameters resolved - The schema is fully determinable from type information alone This includes both: @@ -74,25 +74,25 @@ This includes both: - T ``` -### Explicit Schemas (`derived: false`) +### Schemas Determined by Plan Producer -When a table function's YAML definition **omits the `return` field**, the schema depends on runtime data content and cannot be determined from type information alone. +When a table function's YAML definition **omits the `return` field**, the schema cannot be determined from type information alone. -- In the plan: Set `derived: false` and provide the schema in `table_schema` +- In the plan: Provide the schema in `table_schema` - The plan producer determines the schema (e.g., by inspecting file contents, database metadata, etc.) **Example scenario**: A function like `read_parquet(path)` where the schema depends on the actual Parquet file's structure. !!! note "Required Constraint" - **If a table function's YAML definition includes a `return` field, the `derived` field MUST be set to `true` in the plan, and the `table_schema` field MUST match the YAML definition (with any type parameters resolved based on the bound argument types).** + **If a table function's YAML definition includes a `return` field, the `table_schema` field in the plan MUST match the YAML definition (with any type parameters resolved based on the bound argument types).** ### Plan Examples Now let's see how these two cases appear in actual Substrait plans: -#### Derived Schema Examples +#### Schema Derivable from YAML -For functions where the YAML includes a `return` field, set `derived: true`. The schema is derived from the YAML definition, with any type parameters resolved based on argument types. +For functions where the YAML includes a `return` field, the schema is derived from the YAML definition, with any type parameters resolved based on argument types. **Concrete type example** (`generate_series`): ``` @@ -103,7 +103,6 @@ TableFunctionRel { { value: { literal: { i64: 100 } } }, { value: { literal: { i64: 1 } } } ] - derived: true // Schema from YAML return field table_schema: { names: ["value"] struct: { @@ -120,7 +119,6 @@ TableFunctionRel { arguments: [ { value: { literal: { list: [...] } } } // list ] - derived: true // Schema from YAML with T resolved table_schema: { names: ["element"] struct: { @@ -130,9 +128,9 @@ TableFunctionRel { } ``` -#### Explicit Schema Example +#### Schema Determined by Plan Producer -For functions where the YAML omits the `return` field, set `derived: false` and provide the schema determined by the plan producer: +For functions where the YAML omits the `return` field, provide the schema determined by the plan producer: ``` TableFunctionRel { @@ -140,7 +138,6 @@ TableFunctionRel { arguments: [ { value: { literal: { string: "data.parquet" } } } ] - derived: false // No return field in YAML - schema from runtime inspection table_schema: { names: ["id", "name", "age"] struct: { @@ -161,22 +158,15 @@ Table functions are represented as their own relation type, `TableFunctionRel`. ### TableFunctionRel Components - **function_reference**: Points to a function anchor referencing the table function definition -- **arguments**: Must be constant expressions (currently; literals or expressions evaluable without input data) -- **derived**: Boolean flag indicating schema source: - - `true` - Schema is determinable from the YAML `return` field and argument types (includes both concrete and type-parameterized schemas) - - `false` - Schema depends on runtime data content (no `return` field in YAML) -- **table_schema**: The output schema (always present). For `derived: true`, must match the YAML `return` field (with type parameters resolved). For `derived: false`, provided by the plan producer. +- **arguments**: Function arguments bound to the signature (value, type, or enum arguments) +- **table_schema**: The concrete output schema (always present). When the YAML includes a `return` field, this must match the YAML definition with type parameters resolved. When the YAML omits the `return` field, this is determined by the plan producer. - **common**: Standard relation properties (emit, hints, etc.) -**Quick reference for setting `derived`:** -- YAML has `return` field → `derived: true` -- YAML omits `return` field → `derived: false` - Table functions can be used anywhere a relation is expected - as a leaf node, or as input to other relational operators like `FilterRel`, `ProjectRel`, etc. ## Examples -### Example 1: Generating a Sequence (Derived Schema - Concrete Types) +### Example 1: Generating a Sequence Generate integers from 1 to 100: @@ -187,7 +177,6 @@ TableFunctionRel { { value: { literal: { i64: 1 } } }, { value: { literal: { i64: 100 } } } ] - derived: true table_schema: { names: ["value"] struct: { @@ -210,7 +199,7 @@ value 100 ``` -### Example 2: Unnest a Literal Array (Derived Schema - Type-Parameterized) +### Example 2: Unnest a Literal Array Unnest a literal list into rows: @@ -228,11 +217,10 @@ TableFunctionRel { } } } } // Type is list ] - derived: true // Schema from YAML with T resolved to string table_schema: { names: ["element"] struct: { - types: [{ string: {} }] + types: [{ string: {} }] // T resolved to string from list argument } } } @@ -270,7 +258,6 @@ FilterRel { { value: { literal: { i64: 1 } } }, { value: { literal: { i64: 100 } } } ] - derived: true table_schema: { names: ["value"] struct: { From b652b87b6285066e58f6d921ab79363de2805e61 Mon Sep 17 00:00:00 2001 From: Ben Bellick Date: Sat, 15 Nov 2025 13:45:44 -0500 Subject: [PATCH 08/10] docs: consolidate redundant examples --- site/docs/expressions/table_functions.md | 67 +----------------------- 1 file changed, 1 insertion(+), 66 deletions(-) diff --git a/site/docs/expressions/table_functions.md b/site/docs/expressions/table_functions.md index b68fe6c00..f4f5737aa 100644 --- a/site/docs/expressions/table_functions.md +++ b/site/docs/expressions/table_functions.md @@ -81,76 +81,11 @@ When a table function's YAML definition **omits the `return` field**, the schema - In the plan: Provide the schema in `table_schema` - The plan producer determines the schema (e.g., by inspecting file contents, database metadata, etc.) -**Example scenario**: A function like `read_parquet(path)` where the schema depends on the actual Parquet file's structure. +**Example:** A function like `read_parquet(path)` where the schema depends on the actual Parquet file's structure - the YAML would omit the `return` field, and the plan producer would provide the concrete schema in `table_schema`. !!! note "Required Constraint" **If a table function's YAML definition includes a `return` field, the `table_schema` field in the plan MUST match the YAML definition (with any type parameters resolved based on the bound argument types).** -### Plan Examples - -Now let's see how these two cases appear in actual Substrait plans: - -#### Schema Derivable from YAML - -For functions where the YAML includes a `return` field, the schema is derived from the YAML definition, with any type parameters resolved based on argument types. - -**Concrete type example** (`generate_series`): -``` -TableFunctionRel { - function_reference: - arguments: [ - { value: { literal: { i64: 1 } } }, - { value: { literal: { i64: 100 } } }, - { value: { literal: { i64: 1 } } } - ] - table_schema: { - names: ["value"] - struct: { - types: [{ i64: {} }] // Matches YAML definition exactly - } - } -} -``` - -**Type-parameterized example** (`unnest`): -``` -TableFunctionRel { - function_reference: - arguments: [ - { value: { literal: { list: [...] } } } // list - ] - table_schema: { - names: ["element"] - struct: { - types: [{ string: {} }] // T resolved to string from list argument - } - } -} -``` - -#### Schema Determined by Plan Producer - -For functions where the YAML omits the `return` field, provide the schema determined by the plan producer: - -``` -TableFunctionRel { - function_reference: - arguments: [ - { value: { literal: { string: "data.parquet" } } } - ] - table_schema: { - names: ["id", "name", "age"] - struct: { - types: [ - { i32: {} }, - { string: {} }, - { i32: {} } - ] - } - } -} -``` - ## Usage in Plans Table functions are represented as their own relation type, `TableFunctionRel`. From 2168c5bc6d241e8af435d706c634808107ad378c Mon Sep 17 00:00:00 2001 From: Ben Bellick Date: Thu, 20 Nov 2025 16:21:29 -0500 Subject: [PATCH 09/10] feat: add variadic handling to table functions --- proto/substrait/function.proto | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/proto/substrait/function.proto b/proto/substrait/function.proto index 57528eb14..11c4c72df 100644 --- a/proto/substrait/function.proto +++ b/proto/substrait/function.proto @@ -118,7 +118,12 @@ message FunctionSignature { // - Type-parameterized (field types derived from argument types) DerivationExpression.ExpressionNamedStruct schema = 5; - repeated Implementation implementations = 6; + oneof final_variable_behavior { + FinalArgVariadic variadic = 6; + FinalArgNormal normal = 7; + } + + repeated Implementation implementations = 8; } message Description { From 4a4596f2af05af2e894494bd41aee3388a40aa8d Mon Sep 17 00:00:00 2001 From: Ben Bellick Date: Thu, 20 Nov 2025 16:52:00 -0500 Subject: [PATCH 10/10] cleanup: make some adjustments - use any1 instead of T - add variadic-ness to unnest and to table functions in general --- extensions/functions_table.yaml | 17 ++- site/docs/expressions/table_functions.md | 143 +++++++++++++++-------- site/docs/relations/logical_relations.md | 8 +- text/simple_extensions_schema.yaml | 2 + 4 files changed, 112 insertions(+), 58 deletions(-) diff --git a/extensions/functions_table.yaml b/extensions/functions_table.yaml index 0034fdf56..508980391 100644 --- a/extensions/functions_table.yaml +++ b/extensions/functions_table.yaml @@ -72,17 +72,22 @@ table_functions: types: - i32 - name: "unnest" - description: Expands a list expression into a set of rows, one row per element. + description: >- + Expands one or more list expressions into a set of rows. + + When multiple lists are provided, produces rows by expanding the lists in parallel (like a zip operation). + For example, unnest([1,2], [3,4]) produces rows: (1,3), (2,4). + If lists have different lengths, shorter lists are padded with nulls. impls: - args: - name: input - value: list - description: The list to unnest - # Schema references type parameter T from list - # The field type is derived from the list element type + value: list + description: The list(s) to unnest + variadic: + min: 1 return: names: - element struct: types: - - T + - any1 diff --git a/site/docs/expressions/table_functions.md b/site/docs/expressions/table_functions.md index f4f5737aa..a320cb5f6 100644 --- a/site/docs/expressions/table_functions.md +++ b/site/docs/expressions/table_functions.md @@ -1,29 +1,26 @@ # Table Functions +Table functions are functions that produce a relation (zero or more records) as output. Table functions are similar to scalar and aggregate functions but produce relations instead of individual values. + !!! warning "Partial Implementation" - **Currently implemented:** 0-input table functions - leaf operators that take arguments and produce relations. + **Currently implemented:** 0-input table functions - leaf operators that take constant arguments and produce relations. **Not yet implemented:** Transformation table functions that accept input relations. -## Definition - -Table functions (0-input, currently supported) are **leaf operators** in the query tree that: - -- Take a **fixed number of arguments** -- Produce **zero or more records** as output (a relation/table) -- Do **not consume an input relation as an argument** -- Have a schema that is either **determinable from the YAML definition** (when `return` field is present) or **determined by the plan producer** (when YAML omits the `return` field) - -See [Schema Determination](#schema-determination) for details on how schemas are specified. +## Function Signature Properties -Future extensions may add support for transformation table functions that consume and transform input relations by adding an optional input field to `TableFunctionRel`. +Table function signatures contain the properties defined below: -## Function Signatures +| Property | Description | Required | +| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ----------------------------------- | +| Name | One or more user-friendly, case-sensitive UTF-8 strings that are used to reference this function. | At least one value is required. | +| List of arguments | Argument properties follow the same pattern as [scalar functions](scalar_functions.md#argument-types): value arguments, type arguments, and enumerations. Arguments can be fully defined or calculated with a type expression. | Optional, defaults to niladic. | +| Variadic Behavior | Whether the last argument of the function is variadic or a single argument. If variadic, the argument can optionally have a lower bound (minimum number of instances) and an upper bound (maximum number of instances). | Optional, defaults to single value. | +| Description | Additional description of function for implementers or users. Should be written human-readable to allow exposure to end users. Presented as a map with language => description mappings. | Optional | +| Return Schema | The output schema of the relation, expressed as an `ExpressionNamedStruct`. Can be concrete types or type-parameterized (derived from argument types). May be omitted if schema depends on runtime data content (not determinable from function signature). | Optional (see [Schema Determination](#schema-determination)) | +| Implementation Map | A map of implementation locations for one or more implementations of the given function. Each key is a function implementation type with properties associated with retrieval of that implementation. | Optional | -Table functions are defined in YAML extension files, similar to scalar, aggregate, and window functions. A table function signature specifies: - -- **Arguments**: The parameters the function accepts -- **Schema**: The output schema of the generated relation, expressed as an `ExpressionNamedStruct` that can be static or type-parameterized (may or may not be specified in the YAML definition) +Table functions use the same argument types as scalar functions (value arguments, type arguments, and enumerations). See [Scalar Functions - Argument Types](scalar_functions.md#argument-types) for details. ## Schema Determination @@ -31,19 +28,13 @@ Table function schemas can be specified in two ways, depending on whether the YA ### Schemas Derivable from YAML -When a table function's YAML definition **includes a `return` field**, the schema can be deterministically derived from the function signature and the types of the bound arguments. - -- In the plan: Include the schema from YAML in `table_schema` with any type parameters resolved -- The schema is fully determinable from type information alone - -This includes both: -- **Concrete types**: Schema is fixed (e.g., `generate_series` always produces `{value: i64}`) -- **Type-parameterized**: Schema depends on argument types (e.g., `unnest(list)` produces `{element: T}` where `T` is resolved from the argument) +When a table function's YAML definition includes a `return` field, the schema can be deterministically derived from the function signature and the types of the bound arguments. The plan must include this derived schema in the `table_schema` field with any type parameters resolved. **Example YAML definitions** (from `functions_table.yaml`): +Concrete type example - `generate_series` always produces `{value: i64}`: + ```yaml -# Concrete type example - schema is always {value: i64} - name: "generate_series" impls: - args: @@ -59,45 +50,64 @@ This includes both: struct: types: - i64 +``` + +Type-parameterized and variadic example - `unnest` can take 1 or more lists: -# Type-parameterized example - schema is {element: T} where T comes from list +```yaml - name: "unnest" impls: - args: - name: input - value: "list" + value: list + variadic: + min: 1 return: names: - element struct: types: - - T + - any1 ``` ### Schemas Determined by Plan Producer -When a table function's YAML definition **omits the `return` field**, the schema cannot be determined from type information alone. - -- In the plan: Provide the schema in `table_schema` -- The plan producer determines the schema (e.g., by inspecting file contents, database metadata, etc.) +When a table function's YAML definition omits the `return` field, the schema cannot be determined from type information alone. In these cases, the plan producer must provide the schema in `table_schema`. **Example:** A function like `read_parquet(path)` where the schema depends on the actual Parquet file's structure - the YAML would omit the `return` field, and the plan producer would provide the concrete schema in `table_schema`. !!! note "Required Constraint" **If a table function's YAML definition includes a `return` field, the `table_schema` field in the plan MUST match the YAML definition (with any type parameters resolved based on the bound argument types).** -## Usage in Plans +## Variadic Table Functions + +Table functions can be variadic, meaning the last parameter can be repeated one or more times. This is specified using the `variadic` field in the YAML definition with a `min` value indicating the minimum number of times the parameter must appear. + +**Example:** The `unnest` function is variadic with `min: 1`, allowing it to accept one or more list arguments: -Table functions are represented as their own relation type, `TableFunctionRel`. +```yaml +- name: "unnest" + impls: + - args: + - name: input + value: list + variadic: + min: 1 + return: + names: + - element + struct: + types: + - any1 +``` -### TableFunctionRel Components +When multiple arguments are provided to a variadic table function, the behavior depends on the function's semantics. For example, `unnest` expands multiple lists in parallel (like a zip operation). -- **function_reference**: Points to a function anchor referencing the table function definition -- **arguments**: Function arguments bound to the signature (value, type, or enum arguments) -- **table_schema**: The concrete output schema (always present). When the YAML includes a `return` field, this must match the YAML definition with type parameters resolved. When the YAML omits the `return` field, this is determined by the plan producer. -- **common**: Standard relation properties (emit, hints, etc.) +## Usage in Plans -Table functions can be used anywhere a relation is expected - as a leaf node, or as input to other relational operators like `FilterRel`, `ProjectRel`, etc. +Table functions are invoked in plans using the `TableFunctionRel` relation type. Table functions can be used anywhere a relation is expected - as a leaf node, or as input to other relational operators like `FilterRel`, `ProjectRel`, etc. + +For details on the `TableFunctionRel` message structure and properties, see the [Table Function](../relations/logical_relations.md#table-function) section in Logical Relations. ## Examples @@ -172,6 +182,50 @@ banana cherry ``` +### Example 2b: Variadic Unnest (Multiple Lists) + +Unnest multiple lists to produce their cross product: + +``` +TableFunctionRel { + function_reference: + arguments: [ + { value: { literal: { + list: { + values: [ + { i32: { value: 1 } }, + { i32: { value: 2 } } + ] + } + } } }, // First list: [1, 2] + { value: { literal: { + list: { + values: [ + { i32: { value: 3 } }, + { i32: { value: 4 } } + ] + } + } } } // Second list: [3, 4] + ] + table_schema: { + names: ["element"] // Note: With multiple lists, schema may have multiple fields + struct: { + types: [{ i32: {} }] + } + } +} +``` + +**SQL equivalent:** `SELECT * FROM UNNEST([1, 2], [3, 4])` + +**Output (parallel expansion / zip):** +``` +element +------- +1, 3 +2, 4 +``` + !!! note "Limitation: Correlated Table Functions" **The more sophisticated use case - unnesting a column from an existing table - cannot currently be represented.** For example, the SQL query `SELECT element FROM my_table, UNNEST(my_table.array_column)` would require applying the table function once per row of the input table. @@ -230,10 +284,3 @@ The current specification focuses on 0-input (generator/leaf) table functions. F - **Transformation table functions**: Functions that take an input relation and transform it (by adding an optional `input` field to `TableFunctionRel`) - **Set-returning functions**: Functions that process input records and produce multiple output records per input - **Lateral joins**: Applying table functions to each row of an input relation - - -=== "TableFunctionRel Message" - - ```proto -%%% proto.algebra.TableFunctionRel %%% - ``` diff --git a/site/docs/relations/logical_relations.md b/site/docs/relations/logical_relations.md index 9bd970720..b10cad679 100644 --- a/site/docs/relations/logical_relations.md +++ b/site/docs/relations/logical_relations.md @@ -116,6 +116,8 @@ The table function operator invokes a function that produces a relation (zero or Like scalar function return types, table function schemas can be concrete types or reference type parameters from arguments. The schema is expressed as an `ExpressionNamedStruct` and is either derived from the function signature or must be explicitly provided when it depends on runtime data content. It is preferred to explicitly provide a schema derivation in the YAML file when possible. +Table functions can be **variadic**, meaning the last parameter can be repeated one or more times (specified via the `variadic` field in YAML). + | Signature | Value | | -------------------- | ------------------------------------------- | | Inputs | 0 (leaf operator) | @@ -127,14 +129,13 @@ Like scalar function return types, table function schemas can be concrete types | ------------------ | ------------------------------------------------------------ | -------- | | Function Reference | Points to a function_anchor defined in the plan, referencing a table function in the extension YAML files | Required | | Arguments | Constant expressions to pass as arguments to the function. Must match the function signature exactly. Must be literals or expressions that can be evaluated without input data. | Required | -| Derived | Boolean flag indicating schema source:
• `true` - Schema determinable from function signature (concrete or type-parameterized). **Must be true if YAML defines a `return` field.**
• `false` - Schema depends on runtime data content. **Only allowed if YAML omits `return` field.** | Required | -| Table Schema | The output schema (NamedStruct). Always present. **Must match YAML definition (with type parameters resolved) when derived is true.** | Required | +| Table Schema | The output schema (NamedStruct). Always present. **Must match YAML definition (with type parameters resolved) when YAML defines a `return` field.** | Required | ### Use Cases Common examples of table functions include: - `generate_series`: Generate sequences of numbers -- `unnest`: Expand arrays or lists into rows +- `unnest`: Expand arrays or lists into rows (variadic - can accept multiple lists) ### Example @@ -147,7 +148,6 @@ TableFunctionRel { { value: { literal: { i64: 1 } } }, { value: { literal: { i64: 100 } } } ] - derived: true table_schema: { names: ["value"] struct: { diff --git a/text/simple_extensions_schema.yaml b/text/simple_extensions_schema.yaml index fda1ba523..3f8fb2db1 100644 --- a/text/simple_extensions_schema.yaml +++ b/text/simple_extensions_schema.yaml @@ -330,6 +330,8 @@ $defs: properties: args: $ref: "#/$defs/arguments" + variadic: + $ref: "#/$defs/variadicBehavior" return: # The output schema (DerivationExpression, typically ExpressionNamedStruct). # This can be a static struct definition or a dynamic derivation based on