diff --git a/extensions/extension_types.yaml b/extensions/extension_types.yaml index f05ad2c0d..0b1bc5183 100644 --- a/extensions/extension_types.yaml +++ b/extensions/extension_types.yaml @@ -1,5 +1,5 @@ --- -urn: extension:io.substrait:extension_types +urn: urn:substrait:extension:io.substrait:extension_types types: - name: point structure: diff --git a/extensions/functions_aggregate_approx.yaml b/extensions/functions_aggregate_approx.yaml index d858accf4..622324fe8 100644 --- a/extensions/functions_aggregate_approx.yaml +++ b/extensions/functions_aggregate_approx.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:functions_aggregate_approx +urn: urn:substrait:extension:io.substrait:functions_aggregate_approx aggregate_functions: - name: "approx_count_distinct" description: >- diff --git a/extensions/functions_aggregate_decimal_output.yaml b/extensions/functions_aggregate_decimal_output.yaml index 85a1959b9..89afa2c58 100644 --- a/extensions/functions_aggregate_decimal_output.yaml +++ b/extensions/functions_aggregate_decimal_output.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:functions_aggregate_decimal_output +urn: urn:substrait:extension:io.substrait:functions_aggregate_decimal_output aggregate_functions: - name: "count" description: Count a set of values. Result is returned as a decimal instead of i64. diff --git a/extensions/functions_aggregate_generic.yaml b/extensions/functions_aggregate_generic.yaml index 63553f391..d1ac73615 100644 --- a/extensions/functions_aggregate_generic.yaml +++ b/extensions/functions_aggregate_generic.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:functions_aggregate_generic +urn: urn:substrait:extension:io.substrait:functions_aggregate_generic aggregate_functions: - name: "count" description: Count a set of values diff --git a/extensions/functions_arithmetic.yaml b/extensions/functions_arithmetic.yaml index 8746f0392..57908ac1c 100644 --- a/extensions/functions_arithmetic.yaml +++ b/extensions/functions_arithmetic.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:functions_arithmetic +urn: urn:substrait:extension:io.substrait:functions_arithmetic scalar_functions: - name: "add" diff --git a/extensions/functions_arithmetic_decimal.yaml b/extensions/functions_arithmetic_decimal.yaml index bc131a266..16ffe685a 100644 --- a/extensions/functions_arithmetic_decimal.yaml +++ b/extensions/functions_arithmetic_decimal.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:functions_arithmetic_decimal +urn: urn:substrait:extension:io.substrait:functions_arithmetic_decimal scalar_functions: - name: "add" diff --git a/extensions/functions_boolean.yaml b/extensions/functions_boolean.yaml index fcbe56ca9..5b779f832 100644 --- a/extensions/functions_boolean.yaml +++ b/extensions/functions_boolean.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:functions_boolean +urn: urn:substrait:extension:io.substrait:functions_boolean scalar_functions: - name: or diff --git a/extensions/functions_comparison.yaml b/extensions/functions_comparison.yaml index a07fd29e6..060bd141c 100644 --- a/extensions/functions_comparison.yaml +++ b/extensions/functions_comparison.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:functions_comparison +urn: urn:substrait:extension:io.substrait:functions_comparison scalar_functions: - name: "not_equal" diff --git a/extensions/functions_datetime.yaml b/extensions/functions_datetime.yaml index 754454a9b..71371dc76 100644 --- a/extensions/functions_datetime.yaml +++ b/extensions/functions_datetime.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:functions_datetime +urn: urn:substrait:extension:io.substrait:functions_datetime scalar_functions: - name: extract diff --git a/extensions/functions_geometry.yaml b/extensions/functions_geometry.yaml index 127978c58..1d202221a 100644 --- a/extensions/functions_geometry.yaml +++ b/extensions/functions_geometry.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:functions_geometry +urn: urn:substrait:extension:io.substrait:functions_geometry types: - name: geometry structure: "BINARY" diff --git a/extensions/functions_list.yaml b/extensions/functions_list.yaml index f7776350f..7818f82b7 100644 --- a/extensions/functions_list.yaml +++ b/extensions/functions_list.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:functions_list +urn: urn:substrait:extension:io.substrait:functions_list scalar_functions: - name: "transform" description: >- diff --git a/extensions/functions_logarithmic.yaml b/extensions/functions_logarithmic.yaml index d1f8dd93b..b05f5a581 100644 --- a/extensions/functions_logarithmic.yaml +++ b/extensions/functions_logarithmic.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:functions_logarithmic +urn: urn:substrait:extension:io.substrait:functions_logarithmic scalar_functions: - name: "ln" diff --git a/extensions/functions_rounding.yaml b/extensions/functions_rounding.yaml index 0ca87133b..7220e2ae9 100644 --- a/extensions/functions_rounding.yaml +++ b/extensions/functions_rounding.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:functions_rounding +urn: urn:substrait:extension:io.substrait:functions_rounding scalar_functions: - name: "ceil" diff --git a/extensions/functions_rounding_decimal.yaml b/extensions/functions_rounding_decimal.yaml index 057dc6099..837239f92 100644 --- a/extensions/functions_rounding_decimal.yaml +++ b/extensions/functions_rounding_decimal.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:functions_rounding_decimal +urn: urn:substrait:extension:io.substrait:functions_rounding_decimal scalar_functions: - name: "ceil" diff --git a/extensions/functions_set.yaml b/extensions/functions_set.yaml index 3331b29f0..86cabbecb 100644 --- a/extensions/functions_set.yaml +++ b/extensions/functions_set.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:functions_set +urn: urn:substrait:extension:io.substrait:functions_set scalar_functions: - name: "index_in" diff --git a/extensions/functions_string.yaml b/extensions/functions_string.yaml index 2cd5f84da..2c44a7eea 100644 --- a/extensions/functions_string.yaml +++ b/extensions/functions_string.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:functions_string +urn: urn:substrait:extension:io.substrait:functions_string scalar_functions: - name: concat diff --git a/extensions/type_variations.yaml b/extensions/type_variations.yaml index 6f129b573..39585e08c 100644 --- a/extensions/type_variations.yaml +++ b/extensions/type_variations.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:type_variations +urn: urn:substrait:extension:io.substrait:type_variations type_variations: - parent: string name: dict4 diff --git a/extensions/unknown.yaml b/extensions/unknown.yaml index 3603d6871..b1c24fd62 100644 --- a/extensions/unknown.yaml +++ b/extensions/unknown.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:unknown +urn: urn:substrait:extension:io.substrait:unknown types: - name: unknown scalar_functions: diff --git a/proto/substrait/extensions/extensions.proto b/proto/substrait/extensions/extensions.proto index 105b58bdb..7f5d0b8ab 100644 --- a/proto/substrait/extensions/extensions.proto +++ b/proto/substrait/extensions/extensions.proto @@ -16,8 +16,11 @@ message SimpleExtensionURN { // 0 is a valid anchor/reference, but prefer non-zero values for ergonomics. uint32 extension_urn_anchor = 1; - // The extension URN that uniquely identifies this extension. This must follow the - // format extension:: and serves as the "namespace" of this extension. + // The extension URN that uniquely identifies this extension. The canonical format + // is urn:substrait:extension::. For backwards compatibility, URNs + // beginning with "extension:" have "urn:substrait:" prepended before validation. + // The canonical form must be a valid RFC 8141 URN and conform to the regex: + // ^urn:substrait:extension:[a-z0-9_.-]+:[a-z0-9_.-]+$ string urn = 2; } diff --git a/site/docs/extensions/index.md b/site/docs/extensions/index.md index 8129d0524..675e84801 100644 --- a/site/docs/extensions/index.md +++ b/site/docs/extensions/index.md @@ -13,13 +13,17 @@ Some kinds of primitives are so frequently extended that Substrait defines a sta * Window Functions * Table Functions -To extend these items, developers can create one or more YAML files that describe the properties of each of these extensions. Each YAML file must include a required `urn` field that uniquely identifies the extension. While these identifiers are URN-like but not technically URNs (they lack the `urn:` prefix), they will be referred to as `extension URNs` for clarity. +To extend these items, users can create one or more YAML files that describe the properties of each of these extensions. Each YAML file must include a required `urn` field that uniquely identifies the extension. -This extension URN uses the format `extension::`, where: +The canonical form of an extension URN is `urn:substrait:extension::`, where: -- `OWNER` represents the organization or entity providing the extension and should follow [reverse domain name convention](https://en.wikipedia.org/wiki/Reverse_domain_name_notation) (e.g., `io.substrait`, `com.example`, `org.apache.arrow`) to prevent name collisions +- `NAMESPACE` represents the organization or entity providing the extension and should follow [reverse domain name convention](https://en.wikipedia.org/wiki/Reverse_domain_name_notation) (e.g., `io.substrait`, `com.example`, `org.apache.arrow`) to prevent name collisions - `ID` is the specific identifier for the extension (e.g., `functions_arithmetic`, `custom_types`) +For backwards compatibility, if a URN begins with `extension:` instead of `urn:substrait:extension:`, the prefix `urn:substrait:` is prepended before validation. This means `extension:io.substrait:functions_list` is equivalent to `urn:substrait:extension:io.substrait:functions_list`. + +Extension URNs in canonical form must be valid [RFC 8141](https://www.rfc-editor.org/rfc/rfc8141.html) URNs (with NID `substrait`) and must match the regex `^urn:substrait:extension:[a-z0-9_.-]+:[a-z0-9_.-]+$`. + The YAML file is constructed according to the [YAML Schema](https://github.com/substrait-io/substrait/blob/main/text/simple_extensions_schema.yaml). Each definition in the file corresponds to the YAML-based serialization of the relevant data structure. If a user only wants to extend one of these types of objects (e.g. types), a developer does not have to provide definitions for the other extension points. A Substrait plan can reference one or more YAML files via their extension URN. In the places where these entities are referenced, they will be referenced using an extension URN + name reference. Each extension entity (type, type variation, or function) is assigned an anchor value, which is a non-negative integer starting from 0. The anchor value 0 is valid and can be used to reference extension entities, but prefer non-zero values for ergonomics. The name scheme per type works as follows: diff --git a/site/docs/serialization/binary_serialization.md b/site/docs/serialization/binary_serialization.md index 364410465..533b97d66 100644 --- a/site/docs/serialization/binary_serialization.md +++ b/site/docs/serialization/binary_serialization.md @@ -22,7 +22,7 @@ For simple extensions, a plan references the extension URNs associated with the Simple extensions within a plan are split into three components: an extension URN, an extension declaration and a number of references. -* **Extension URN**: A unique identifier for the extension following the format `extension::` that identifies a YAML document specifying one or more specific extensions. Declares an anchor that can be used in extension declarations. +* **Extension URN**: A unique identifier for the extension in the canonical format `urn:substrait:extension::` that identifies a YAML document specifying one or more specific extensions. Declares an anchor that can be used in extension declarations. For backwards compatibility, URNs beginning with `extension:` have `urn:substrait:` prepended before validation. See [Simple Extensions](../extensions/index.md#simple-extensions) for details. * **Extension Declaration**: A specific extension within a single YAML document. The declaration combines a reference to the associated extension URN along with a unique key identifying the specific item within that YAML document (see [Function Signature](../extensions/index.md#function-signature)). It also defines a declaration anchor. The anchor is a plan-specific unique value that the producer creates as a key to be referenced elsewhere. * **Extension Reference**: A specific instance or use of an extension declaration within the plan body. diff --git a/site/examples/extensions/any1_type_function.yaml b/site/examples/extensions/any1_type_function.yaml index 2263c2db5..edef731c6 100644 --- a/site/examples/extensions/any1_type_function.yaml +++ b/site/examples/extensions/any1_type_function.yaml @@ -1,5 +1,5 @@ # Example showing the 'any1' type - arguments must be of the same type -urn: extension:example:any1_type +urn: urn:substrait:extension:example:any1_type scalar_functions: - name: bar impls: diff --git a/site/examples/extensions/any_type_function.yaml b/site/examples/extensions/any_type_function.yaml index 1e0636462..0f5ffd517 100644 --- a/site/examples/extensions/any_type_function.yaml +++ b/site/examples/extensions/any_type_function.yaml @@ -1,5 +1,5 @@ # Example showing the 'any' type - arguments can be of any type -urn: extension:example:any_type +urn: urn:substrait:extension:example:any_type scalar_functions: - name: foo impls: diff --git a/site/examples/extensions/distance_functions.yaml b/site/examples/extensions/distance_functions.yaml index 214f1433c..bdee96ec3 100644 --- a/site/examples/extensions/distance_functions.yaml +++ b/site/examples/extensions/distance_functions.yaml @@ -1,4 +1,4 @@ -urn: extension:example:distance_functions +urn: urn:substrait:extension:example:distance_functions dependencies: ext: extension:io.substrait:extension_types scalar_functions: diff --git a/site/examples/extensions/double_function.yaml b/site/examples/extensions/double_function.yaml index f428fb1c7..f191be750 100644 --- a/site/examples/extensions/double_function.yaml +++ b/site/examples/extensions/double_function.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:example:double_function +urn: urn:substrait:extension:example:double_function scalar_functions: - name: "double" diff --git a/site/examples/extensions/lambda_function_example.yaml b/site/examples/extensions/lambda_function_example.yaml index cb28d0d23..2c24c4151 100644 --- a/site/examples/extensions/lambda_function_example.yaml +++ b/site/examples/extensions/lambda_function_example.yaml @@ -1,6 +1,6 @@ %YAML 1.2 --- -urn: extension:io.substrait:functions_list +urn: urn:substrait:extension:io.substrait:functions_list scalar_functions: - name: "transform" description: >- diff --git a/site/examples/extensions/metadata_example.yaml b/site/examples/extensions/metadata_example.yaml index a890def74..269b9f8fc 100644 --- a/site/examples/extensions/metadata_example.yaml +++ b/site/examples/extensions/metadata_example.yaml @@ -1,5 +1,5 @@ # Example showing the metadata field at multiple levels -urn: extension:io.substrait:metadata_examples +urn: urn:substrait:extension:io.substrait:metadata_examples metadata: version: 2.0 maintainer: example-team diff --git a/site/examples/types/point_with_datatype_param.yaml b/site/examples/types/point_with_datatype_param.yaml index d16d77319..737a5be23 100644 --- a/site/examples/types/point_with_datatype_param.yaml +++ b/site/examples/types/point_with_datatype_param.yaml @@ -1,5 +1,5 @@ # Compound user-defined type with a data type parameter -urn: extension:example:point_parameterized +urn: urn:substrait:extension:example:point_parameterized types: - name: point parameters: diff --git a/site/examples/types/point_with_enum_param.yaml b/site/examples/types/point_with_enum_param.yaml index f4e3b8fd0..ed8d3ca39 100644 --- a/site/examples/types/point_with_enum_param.yaml +++ b/site/examples/types/point_with_enum_param.yaml @@ -1,5 +1,5 @@ # Compound user-defined type with an enumeration parameter -urn: extension:example:point_enum_param +urn: urn:substrait:extension:example:point_enum_param types: - name: point parameters: diff --git a/site/examples/types/point_with_nstruct.yaml b/site/examples/types/point_with_nstruct.yaml index b320e0cab..eb0e04cdc 100644 --- a/site/examples/types/point_with_nstruct.yaml +++ b/site/examples/types/point_with_nstruct.yaml @@ -1,5 +1,5 @@ # Alternative way to define point structure using NSTRUCT syntax -urn: extension:example:point_nstruct +urn: urn:substrait:extension:example:point_nstruct types: - name: point structure: "NSTRUCT" diff --git a/site/examples/types/point_with_structure.yaml b/site/examples/types/point_with_structure.yaml index f1999e6f0..4b7b89c11 100644 --- a/site/examples/types/point_with_structure.yaml +++ b/site/examples/types/point_with_structure.yaml @@ -1,5 +1,5 @@ # User-defined point type with structure information -urn: extension:example:point_with_structure +urn: urn:substrait:extension:example:point_with_structure types: - name: point structure: diff --git a/site/examples/types/point_with_two_params.yaml b/site/examples/types/point_with_two_params.yaml index fd7585b32..36a0c65c0 100644 --- a/site/examples/types/point_with_two_params.yaml +++ b/site/examples/types/point_with_two_params.yaml @@ -1,5 +1,5 @@ # Compound user-defined type with two data type parameters -urn: extension:example:point_two_params +urn: urn:substrait:extension:example:point_two_params types: - name: point parameters: diff --git a/site/examples/types/tuple_optional_variadic.yaml b/site/examples/types/tuple_optional_variadic.yaml index f72565dd3..213c939fd 100644 --- a/site/examples/types/tuple_optional_variadic.yaml +++ b/site/examples/types/tuple_optional_variadic.yaml @@ -1,5 +1,5 @@ # Tuple type with optional variadic parameter (zero or more types) -urn: extension:example:tuple_variadic +urn: urn:substrait:extension:example:tuple_variadic types: - name: tuple parameters: diff --git a/site/examples/types/union_variadic.yaml b/site/examples/types/union_variadic.yaml index 4b39d23cd..bf8922403 100644 --- a/site/examples/types/union_variadic.yaml +++ b/site/examples/types/union_variadic.yaml @@ -1,5 +1,5 @@ # Union type with variadic parameter (one or more types) -urn: extension:example:union_variadic +urn: urn:substrait:extension:example:union_variadic types: - name: union parameters: diff --git a/site/examples/types/user_defined_point.yaml b/site/examples/types/user_defined_point.yaml index 311e99278..199dcb26e 100644 --- a/site/examples/types/user_defined_point.yaml +++ b/site/examples/types/user_defined_point.yaml @@ -1,5 +1,5 @@ # User-defined type example: a point type with two scalar functions -urn: extension:example:point_type +urn: urn:substrait:extension:example:point_type types: - name: "point" diff --git a/site/examples/types/vector_with_constraints.yaml b/site/examples/types/vector_with_constraints.yaml index 60f72c953..d962ad6f9 100644 --- a/site/examples/types/vector_with_constraints.yaml +++ b/site/examples/types/vector_with_constraints.yaml @@ -1,5 +1,5 @@ # Vector type with integer parameter constrained to 2 or 3 dimensions -urn: extension:example:vector_constrained +urn: urn:substrait:extension:example:vector_constrained types: - name: vector parameters: diff --git a/text/simple_extensions_schema.yaml b/text/simple_extensions_schema.yaml index fdcd990bd..9d8961044 100644 --- a/text/simple_extensions_schema.yaml +++ b/text/simple_extensions_schema.yaml @@ -7,6 +7,7 @@ required: [urn] properties: urn: type: string + pattern: "^urn:substrait:extension:[a-z0-9_.-]+:[a-z0-9_.-]+$" dependencies: # For reusing type classes and type variations from other extension files. # The keys are namespace identifiers that you can then use as dot-separated