diff --git a/extensions/unsigned_integers.yaml b/extensions/unsigned_integers.yaml new file mode 100644 index 000000000..2e03208d4 --- /dev/null +++ b/extensions/unsigned_integers.yaml @@ -0,0 +1,309 @@ +%YAML 1.2 +--- +urn: extension:io.substrait:unsigned_integers + +types: + - name: u8 + description: > + Unsigned 8-bit integer (0 to 255). + Values are encoded as decimal strings in the structure representation. + structure: + value: string + - name: u16 + description: > + Unsigned 16-bit integer (0 to 65535). + Values are encoded as decimal strings in the structure representation. + structure: + value: string + - name: u32 + description: > + Unsigned 32-bit integer (0 to 4294967295). + Values are encoded as decimal strings in the structure representation. + structure: + value: string + - name: u64 + description: > + Unsigned 64-bit integer (0 to 18446744073709551615). + Values are encoded as decimal strings in the structure representation. + structure: + value: string + +scalar_functions: + - + name: "add" + description: "Add two unsigned integer values." + impls: + - args: + - name: x + value: u!u8 + - name: y + value: u!u8 + options: + overflow: + values: [ SILENT, SATURATE, ERROR ] + return: u!u8 + - args: + - name: x + value: u!u16 + - name: y + value: u!u16 + options: + overflow: + values: [ SILENT, SATURATE, ERROR ] + return: u!u16 + - args: + - name: x + value: u!u32 + - name: y + value: u!u32 + options: + overflow: + values: [ SILENT, SATURATE, ERROR ] + return: u!u32 + - args: + - name: x + value: u!u64 + - name: y + value: u!u64 + options: + overflow: + values: [ SILENT, SATURATE, ERROR ] + return: u!u64 + - + name: "subtract" + description: "Subtract one unsigned integer value from another." + impls: + - args: + - name: x + value: u!u8 + - name: y + value: u!u8 + options: + overflow: + values: [ SILENT, SATURATE, ERROR ] + return: u!u8 + - args: + - name: x + value: u!u16 + - name: y + value: u!u16 + options: + overflow: + values: [ SILENT, SATURATE, ERROR ] + return: u!u16 + - args: + - name: x + value: u!u32 + - name: y + value: u!u32 + options: + overflow: + values: [ SILENT, SATURATE, ERROR ] + return: u!u32 + - args: + - name: x + value: u!u64 + - name: y + value: u!u64 + options: + overflow: + values: [ SILENT, SATURATE, ERROR ] + return: u!u64 + - + name: "multiply" + description: "Multiply two unsigned integer values." + impls: + - args: + - name: x + value: u!u8 + - name: y + value: u!u8 + options: + overflow: + values: [ SILENT, SATURATE, ERROR ] + return: u!u8 + - args: + - name: x + value: u!u16 + - name: y + value: u!u16 + options: + overflow: + values: [ SILENT, SATURATE, ERROR ] + return: u!u16 + - args: + - name: x + value: u!u32 + - name: y + value: u!u32 + options: + overflow: + values: [ SILENT, SATURATE, ERROR ] + return: u!u32 + - args: + - name: x + value: u!u64 + - name: y + value: u!u64 + options: + overflow: + values: [ SILENT, SATURATE, ERROR ] + return: u!u64 + - + name: "divide" + description: > + Divide x by y. Partial values are truncated (i.e. rounded towards 0). + The `on_division_by_zero` option governs behavior in cases where y is 0. + If either x or y are out of range, behavior will be governed by `on_domain_error`. + impls: + - args: + - name: x + value: u!u8 + - name: y + value: u!u8 + options: + on_domain_error: + values: [ "NULL", ERROR ] + on_division_by_zero: + values: [ "NULL", ERROR ] + return: u!u8 + - args: + - name: x + value: u!u16 + - name: y + value: u!u16 + options: + on_domain_error: + values: [ "NULL", ERROR ] + on_division_by_zero: + values: [ "NULL", ERROR ] + return: u!u16 + - args: + - name: x + value: u!u32 + - name: y + value: u!u32 + options: + on_domain_error: + values: [ "NULL", ERROR ] + on_division_by_zero: + values: [ "NULL", ERROR ] + return: u!u32 + - args: + - name: x + value: u!u64 + - name: y + value: u!u64 + options: + on_domain_error: + values: [ "NULL", ERROR ] + on_division_by_zero: + values: [ "NULL", ERROR ] + return: u!u64 + +aggregate_functions: + - name: "sum" + description: Sum a set of unsigned integer values. The sum of zero elements yields null. + impls: + - args: + - name: x + value: u!u8 + options: + overflow: + values: [ SILENT, SATURATE, ERROR ] + nullability: DECLARED_OUTPUT + decomposable: MANY + intermediate: u!u64? + return: u!u64? + - args: + - name: x + value: u!u16 + options: + overflow: + values: [ SILENT, SATURATE, ERROR ] + nullability: DECLARED_OUTPUT + decomposable: MANY + intermediate: u!u64? + return: u!u64? + - args: + - name: x + value: u!u32 + options: + overflow: + values: [ SILENT, SATURATE, ERROR ] + nullability: DECLARED_OUTPUT + decomposable: MANY + intermediate: u!u64? + return: u!u64? + - args: + - name: x + value: u!u64 + options: + overflow: + values: [ SILENT, SATURATE, ERROR ] + nullability: DECLARED_OUTPUT + decomposable: MANY + intermediate: u!u64? + return: u!u64? + - name: "min" + description: Min of a set of unsigned integer values. + impls: + - args: + - name: x + value: u!u8 + nullability: DECLARED_OUTPUT + decomposable: MANY + intermediate: u!u8? + return: u!u8? + - args: + - name: x + value: u!u16 + nullability: DECLARED_OUTPUT + decomposable: MANY + intermediate: u!u16? + return: u!u16? + - args: + - name: x + value: u!u32 + nullability: DECLARED_OUTPUT + decomposable: MANY + intermediate: u!u32? + return: u!u32? + - args: + - name: x + value: u!u64 + nullability: DECLARED_OUTPUT + decomposable: MANY + intermediate: u!u64? + return: u!u64? + - name: "max" + description: Max of a set of unsigned integer values. + impls: + - args: + - name: x + value: u!u8 + nullability: DECLARED_OUTPUT + decomposable: MANY + intermediate: u!u8? + return: u!u8? + - args: + - name: x + value: u!u16 + nullability: DECLARED_OUTPUT + decomposable: MANY + intermediate: u!u16? + return: u!u16? + - args: + - name: x + value: u!u32 + nullability: DECLARED_OUTPUT + decomposable: MANY + intermediate: u!u32? + return: u!u32? + - args: + - name: x + value: u!u64 + nullability: DECLARED_OUTPUT + decomposable: MANY + intermediate: u!u64? + return: u!u64? diff --git a/site/docs/extensions/generate_function_docs.py b/site/docs/extensions/generate_function_docs.py index 3a4425aa3..d68410411 100644 --- a/site/docs/extensions/generate_function_docs.py +++ b/site/docs/extensions/generate_function_docs.py @@ -21,7 +21,7 @@ def write_markdown(file_obj: dict, file_name: str) -> None: mdFile.new_line(f"{key}: {value}") for function_classification, _value in file_obj.items(): - if function_classification == "urn": + if function_classification in ("urn", "dependencies"): continue function_classification_str = function_classification.replace("_", " ").title() mdFile.new_header(level=2, title=f"{function_classification_str}") diff --git a/tests/baseline.json b/tests/baseline.json index 07b82ff00..8cccd8fc0 100644 --- a/tests/baseline.json +++ b/tests/baseline.json @@ -1,16 +1,16 @@ { "registry": { - "extension_count": 15, - "dependency_count": 15, + "extension_count": 16, + "dependency_count": 18, "function_count": 174, - "num_aggregate_functions": 28, - "num_scalar_functions": 170, + "num_aggregate_functions": 31, + "num_scalar_functions": 174, "num_window_functions": 11, - "num_function_overloads": 498 + "num_function_overloads":530 }, "coverage": { - "total_test_count": 1177, - "num_function_variants": 498, - "num_covered_function_variants": 245 + "total_test_count": 1297, + "num_function_variants": 530, + "num_covered_function_variants": 277 } } diff --git a/tests/cases/arithmetic_unsigned/add.test b/tests/cases/arithmetic_unsigned/add.test new file mode 100644 index 000000000..56bddde77 --- /dev/null +++ b/tests/cases/arithmetic_unsigned/add.test @@ -0,0 +1,16 @@ +### SUBSTRAIT_SCALAR_TEST: v1.0 +### SUBSTRAIT_INCLUDE: extension:io.substrait:unsigned_integers + +# basic: Basic unsigned integer examples +add(('200')::u!u8, ('50')::u!u8) = ('250')::u!u8 +add(('50000')::u!u16, ('10000')::u!u16) = ('60000')::u!u16 +add(('3000000000')::u!u32, ('1000000000')::u!u32) = ('4000000000')::u!u32 +add(('10000000000000000000')::u!u64, ('1000000000000000000')::u!u64) = ('11000000000000000000')::u!u64 + +# overflow: Examples demonstrating overflow behavior +add(('200')::u!u8, ('100')::u!u8) [overflow:ERROR] = +add(('60000')::u!u16, ('10000')::u!u16) [overflow:ERROR] = +add(('4000000000')::u!u32, ('1000000000')::u!u32) [overflow:ERROR] = +add(('18446744073709551615')::u!u64, ('1')::u!u64) [overflow:ERROR] = +add(('200')::u!u8, ('100')::u!u8) [overflow:SATURATE] = ('255')::u!u8 +add(('200')::u!u8, ('100')::u!u8) [overflow:SILENT] = diff --git a/tests/cases/arithmetic_unsigned/divide.test b/tests/cases/arithmetic_unsigned/divide.test new file mode 100644 index 000000000..37c11b528 --- /dev/null +++ b/tests/cases/arithmetic_unsigned/divide.test @@ -0,0 +1,12 @@ +### SUBSTRAIT_SCALAR_TEST: v1.0 +### SUBSTRAIT_INCLUDE: extension:io.substrait:unsigned_integers + +# basic: Basic unsigned integer examples +divide(('250')::u!u8, ('5')::u!u8) = ('50')::u!u8 +divide(('60000')::u!u16, ('100')::u!u16) = ('600')::u!u16 +divide(('4000000000')::u!u32, ('200')::u!u32) = ('20000000')::u!u32 +divide(('10000000000000000000')::u!u64, ('5000')::u!u64) = ('2000000000000000')::u!u64 + +# division_by_zero: Examples demonstrating division by zero +divide(('5')::u!u8, ('0')::u!u8) [on_division_by_zero:NULL] = null::u!u8? +divide(('5')::u!u8, ('0')::u!u8) [on_division_by_zero:ERROR] = diff --git a/tests/cases/arithmetic_unsigned/max.test b/tests/cases/arithmetic_unsigned/max.test new file mode 100644 index 000000000..a0cfea748 --- /dev/null +++ b/tests/cases/arithmetic_unsigned/max.test @@ -0,0 +1,13 @@ +### SUBSTRAIT_AGGREGATE_TEST: v1.0 +### SUBSTRAIT_INCLUDE: extension:io.substrait:unsigned_integers + +# basic: Basic unsigned integer examples +max(('20', '3', '1', '10', '0', '5')::u!u8) = ('20')::u!u8? +max(('32768', '32767', '20000', '30000')::u!u16) = ('32768')::u!u16? +max(('214748648', '214748647', '21470048', '4000000')::u!u32) = ('214748648')::u!u32? +max(('2000000000', '3217908979', '629000000', '100000000', '0', '987654321')::u!u64) = ('3217908979')::u!u64? + +# null_handling: Examples with null as input or output +max((Null, Null, Null)::u!u16) = Null::u!u16? +max(()::u!u16) = Null::u!u16? +max(('200', Null, '50', '100', Null, '5')::u!u64) = ('200')::u!u64? diff --git a/tests/cases/arithmetic_unsigned/min.test b/tests/cases/arithmetic_unsigned/min.test new file mode 100644 index 000000000..9cb4a1bf8 --- /dev/null +++ b/tests/cases/arithmetic_unsigned/min.test @@ -0,0 +1,13 @@ +### SUBSTRAIT_AGGREGATE_TEST: v1.0 +### SUBSTRAIT_INCLUDE: extension:io.substrait:unsigned_integers + +# basic: Basic unsigned integer examples +min(('20', '3', '1', '10', '0', '5')::u!u8) = ('0')::u!u8? +min(('32768', '32767', '20000', '30000')::u!u16) = ('20000')::u!u16? +min(('214748648', '214748647', '21470048', '4000000')::u!u32) = ('4000000')::u!u32? +min(('2000000000', '3217908979', '629000000', '100000000', '0', '987654321')::u!u64) = ('0')::u!u64? + +# null_handling: Examples with null as input or output +min((Null, Null, Null)::u!u16) = Null::u!u16? +min(()::u!u16) = Null::u!u16? +min(('200', Null, '50', '100', Null, '5')::u!u64) = ('5')::u!u64? diff --git a/tests/cases/arithmetic_unsigned/multiply.test b/tests/cases/arithmetic_unsigned/multiply.test new file mode 100644 index 000000000..b1813294c --- /dev/null +++ b/tests/cases/arithmetic_unsigned/multiply.test @@ -0,0 +1,16 @@ +### SUBSTRAIT_SCALAR_TEST: v1.0 +### SUBSTRAIT_INCLUDE: extension:io.substrait:unsigned_integers + +# basic: Basic unsigned integer examples +multiply(('25')::u!u8, ('5')::u!u8) = ('125')::u!u8 +multiply(('200')::u!u16, ('100')::u!u16) = ('20000')::u!u16 +multiply(('300')::u!u32, ('200')::u!u32) = ('60000')::u!u32 +multiply(('80000')::u!u64, ('5000')::u!u64) = ('400000000')::u!u64 + +# overflow: Examples demonstrating overflow behavior +multiply(('200')::u!u8, ('2')::u!u8) [overflow:ERROR] = +multiply(('40000')::u!u16, ('2')::u!u16) [overflow:ERROR] = +multiply(('3000000000')::u!u32, ('2')::u!u32) [overflow:ERROR] = +multiply(('10000000000000000000')::u!u64, ('2')::u!u64) [overflow:ERROR] = +multiply(('200')::u!u8, ('2')::u!u8) [overflow:SATURATE] = ('255')::u!u8 +multiply(('200')::u!u8, ('2')::u!u8) [overflow:SILENT] = diff --git a/tests/cases/arithmetic_unsigned/subtract.test b/tests/cases/arithmetic_unsigned/subtract.test new file mode 100644 index 000000000..764701c2d --- /dev/null +++ b/tests/cases/arithmetic_unsigned/subtract.test @@ -0,0 +1,16 @@ +### SUBSTRAIT_SCALAR_TEST: v1.0 +### SUBSTRAIT_INCLUDE: extension:io.substrait:unsigned_integers + +# basic: Basic unsigned integer examples +subtract(('250')::u!u8, ('50')::u!u8) = ('200')::u!u8 +subtract(('60000')::u!u16, ('10000')::u!u16) = ('50000')::u!u16 +subtract(('4000000000')::u!u32, ('1000000000')::u!u32) = ('3000000000')::u!u32 +subtract(('11000000000000000000')::u!u64, ('1000000000000000000')::u!u64) = ('10000000000000000000')::u!u64 + +# overflow: Examples demonstrating overflow behavior +subtract(('0')::u!u8, ('1')::u!u8) [overflow:ERROR] = +subtract(('0')::u!u16, ('1')::u!u16) [overflow:ERROR] = +subtract(('0')::u!u32, ('1')::u!u32) [overflow:ERROR] = +subtract(('0')::u!u64, ('1')::u!u64) [overflow:ERROR] = +subtract(('0')::u!u8, ('1')::u!u8) [overflow:SATURATE] = ('0')::u!u8 +subtract(('0')::u!u8, ('1')::u!u8) [overflow:SILENT] = diff --git a/tests/cases/arithmetic_unsigned/sum.test b/tests/cases/arithmetic_unsigned/sum.test new file mode 100644 index 000000000..622f57309 --- /dev/null +++ b/tests/cases/arithmetic_unsigned/sum.test @@ -0,0 +1,16 @@ +### SUBSTRAIT_AGGREGATE_TEST: v1.0 +### SUBSTRAIT_INCLUDE: extension:io.substrait:unsigned_integers + +# basic: Basic unsigned integer examples +sum(('0', '1', '2', '20')::u!u8) = ('23')::u!u64? +sum(('100', '200', '50')::u!u16) = ('350')::u!u64? +sum(('2000000', '3217908', '629000', '100000', '0', '987654')::u!u32) = ('6934562')::u!u64? +sum(('1000000000000', '2000000000000', '3000000000000')::u!u64) = ('6000000000000')::u!u64? + +# overflow: Examples demonstrating overflow behavior +sum(('18446744073709551614', '1', '1', '1', '1', '10000000000')::u!u64) [overflow:ERROR] = + +# null_handling: Examples with null as input or output +sum((Null, Null, Null)::u!u16) = Null::u!u64? +sum(()::u!u16) = Null::u!u64? +sum(('200', Null, '50', '0', Null, '100')::u!u32) = ('350')::u!u64? diff --git a/tests/coverage/extensions.py b/tests/coverage/extensions.py index 65a364a01..40920a4e1 100644 --- a/tests/coverage/extensions.py +++ b/tests/coverage/extensions.py @@ -161,10 +161,16 @@ def add_functions_to_map(func_list, function_map, suffix, extension, urn): @staticmethod def read_substrait_extensions(dir_path: str): # read files from extensions directory + # Scan all YAML files, not just functions_* prefixed ones, so that + # self-contained extension type files (e.g. unsigned_integers.yaml) + # with function definitions are also discovered. Files without + # scalar_functions/aggregate_functions/window_functions sections are + # safely skipped by the guards below. unknown.yaml is excluded + # because it defines an "unknown" type not in the type mapping. extensions = [] for root, _dirs, files in os.walk(dir_path): for file in files: - if file.endswith(".yaml") and file.startswith("functions_"): + if file.endswith(".yaml") and file != "unknown.yaml": extensions.append(os.path.join(root, file)) extensions.sort()