diff --git a/core/docker/Dockerfile b/core/docker/Dockerfile index 232856796106..22a9d0cee9c0 100644 --- a/core/docker/Dockerfile +++ b/core/docker/Dockerfile @@ -50,6 +50,20 @@ RUN \ FROM registry.access.redhat.com/hi/core-runtime:latest ARG JDK_VERSION ARG ARCH +ARG TRINO_SOURCE_REVISION +ARG HOGQL_LANGUAGE_VERSION +ARG HOGQL_CATALOG_PROTOCOL_VERSION +ARG HOGQL_CATALOG_SCHEMA_VERSION +ARG TRINO_SERVER_SHA256 +ARG TRINO_CLI_SHA256 +LABEL org.opencontainers.image.revision="${TRINO_SOURCE_REVISION}" \ + io.posthog.trino.ducklake.revision="${TRINO_SOURCE_REVISION}" \ + io.posthog.trino.hogql.compiler-build="${TRINO_SOURCE_REVISION}" \ + io.posthog.trino.hogql.language-version="${HOGQL_LANGUAGE_VERSION}" \ + io.posthog.trino.hogql.catalog-protocol-version="${HOGQL_CATALOG_PROTOCOL_VERSION}" \ + io.posthog.trino.hogql.catalog-schema-version="${HOGQL_CATALOG_SCHEMA_VERSION}" \ + io.posthog.trino.server-sha256="${TRINO_SERVER_SHA256}" \ + io.posthog.trino.cli-sha256="${TRINO_CLI_SHA256}" ENV JAVA_HOME="/usr/lib/jvm/${JDK_VERSION}" ENV PATH=$PATH:$JAVA_HOME/bin ENV LANG=C.UTF-8 diff --git a/core/docker/build.sh b/core/docker/build.sh index 695b28867249..be61d29d3da2 100755 --- a/core/docker/build.sh +++ b/core/docker/build.sh @@ -32,6 +32,15 @@ TEMURIN_RELEASE=$("${SOURCE_DIR}/mvnw" -f "${SOURCE_DIR}/pom.xml" --quiet help:e TEMURIN_DOWNLOAD_URL="https://api.adoptium.net/v3/binary/version/{release_name}/linux/{arch}/jdk/hotspot/normal/eclipse?project=jdk" SKIP_TESTS=false +EXPLICIT_TRINO_SOURCE_REVISION="${TRINO_SOURCE_REVISION:-}" +TRINO_SOURCE_REVISION="${EXPLICIT_TRINO_SOURCE_REVISION}" +if [ -z "${TRINO_SOURCE_REVISION}" ] && git -C "${SOURCE_DIR}" rev-parse HEAD >/dev/null 2>&1; then + TRINO_SOURCE_REVISION="$(git -C "${SOURCE_DIR}" rev-parse HEAD)" +fi +TRINO_SOURCE_REVISION="${TRINO_SOURCE_REVISION:-unknown}" +HOGQL_LANGUAGE_VERSION="${HOGQL_LANGUAGE_VERSION:-1.0.0}" +HOGQL_CATALOG_PROTOCOL_VERSION="${HOGQL_CATALOG_PROTOCOL_VERSION:-1}" +HOGQL_CATALOG_SCHEMA_VERSION="${HOGQL_CATALOG_SCHEMA_VERSION:-2}" while getopts ":a:h:r:p:t:j:x" o; do case "${o}" in @@ -72,6 +81,11 @@ while getopts ":a:h:r:p:t:j:x" o; do done shift $((OPTIND - 1)) +if [ -n "${TRINO_VERSION}" ] && [ -z "${EXPLICIT_TRINO_SOURCE_REVISION}" ]; then + echo >&2 "TRINO_SOURCE_REVISION must be set when building downloaded release artifacts" + exit 1 +fi + function check_environment() { if ! command -v jq &> /dev/null; then echo >&2 "Please install jq" @@ -79,6 +93,14 @@ function check_environment() { fi } +function sha256_file() { + if command -v sha256sum &> /dev/null; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi +} + function temurin_download_uri() { local RELEASE_NAME="${1}" local ARCH="${2}" @@ -112,6 +134,8 @@ else trino_server="${SOURCE_DIR}/core/${SERVER_ARTIFACT}/target/${SERVER_ARTIFACT}-${TRINO_VERSION}.tar.gz" trino_client="${SOURCE_DIR}/client/trino-cli/target/trino-cli-${TRINO_VERSION}-executable.jar" fi +TRINO_SERVER_SHA256="$(sha256_file "${trino_server}")" +TRINO_CLI_SHA256="$(sha256_file "${trino_client}")" echo "🧱 Preparing the image build context directory" WORK_DIR="$(mktemp -d)" @@ -138,11 +162,39 @@ for arch in "${ARCHITECTURES[@]}"; do --build-arg ARCH="${arch}" \ --build-arg JDK_VERSION="${TEMURIN_RELEASE}" \ --build-arg JDK_DOWNLOAD_LINK="${JDK_DOWNLOAD_LINK}" \ + --build-arg TRINO_SOURCE_REVISION="${TRINO_SOURCE_REVISION}" \ + --build-arg HOGQL_LANGUAGE_VERSION="${HOGQL_LANGUAGE_VERSION}" \ + --build-arg HOGQL_CATALOG_PROTOCOL_VERSION="${HOGQL_CATALOG_PROTOCOL_VERSION}" \ + --build-arg HOGQL_CATALOG_SCHEMA_VERSION="${HOGQL_CATALOG_SCHEMA_VERSION}" \ + --build-arg TRINO_SERVER_SHA256="${TRINO_SERVER_SHA256}" \ + --build-arg TRINO_CLI_SHA256="${TRINO_CLI_SHA256}" \ --platform "linux/$arch" \ -f Dockerfile \ -t "${TAG}-$arch" done +function require_image_label() { + local image="$1" + local name="$2" + local expected="$3" + local actual + actual="$(docker image inspect --format "{{index .Config.Labels \"${name}\"}}" "${image}")" + if [ "${actual}" != "${expected}" ]; then + echo >&2 "Image label ${name} is ${actual}, expected ${expected}" + exit 1 + fi +} + +for arch in "${ARCHITECTURES[@]}"; do + image="${TAG}-${arch}" + require_image_label "${image}" "org.opencontainers.image.revision" "${TRINO_SOURCE_REVISION}" + require_image_label "${image}" "io.posthog.trino.hogql.language-version" "${HOGQL_LANGUAGE_VERSION}" + require_image_label "${image}" "io.posthog.trino.hogql.catalog-protocol-version" "${HOGQL_CATALOG_PROTOCOL_VERSION}" + require_image_label "${image}" "io.posthog.trino.hogql.catalog-schema-version" "${HOGQL_CATALOG_SCHEMA_VERSION}" + require_image_label "${image}" "io.posthog.trino.server-sha256" "${TRINO_SERVER_SHA256}" + require_image_label "${image}" "io.posthog.trino.cli-sha256" "${TRINO_CLI_SHA256}" +done + echo "🧹 Cleaning up the build context directory" rm -r "${WORK_DIR}" @@ -157,4 +209,3 @@ else docker image inspect -f '🚀 Built {{.RepoTags}} {{.Id}}' "${TAG}-$arch" done fi - diff --git a/core/trino-hogql-compiler/README.md b/core/trino-hogql-compiler/README.md new file mode 100644 index 000000000000..b4ecdd2739bc --- /dev/null +++ b/core/trino-hogql-compiler/README.md @@ -0,0 +1,220 @@ +# Native HogQL compiler + +This module compiles HogQL directly to Trino's public SQL AST. It deliberately +does not render an intermediate SQL string or add HogQL nodes to Trino's +analyzer and planner. + +The implementation is split into two modules: + +- `trino-hogql-parser` owns the HogQL grammar and a private, source-located + parser AST. It has no dependency on Trino's SQL tree. +- `trino-hogql-compiler` is the only translation boundary. It converts every + parser node into an `io.trino.sql.tree` node. + +The coordinator submits the resulting statement through the standard Trino +analysis, planning, scheduling, paging, and cancellation paths. The original +HogQL text remains the query text used by limits, history, and resource-group +selection. + +## M0 contract + +M0 accepts a deliberately small grammar: + +```text +SELECT [, ...] [FROM ] +``` + +Projections can be `*`, qualified column references, integers, strings, +booleans, or `NULL`. Identifiers can be unquoted, double quoted, or backquoted. +The parser rejects unsupported clauses and multiple statements instead of +silently interpreting them as SQL. + +## M1 syntax contract + +`HogQlParser.parseSyntax` accepts the complete canonical read-only query entry +point and returns an immutable, source-located tree containing grammar rules, +ANTLR alternative labels when present, and tokens. The tree contains no ANTLR +or Trino SQL objects. It classifies declarative HogQLX separately from read-only +queries. + +This syntax contract is intentionally independent from executable lowering. +`HogQlParser.parseStatement` and `HogQlCompiler.compile` continue to return an +explicit error for valid constructs that do not yet have a stock Trino AST +mapping. Adding syntax support therefore cannot silently claim semantic or +execution parity. + +## Executable contract + +The MVP lowers a frozen, fail-closed subset directly to stock Trino AST nodes: + +- logical `events` and `persons` tables supplied by a pinned Duckgres manifest; +- scalar VARCHAR JSON property reads and the `events.person` lazy relationship; +- parameters, CTEs, subqueries, joins, set operations, arrays, maps, rows, + intervals, grouping, ordering, limits, and windows that have exact stock + Trino representations; +- `LIMIT BY` over explicit projected outputs, lowered through a partitioned + `row_number` query while preserving global ordering and pagination; +- numeric/conditional/string functions (`abs`, `coalesce`, `if`, `lower`, `upper`, `length`, + `concat`, `replace`), collection functions (`map`, `arraySort`, + `arrayDistinct`, `arrayFlatten`, `arrayStringConcat`), date functions + (`dateAdd`, `dateDiff`, `dateTrunc`), aggregates (`count`, `sum`, `min`, + `max`, `avg`, `any`, `argMin`, `argMax`, `array_agg`), and windows + (`first_value`, `rank`, `row_number`). + +The compatibility extension also lowers conditional aggregates (`countIf`, +`sumIf`, `minIf`, `maxIf`, `avgIf`, `anyIf`, `argMaxIf`, `groupArrayIf`, +`uniqIf`, `uniqExactIf`, and `groupUniqArrayIf`), distinct aggregates +(`uniqExact`, `countDistinct`, and `groupUniqArray`), and `multiIf` directly to +stock Trino aggregate filters, DISTINCT aggregates, and searched CASE +expressions. +Aggregate rewrites preserve `OVER` specifications when the target Trino +aggregate is used as a window function. +JSON extraction lowers literal and runtime string/integer segments to Trino +JSON paths with HogQL-compatible defaults. The compatibility profile also +supports the corpus's typed map and key/value extraction forms through checked +JSON-to-map casts. +Date/time compatibility includes calendar arithmetic, interval-day values, +epoch conversion, timezone attachment/conversion, formatting, and checked +best-effort timestamp parsing. +Array-form membership and the nullish operator lower to stock `IN`, `IF`, and +`COALESCE` nodes, including nullable empty-membership behavior. +Function-form boolean/comparison operators and common string/math aliases +lower to their equivalent stock operators and functions. Checked/defaulting +float conversions, Decimal64 casts, and integer division lower to stock casts +and arithmetic. +Literal regex extraction preserves HogQL's capture-group and empty-default +behavior; matching and replace-all lower to Trino regex functions. +The extended profile also lowers extract-all and first-match replacement, plus +boolean, unsigned-integer, raw-array, and nested raw-map JSON extraction. +Common array transforms, predicates, indexing, summation, ranges, tuple +indexing, splitting, and membership lower to Trino higher-order functions. +Map key/value constructors and `mapUpdate` lower to stock map construction and +last-map-wins concatenation. +The row-expanding `arrayJoin` function is hoisted to a generated lateral +`CROSS JOIN UNNEST`; repeated calls remain independent row expansions. +Range and membership rewrites preserve empty-range and null-search behavior. +Typed defaults for out-of-range and no-match element lookups require resolved +element types and remain outside the frozen compatibility subset. +Expression-form comparison, arithmetic, membership, tuples, nullability +assertions, string splitting, month arithmetic, and common window aliases also +lower to stock Trino nodes. +Additional scalar coverage includes array slicing/enumeration, UUID and integer +conversion, JSON existence/value/serialization, hashing, powers, UTF-8 +substrings, banker-rounding aliases, and conditional median aggregation. +Simple floor, calendar-part, map construction, and null-predicate aliases are +mapped to their stock equivalents. Two-bound ranges, comparison and +subtraction aliases, calendar month/year extraction, ceiling aliases, narrow +integer casts, and JSON object-key extraction also lower to stock expressions. +The static-ID, single-choice `getSurveyResponse(index, question_id)` macro +prefers the modern response property and falls back to the legacy index key. +Parametric quantiles lower to Trino percentile aggregates, and limited +conditional group arrays lower to filtered `array_agg` plus `slice`. +Algorithm-level parity for `quantileExact` remains a compatibility-validation +item even though its query shape has a stock AST lowering. + +Function aliases are resolved before Trino analysis. Unknown functions fail as +HogQL resolution errors. The corpus compatibility extension also accepts every +logical table declared by the pinned manifest and expands declarative actions +by name or ID into stock predicates or relation membership. Cohorts, saved +queries, explicit modifiers without an enabled behavior, HogQLX, +PIVOT/UNPIVOT execution, unsupported ClickHouse clauses, and complete +type/function parity remain outside the endpoint profile. + +Set `hogql.enabled=true` on the coordinator to register `POST /v1/hogql`. The +request body is the versioned JSON envelope containing the raw HogQL query, +language/protocol versions, and optional typed bindings. The response uses +Trino's existing statement protocol. Follow-up result and cancellation URIs +remain under `/v1/statement`. +The endpoint is absent when the property is false, which is the default. +Production compilation uses a dedicated fixed-size executor with a bounded +queue and the `hogql.compilation-timeout` wall-time limit. Saturation and +timeout return retryable insufficient-resource errors; standard SQL bypasses +the compiler executor. + +Development is based on the fork's `ducklake-connector` integration branch. At +the start of this milestone its head was `posthog-484-ducklake.9`, commit +`3065d56e4d5e256962c8297b4299909db8301c0a`. + +Charts `main` still provides the deployment compatibility baseline: +`posthog-484-ducklake.6` at commit +`3bb054e5b80bc6ad740b26f6748b7b3f5548ff32`, published as +`ghcr.io/posthog/trino:484-ducklake.6`. Its linux/arm64 digest is +`sha256:85d2b37dfb7c966a1bfe7b0470efaa6a0116c86cb6a05d14a6bdc8c02f7a6787`. + +## Fork maintenance + +The `ducklake-connector` branch's common ancestor with `trinodb/trino` is +`67f588f0c81b21b425e1a43b05d70f9cf8798d6c`. The first nine PostHog commits +produce the Charts `main` `.6` baseline: + +1. `a002cc56980` adds the DuckLake connector and PostHog image workflow. +2. `4cd0573a57f` adds column-name mapping. +3. `500cdf40aa3` fixes Parquet millisecond timestamp reads. +4. `c03df89358a` reads fully visible partial files. +5. `6ef914e4149` splits large data files. +6. `09690b4fc8b` pools metadata connections and supports password files. +7. `e6c3fbd9d83` adds the PostgreSQL catalog store. +8. `0bab82c7520` counts rows from catalog metadata. +9. `3bb054e5b80` reads Parquet footers using the recorded size. + +The rolling integration branch continues through `.7`-`.9` with the DuckLake +write path, Trino views and schema evolution, partitioned-table reads, field-ID +based reads, and missing-file-column handling. Merge commits and CI maintenance +also live on this line; inspect the exact range from the upstream common +ancestor rather than assuming the release-tag history is linear. + +Keep native HogQL changes as a separate ordered stack above those commits: +parser/compiler modules, coordinator submission seam, metadata provider, +optional functions, then packaging and tests. During an upstream update, fetch +`trinodb/trino`, identify the new common ancestor, rebase the DuckLake series, +then rebase the HogQL series onto `ducklake-connector`. Resolve AST API changes +in `TrinoAstFactory` and submission conflicts in `QueuedStatementResource`, +`DispatchManager`, and `QueryPreparer`; do not introduce HogQL branches into +later execution stages. +Run the unchanged SQL/coordinator suites, the complete DuckLake suite, and the +HogQL suite before creating a release tag. + +`.github/workflows/docker-publish-posthog.yml` publishes linux/arm64 images. +Pushing `posthog-` builds the complete distribution and publishes +`ghcr.io/posthog/trino:`; a manual workflow dispatch accepts the same +image-tag without the `posthog-` prefix. The workflow intentionally skips tests +and checks, so its source commit must already have passed the suites above. + +## Validation + +On a fresh checkout, install their reactor dependencies without running tests: + +```shell +./mvnw -pl :trino-hogql-parser,:trino-hogql-compiler,:trino-main -am install \ + -DskipTests +``` + +Then run the parser, compiler, and coordinator tests together: + +```shell +./mvnw -pl :trino-hogql-parser,:trino-hogql-compiler,:trino-main test \ + -Dtest=TestHogQlParser,TestHogQlCompiler,TestQueryPreparer,TestHogQlConfig,TestHogQlStatementResource \ + -Dsurefire.failIfNoSpecifiedTests=false +``` + +The compiler tests compare the shared SQL subset with Trino's SQL parser, +format and reparse the generated tree, verify source locations, and assert that +no private HogQL AST node crosses the compiler boundary. + +Measure the standard SQL preparation path with and without the compiler +available using `BenchmarkQueryPreparer`. On JDK 25, enable annotation +processing when generating the JMH benchmark index: + +```shell +./mvnw -pl :trino-main clean test-compile exec:exec \ + -Dmaven.compiler.proc=full \ + -Dexec.executable=java \ + -Dexec.classpathScope=test \ + -Dexec.args='-cp %classpath io.trino.execution.BenchmarkQueryPreparer' +``` + +On 2026-08-26, one linux/arm64 Temurin 25 fork measured standard SQL +preparation at `3.792 ± 0.096 us/op` with the HogQL compiler available and +`3.708 ± 0.066 us/op` without it. The confidence intervals overlap; this M0 +sample does not show a distinguishable unused-path regression. Retain the +benchmark for repeated measurements on release hardware. diff --git a/core/trino-hogql-compiler/pom.xml b/core/trino-hogql-compiler/pom.xml new file mode 100644 index 000000000000..d4ae8742ebfa --- /dev/null +++ b/core/trino-hogql-compiler/pom.xml @@ -0,0 +1,94 @@ + + + 4.0.0 + + + io.trino + trino-root + 484-SNAPSHOT + ../../pom.xml + + + trino-hogql-compiler + ${project.artifactId} + Trino - HogQL compiler + + + + com.fasterxml.jackson.core + jackson-core + + + + com.fasterxml.jackson.core + jackson-databind + + + + io.airlift + slice + + + + io.trino + trino-hogql-parser + + + + io.trino + trino-parser + + + + io.trino + trino-re2j + + + + io.trino + trino-spi + + + + io.trino + trino-testing-services + test + + + + org.assertj + assertj-core + test + + + + org.junit.jupiter + junit-jupiter-api + test + + + + org.junit.jupiter + junit-jupiter-engine + test + + + + org.junit.jupiter + junit-jupiter-params + test + + + + org.openjdk.jmh + jmh-core + test + + + + org.openjdk.jmh + jmh-generator-annprocess + test + + + diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCastTypeTranslator.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCastTypeTranslator.java new file mode 100644 index 000000000000..8a56220954b7 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCastTypeTranslator.java @@ -0,0 +1,350 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Optional; + +import static java.lang.Integer.parseInt; +import static java.util.Objects.requireNonNull; + +final class HogQlCastTypeTranslator +{ + private HogQlCastTypeTranslator() {} + + public static String translate(String type) + { + Translation translation = translateType(requireNonNull(type, "type is null").trim()); + return translation.sql(); + } + + private static Translation translateType(String type) + { + if (type.isEmpty()) { + throw unsupported("empty type"); + } + if (type.endsWith("[]")) { + return composite("array(" + translateType(type.substring(0, type.length() - 2)).sql() + ")"); + } + if (type.matches(".*\\[\\s*\\d+\\s*]$")) { + throw unsupported("fixed-size array syntax"); + } + + Optional call = parseCall(type); + String name = call.map(Call::name).orElse(type).trim().toLowerCase(Locale.ENGLISH); + List arguments = call.map(Call::arguments).orElseGet(List::of); + String suffix = call.map(Call::suffix).orElse("").trim().toLowerCase(Locale.ENGLISH); + + if (name.matches("u?int(8|16|32|64|128|256)")) { + if (name.startsWith("u")) { + throw unsupported("unsigned integer range"); + } + return switch (name) { + case "int8" -> scalar("tinyint"); + case "int16" -> scalar("smallint"); + case "int32" -> scalar("integer"); + case "int64" -> scalar("bigint"); + default -> throw unsupported("integer width exceeds Trino bigint"); + }; + } + if (name.matches("decimal(32|64|128|256)")) { + requireArity(name, arguments, 1); + int precision = switch (name) { + case "decimal32" -> 9; + case "decimal64" -> 18; + case "decimal128" -> 38; + default -> throw unsupported("decimal precision exceeds Trino decimal(38, s)"); + }; + return decimal(precision, parseUnsigned(arguments.getFirst(), "decimal scale")); + } + + return switch (name) { + case "int", "integer" -> noArguments(name, arguments, scalar("integer")); + case "bigint" -> noArguments(name, arguments, scalar("bigint")); + case "smallint" -> noArguments(name, arguments, scalar("smallint")); + case "tinyint" -> noArguments(name, arguments, scalar("tinyint")); + case "float", "real" -> noArguments(name, arguments, scalar("real")); + case "float64", "double", "double precision" -> noArguments(name, arguments, scalar("double")); + case "float32" -> noArguments(name, arguments, scalar("real")); + case "string", "text" -> noArguments(name, arguments, scalar("varchar")); + case "varchar", "char" -> characterType(name, arguments); + case "bool", "boolean" -> noArguments(name, arguments, scalar("boolean")); + case "date" -> noArguments(name, arguments, scalar("date")); + case "uuid" -> noArguments(name, arguments, scalar("uuid")); + case "json" -> noArguments(name, arguments, scalar("json")); + case "decimal", "numeric" -> decimal(arguments); + case "nullable" -> nullable(arguments); + case "array" -> unaryComposite("array", arguments); + case "map" -> map(arguments); + case "tuple", "row" -> row(arguments); + case "timestamp", "time" -> temporal(name, arguments, suffix); + case "timestamp with time zone" -> noArguments(name, arguments, scalar("timestamp(3) with time zone")); + case "time with time zone" -> noArguments(name, arguments, scalar("time(3) with time zone")); + case "timestamp with local time zone", "time with local time zone" -> throw unsupported("WITH LOCAL TIME ZONE semantics differ from Trino WITH TIME ZONE"); + case "interval day to second", "interval year to month" -> noArguments(name, arguments, scalar(name)); + case "fixedstring" -> throw unsupported("FixedString padding semantics"); + case "date32" -> throw unsupported("Date32 range"); + case "datetime", "datetime64", "timestamptz" -> throw unsupported("ClickHouse time-zone and range semantics"); + case "interval" -> throw unsupported("interval qualifier is required"); + default -> throw unsupported("unknown type family"); + }; + } + + private static Translation characterType(String name, List arguments) + { + if (arguments.isEmpty()) { + return scalar(name); + } + requireArity(name, arguments, 1); + int length = parseUnsigned(arguments.getFirst(), name + " length"); + if (length < 1) { + throw unsupported(name + " length must be positive"); + } + return scalar(name + "(" + length + ")"); + } + + private static Translation decimal(List arguments) + { + requireArity("decimal", arguments, 2); + return decimal( + parseUnsigned(arguments.getFirst(), "decimal precision"), + parseUnsigned(arguments.get(1), "decimal scale")); + } + + private static Translation decimal(int precision, int scale) + { + if (precision < 1 || precision > 38 || scale > precision) { + throw unsupported("decimal requires 1 <= precision <= 38 and 0 <= scale <= precision"); + } + return scalar("decimal(" + precision + ", " + scale + ")"); + } + + private static Translation nullable(List arguments) + { + requireArity("nullable", arguments, 1); + Translation nested = translateType(arguments.getFirst()); + if (nested.composite()) { + throw unsupported("ClickHouse Nullable cannot wrap Array, Map, Tuple, or Row"); + } + return nested; + } + + private static Translation unaryComposite(String name, List arguments) + { + requireArity(name, arguments, 1); + return composite(name + "(" + translateType(arguments.getFirst()).sql() + ")"); + } + + private static Translation map(List arguments) + { + requireArity("map", arguments, 2); + return composite("map(" + translateType(arguments.getFirst()).sql() + ", " + translateType(arguments.get(1)).sql() + ")"); + } + + private static Translation row(List arguments) + { + if (arguments.isEmpty()) { + throw unsupported("row requires at least one field"); + } + return composite("row(" + String.join(", ", arguments.stream().map(HogQlCastTypeTranslator::rowField).toList()) + ")"); + } + + private static String rowField(String argument) + { + try { + return translateType(argument).sql(); + } + catch (IllegalArgumentException _) { + int separator = topLevelWhitespace(argument); + if (separator < 0) { + throw unsupported("invalid tuple or row field"); + } + String fieldName = argument.substring(0, separator).trim(); + String fieldType = argument.substring(separator).trim(); + if (!fieldName.matches("[A-Za-z_][A-Za-z0-9_]*|\"(?:\"\"|[^\"])+\"")) { + throw unsupported("invalid tuple or row field name"); + } + return fieldName + " " + translateType(fieldType).sql(); + } + } + + private static Translation temporal(String name, List arguments, String suffix) + { + if (arguments.size() > 1) { + throw unsupported(name + " accepts at most one precision"); + } + int precision = arguments.isEmpty() ? 3 : parseUnsigned(arguments.getFirst(), name + " precision"); + if (precision > 12) { + throw unsupported(name + " precision exceeds 12"); + } + if (suffix.equals("with local time zone")) { + throw unsupported("WITH LOCAL TIME ZONE semantics differ from Trino WITH TIME ZONE"); + } + if (!suffix.isEmpty() && !suffix.equals("with time zone")) { + throw unsupported("invalid time-zone qualifier"); + } + String qualifier = suffix.isEmpty() ? "" : " with time zone"; + return scalar(name + "(" + precision + ")" + qualifier); + } + + private static Optional parseCall(String type) + { + int opening = type.indexOf('('); + if (opening < 0) { + return Optional.empty(); + } + int closing = matchingParenthesis(type, opening); + if (closing < 0) { + throw unsupported("unbalanced parentheses"); + } + String suffix = type.substring(closing + 1); + if (suffix.contains("(") || suffix.contains(")")) { + throw unsupported("invalid type suffix"); + } + return Optional.of(new Call( + type.substring(0, opening).trim(), + splitArguments(type.substring(opening + 1, closing)), + suffix)); + } + + private static List splitArguments(String input) + { + if (input.isBlank()) { + return List.of(); + } + List arguments = new ArrayList<>(); + int depth = 0; + boolean quoted = false; + int start = 0; + for (int index = 0; index < input.length(); index++) { + char character = input.charAt(index); + if (character == '\'' || character == '"') { + quoted = !quoted; + } + else if (!quoted && character == '(') { + depth++; + } + else if (!quoted && character == ')') { + depth--; + } + else if (!quoted && depth == 0 && character == ',') { + arguments.add(nonEmpty(input.substring(start, index))); + start = index + 1; + } + } + if (depth != 0 || quoted) { + throw unsupported("unbalanced nested type"); + } + arguments.add(nonEmpty(input.substring(start))); + return List.copyOf(arguments); + } + + private static int matchingParenthesis(String input, int opening) + { + int depth = 0; + boolean quoted = false; + for (int index = opening; index < input.length(); index++) { + char character = input.charAt(index); + if (character == '\'' || character == '"') { + quoted = !quoted; + } + else if (!quoted && character == '(') { + depth++; + } + else if (!quoted && character == ')' && --depth == 0) { + return index; + } + } + return -1; + } + + private static int topLevelWhitespace(String input) + { + int depth = 0; + boolean quoted = false; + for (int index = 0; index < input.length(); index++) { + char character = input.charAt(index); + if (character == '"') { + quoted = !quoted; + } + else if (!quoted && character == '(') { + depth++; + } + else if (!quoted && character == ')') { + depth--; + } + else if (!quoted && depth == 0 && Character.isWhitespace(character)) { + return index; + } + } + return -1; + } + + private static int parseUnsigned(String value, String description) + { + String normalized = value.trim(); + if (!normalized.matches("\\d+")) { + throw unsupported(description + " must be an unsigned integer"); + } + try { + return parseInt(normalized); + } + catch (NumberFormatException _) { + throw unsupported(description + " is too large"); + } + } + + private static Translation noArguments(String name, List arguments, Translation translation) + { + requireArity(name, arguments, 0); + return translation; + } + + private static void requireArity(String name, List arguments, int expected) + { + if (arguments.size() != expected) { + throw unsupported(name + " requires " + expected + " type argument(s)"); + } + } + + private static String nonEmpty(String value) + { + String normalized = value.trim(); + if (normalized.isEmpty()) { + throw unsupported("empty type argument"); + } + return normalized; + } + + private static Translation scalar(String sql) + { + return new Translation(sql, false); + } + + private static Translation composite(String sql) + { + return new Translation(sql, true); + } + + private static IllegalArgumentException unsupported(String reason) + { + return new IllegalArgumentException(reason); + } + + private record Call(String name, List arguments, String suffix) {} + + private record Translation(String sql, boolean composite) {} +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompilationResult.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompilationResult.java new file mode 100644 index 000000000000..487f17299d58 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompilationResult.java @@ -0,0 +1,48 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.sql.tree.Statement; + +import java.util.List; +import java.util.OptionalLong; + +import static java.util.Objects.requireNonNull; + +public record HogQlCompilationResult( + Statement statement, + List parameterNames, + List modifierBindings, + OptionalLong catalogGeneration, + OptionalLong exchangeRateGeneration) +{ + public HogQlCompilationResult + { + statement = requireNonNull(statement, "statement is null"); + parameterNames = List.copyOf(requireNonNull(parameterNames, "parameterNames is null")); + modifierBindings = List.copyOf(requireNonNull(modifierBindings, "modifierBindings is null")); + catalogGeneration = requireNonNull(catalogGeneration, "catalogGeneration is null"); + exchangeRateGeneration = requireNonNull(exchangeRateGeneration, "exchangeRateGeneration is null"); + } + + public HogQlCompilationResult(Statement statement, List parameterNames, OptionalLong catalogGeneration) + { + this(statement, parameterNames, List.of(), catalogGeneration, OptionalLong.empty()); + } + + public HogQlCompilationResult(Statement statement, List parameterNames, List modifierBindings, OptionalLong catalogGeneration) + { + this(statement, parameterNames, modifierBindings, catalogGeneration, OptionalLong.empty()); + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompileEnvelope.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompileEnvelope.java new file mode 100644 index 000000000000..233e2a24b14a --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompileEnvelope.java @@ -0,0 +1,130 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.hogql.parser.HogQlLanguageVersion; + +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.Set; + +import static java.util.Objects.requireNonNull; + +public record HogQlCompileEnvelope( + String query, + int protocolVersion, + HogQlLanguageVersion languageVersion, + Map parameters, + Map variables, + Map filters, + Map modifiers, + OptionalLong catalogGeneration) +{ + public static final int PROTOCOL_VERSION = 1; + + private static final Set SEMANTIC_FIELDS = Set.of("parameters", "variables", "filters", "modifiers"); + + public HogQlCompileEnvelope + { + query = requireNonNull(query, "query is null"); + if (query.isBlank()) { + throw new IllegalArgumentException("query is empty"); + } + if (protocolVersion != PROTOCOL_VERSION) { + throw new IllegalArgumentException("unsupported HogQL protocol version"); + } + languageVersion = requireNonNull(languageVersion, "languageVersion is null"); + if (!languageVersion.equals(HogQlLanguageContract.current().languageVersion())) { + throw new IllegalArgumentException("unsupported HogQL language version"); + } + parameters = immutableValues("parameters", parameters); + variables = immutableValues("variables", variables); + filters = immutableValues("filters", filters); + modifiers = immutableValues("modifiers", modifiers); + catalogGeneration = requireNonNull(catalogGeneration, "catalogGeneration is null"); + catalogGeneration.ifPresent(generation -> { + if (generation <= 0) { + throw new IllegalArgumentException("catalog generation must be positive"); + } + }); + } + + public static HogQlCompileEnvelope fromSemanticFields( + String query, + int protocolVersion, + HogQlLanguageVersion languageVersion, + Map> semanticFields, + OptionalLong catalogGeneration) + { + requireNonNull(semanticFields, "semanticFields is null"); + if (!SEMANTIC_FIELDS.containsAll(semanticFields.keySet())) { + throw new IllegalArgumentException("unknown HogQL semantic field"); + } + return new HogQlCompileEnvelope( + query, + protocolVersion, + languageVersion, + semanticFields.getOrDefault("parameters", Map.of()), + semanticFields.getOrDefault("variables", Map.of()), + semanticFields.getOrDefault("filters", Map.of()), + semanticFields.getOrDefault("modifiers", Map.of()), + catalogGeneration); + } + + public Optional bindingForPlaceholder(String placeholder) + { + requireNonNull(placeholder, "placeholder is null"); + int separator = placeholder.indexOf('.'); + if (separator < 0) { + return Optional.ofNullable(parameters.get(placeholder)); + } + String namespace = placeholder.substring(0, separator); + String name = placeholder.substring(separator + 1); + return switch (namespace) { + case "variables" -> Optional.ofNullable(variables.get(name)); + case "filters" -> Optional.ofNullable(filters.get(name)); + default -> Optional.empty(); + }; + } + + private static Map immutableValues(String field, Map values) + { + requireNonNull(values, field + " is null"); + for (Map.Entry entry : values.entrySet()) { + if (entry.getKey() == null || entry.getKey().isBlank()) { + throw new IllegalArgumentException(field + " contains an empty name"); + } + if (entry.getValue() == null) { + throw new IllegalArgumentException(field + " contains a missing typed value"); + } + } + return Map.copyOf(values); + } + + @Override + public String toString() + { + return "HogQlCompileEnvelope[protocolVersion=%s, languageVersion=%s, query=, parameters=%s, variables=%s, filters=%s, modifiers=%s, catalogGenerationPresent=%s]" + .formatted( + protocolVersion, + languageVersion, + parameters.size(), + variables.size(), + filters.size(), + modifiers.size(), + catalogGeneration.isPresent()); + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java new file mode 100644 index 000000000000..82b57b3f383c --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlCompiler.java @@ -0,0 +1,663 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotProvider; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogException; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogException.Failure; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinRequest; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.hogql.parser.HogQlLanguageVersion; +import io.trino.hogql.parser.HogQlParser; +import io.trino.hogql.parser.HogQlParsingException; +import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.AliasedRelation; +import io.trino.hogql.parser.tree.HogQlQuery.ArrayExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BetweenExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CaseExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CastExpression; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsList; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsRegex; +import io.trino.hogql.parser.tree.HogQlQuery.CommonTableReference; +import io.trino.hogql.parser.tree.HogQlQuery.Expression; +import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; +import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; +import io.trino.hogql.parser.tree.HogQlQuery.InCohortExpression; +import io.trino.hogql.parser.tree.HogQlQuery.InExpression; +import io.trino.hogql.parser.tree.HogQlQuery.InSubqueryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IntervalExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; +import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; +import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; +import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.LambdaExpression; +import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; +import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; +import io.trino.hogql.parser.tree.HogQlQuery.Projection; +import io.trino.hogql.parser.tree.HogQlQuery.Relation; +import io.trino.hogql.parser.tree.HogQlQuery.ScalarSubqueryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.SelectQueryBody; +import io.trino.hogql.parser.tree.HogQlQuery.SetOperation; +import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; +import io.trino.hogql.parser.tree.HogQlQuery.Star; +import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; +import io.trino.hogql.parser.tree.HogQlQuery.SubscriptExpression; +import io.trino.hogql.parser.tree.HogQlQuery.TablePlaceholder; +import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; +import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.UnnestRelation; +import io.trino.hogql.parser.tree.HogQlQuery.ValuesRelation; +import io.trino.hogql.parser.tree.HogQlQuery.Window; +import io.trino.hogql.parser.tree.HogQlQuery.WindowDefinition; +import io.trino.hogql.parser.tree.HogQlQuery.WindowReference; +import io.trino.hogql.parser.tree.HogQlQuery.WindowSpecification; +import io.trino.spi.Location; +import io.trino.spi.TrinoException; +import io.trino.sql.tree.Statement; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.Set; + +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_BINDING_ERROR; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_SYNTAX_ERROR; +import static java.util.Objects.requireNonNull; + +public final class HogQlCompiler +{ + private final HogQlParser parser; + private final Optional exchangeRateSnapshotProvider; + + public HogQlCompiler() + { + this(new HogQlParser(), Optional.empty()); + } + + HogQlCompiler(HogQlParser parser) + { + this(parser, Optional.empty()); + } + + public HogQlCompiler(HogQlExchangeRateSnapshotProvider exchangeRateSnapshotProvider) + { + this(new HogQlParser(), Optional.of(requireNonNull(exchangeRateSnapshotProvider, "exchangeRateSnapshotProvider is null"))); + } + + private HogQlCompiler(HogQlParser parser, Optional exchangeRateSnapshotProvider) + { + this.parser = requireNonNull(parser, "parser is null"); + this.exchangeRateSnapshotProvider = requireNonNull(exchangeRateSnapshotProvider, "exchangeRateSnapshotProvider is null"); + } + + public Statement compile(String hogql) + { + return compile(hogql, Map.of()).statement(); + } + + public HogQlCompilationResult compile(HogQlCompileEnvelope envelope) + { + return compile(envelope, Optional.empty()); + } + + public HogQlCompilationResult compile(HogQlCompileEnvelope envelope, Optional catalogContext) + { + requireNonNull(envelope, "envelope is null"); + requireNonNull(catalogContext, "catalogContext is null"); + return compile( + parse(envelope.query()), + envelope, + envelope.modifiers(), + catalogContext, + envelope.languageVersion(), + envelope.catalogGeneration(), + false, + exchangeRateSnapshotProvider); + } + + public HogQlCompilationResult compileV0(HogQlCompileEnvelope envelope, Optional catalogContext) + { + requireNonNull(envelope, "envelope is null"); + requireNonNull(catalogContext, "catalogContext is null"); + if (!envelope.modifiers().isEmpty()) { + throw unsupportedError(parse(envelope.query()).span(), "Modifiers are outside the HogQL v0 profile"); + } + return compile( + parse(envelope.query()), + envelope, + Map.of(), + catalogContext, + envelope.languageVersion(), + envelope.catalogGeneration(), + true, + exchangeRateSnapshotProvider); + } + + public HogQlCompilationResult compile(String hogql, Map parameters) + { + requireNonNull(hogql, "hogql is null"); + requireNonNull(parameters, "parameters is null"); + return compile( + parse(hogql), + new HogQlCompileEnvelope( + hogql, + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + parameters, + Map.of(), + Map.of(), + Map.of(), + OptionalLong.empty()), + Map.of(), + Optional.empty(), + HogQlLanguageContract.current().languageVersion(), + OptionalLong.empty(), + false, + exchangeRateSnapshotProvider); + } + + private HogQlQuery parse(String hogql) + { + requireNonNull(hogql, "hogql is null"); + try { + return parser.parseStatement(hogql); + } + catch (HogQlParsingException exception) { + throw new TrinoException( + HOGQL_SYNTAX_ERROR, + Optional.of(new Location(exception.getLineNumber(), exception.getColumnNumber())), + exception.getErrorMessage(), + exception); + } + } + + private static HogQlCompilationResult compile( + HogQlQuery query, + HogQlCompileEnvelope envelope, + Map modifiers, + Optional catalogContext, + HogQlLanguageVersion languageVersion, + OptionalLong expectedCatalogGeneration, + boolean v0Profile, + Optional exchangeRateSnapshotProvider) + { + validateParameters(query, envelope.parameters()); + validateQuery(query); + + List placeholders = new ArrayList<>(); + collectPlaceholders(query, placeholders); + placeholders.sort(Comparator.comparingInt(placeholder -> placeholder.span().startOffset())); + + List missing = placeholders.stream() + .map(Placeholder::name) + .filter(name -> envelope.bindingForPlaceholder(name).isEmpty()) + .distinct() + .toList(); + if (!missing.isEmpty()) { + Placeholder firstMissing = placeholders.stream() + .filter(placeholder -> missing.contains(placeholder.name())) + .findFirst() + .orElseThrow(); + throw bindingError(firstMissing.span(), "Missing HogQL parameter bindings: " + String.join(", ", missing)); + } + + Set parameterPlaceholderNames = new HashSet<>(); + placeholders.stream() + .map(Placeholder::name) + .filter(name -> !name.contains(".")) + .forEach(parameterPlaceholderNames::add); + List extra = envelope.parameters().keySet().stream() + .filter(name -> !parameterPlaceholderNames.contains(name)) + .sorted() + .toList(); + if (!extra.isEmpty()) { + throw bindingError(query.span(), "Unused HogQL parameter bindings: " + String.join(", ", extra)); + } + + if (v0Profile) { + HogQlV0ProfileValidator.validate(query, Optional.empty()); + } + ResolvedQuery resolved = resolveQuery(query, catalogContext, languageVersion, expectedCatalogGeneration, !modifiers.isEmpty(), v0Profile, exchangeRateSnapshotProvider); + List resolvedPlaceholders = new ArrayList<>(); + collectPlaceholders(resolved.query(), resolvedPlaceholders); + resolvedPlaceholders.sort(Comparator.comparingInt(placeholder -> placeholder.span().startOffset())); + Map parameterIds = new HashMap<>(); + for (int index = 0; index < resolvedPlaceholders.size(); index++) { + parameterIds.put(resolvedPlaceholders.get(index).span(), index); + } + Statement statement = TrinoAstFactory.createStatement(resolved.query(), parameterIds); + List modifierBindings = resolved.pinnedSnapshot() + .map(snapshot -> HogQlModifierResolver.resolve(snapshot, modifiers, query.span())) + .orElseGet(List::of); + return new HogQlCompilationResult( + statement, + resolvedPlaceholders.stream() + .map(Placeholder::name) + .toList(), + modifierBindings, + resolved.catalogGeneration(), + resolved.exchangeRateGeneration()); + } + + private static void collectPlaceholders(HogQlQuery query, List placeholders) + { + query.with().forEach(commonTable -> collectPlaceholders(commonTable.query(), placeholders)); + switch (query.body()) { + case SelectQueryBody select -> { + select.projections().forEach(projection -> { + switch (projection) { + case ColumnsList columns -> columns.expressions().forEach(expression -> collectPlaceholders(expression, placeholders)); + case ColumnsRegex _ -> {} + case ExpressionProjection expression -> collectPlaceholders(expression.expression(), placeholders); + case Star star -> star.replacements().forEach(replacement -> collectPlaceholders(replacement.expression(), placeholders)); + } + }); + select.from().ifPresent(relation -> collectPlaceholders(relation, placeholders)); + select.where().ifPresent(expression -> collectPlaceholders(expression, placeholders)); + select.groupBy().forEach(expression -> collectPlaceholders(expression, placeholders)); + select.having().ifPresent(expression -> collectPlaceholders(expression, placeholders)); + select.windows().forEach(window -> collectPlaceholders(window.specification(), placeholders)); + select.limitBy().ifPresent(limitBy -> { + collectPlaceholders(limitBy.limit(), placeholders); + limitBy.offset().ifPresent(expression -> collectPlaceholders(expression, placeholders)); + limitBy.partitionBy().forEach(expression -> collectPlaceholders(expression, placeholders)); + }); + } + case SetOperation setOperation -> { + collectPlaceholders(setOperation.left(), placeholders); + collectPlaceholders(setOperation.right(), placeholders); + } + } + query.orderBy().forEach(sortItem -> collectPlaceholders(sortItem.expression(), placeholders)); + query.limit().ifPresent(expression -> collectPlaceholders(expression, placeholders)); + query.offset().ifPresent(expression -> collectPlaceholders(expression, placeholders)); + } + + private static ResolvedQuery resolveQuery( + HogQlQuery query, + Optional catalogContext, + HogQlLanguageVersion languageVersion, + OptionalLong expectedCatalogGeneration, + boolean modifiersRequireSnapshot, + boolean v0Profile, + Optional exchangeRateSnapshotProvider) + { + boolean semanticCandidate = containsSemanticCandidate(query); + if (!semanticCandidate && !modifiersRequireSnapshot) { + return new ResolvedQuery(HogQlSelectAliasRewriter.rewrite(query), Optional.empty(), OptionalLong.empty()); + } + if (catalogContext.isEmpty()) { + if (modifiersRequireSnapshot) { + throw new HogQlSemanticCatalogException(Failure.UNAVAILABLE, "HogQL semantic catalog snapshot is required for modifiers"); + } + if (!containsFunctionCall(query)) { + return new ResolvedQuery(query, Optional.empty(), OptionalLong.empty()); + } + HogQlFunctionResolver.Resolution resolution = HogQlFunctionResolver.resolve(query, exchangeRateSnapshotProvider); + return new ResolvedQuery(HogQlSelectAliasRewriter.rewrite(resolution.query()), Optional.empty(), resolution.exchangeRateGeneration()); + } + HogQlSemanticCatalogContext context = catalogContext.orElseThrow(); + PinnedSnapshot pinned = context.snapshotProvider().pin(new PinRequest( + context.catalog(), + requireNonNull(languageVersion, "languageVersion is null"), + expectedCatalogGeneration)); + if (v0Profile) { + HogQlV0ProfileValidator.validate(query, Optional.of(pinned.snapshot())); + } + HogQlQuery resolved = HogQlSelectAliasRewriter.rewrite(query); + OptionalLong exchangeRateGeneration = OptionalLong.empty(); + if (semanticCandidate) { + HogQlFunctionResolver.Resolution functionResolution = v0Profile + ? HogQlFunctionResolver.resolveV0(query, exchangeRateSnapshotProvider, true) + : HogQlFunctionResolver.resolve(pinned, query, exchangeRateSnapshotProvider); + exchangeRateGeneration = functionResolution.exchangeRateGeneration(); + HogQlQuery functionsResolved = HogQlSelectAliasRewriter.rewrite(functionResolution.query()); + if (hasSemanticDefinitions(pinned.snapshot())) { + resolved = HogQlSemanticResolver.resolve(pinned, functionsResolved) + .map(HogQlSemanticResolver.ResolvedQuery::query) + .orElse(functionsResolved); + } + else { + resolved = functionsResolved; + } + } + return new ResolvedQuery(resolved, Optional.of(pinned), exchangeRateGeneration); + } + + private static boolean hasSemanticDefinitions(io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot snapshot) + { + return !snapshot.logicalTables().isEmpty() || + !snapshot.virtualTables().isEmpty() || + !snapshot.savedQueries().isEmpty() || + !snapshot.materializedViews().isEmpty() || + !snapshot.expressionFields().isEmpty() || + !snapshot.actions().isEmpty() || + !snapshot.cohorts().isEmpty(); + } + + private static boolean containsSemanticCandidate(HogQlQuery query) + { + return containsFunctionCall(query) || + query.with().stream().anyMatch(commonTable -> containsSemanticCandidate(commonTable.query())) || + switch (query.body()) { + case SelectQueryBody select -> select.from().map(HogQlCompiler::containsSemanticCandidate).orElse(false); + case SetOperation setOperation -> containsSemanticCandidate(setOperation.left()) || containsSemanticCandidate(setOperation.right()); + }; + } + + private static boolean containsSemanticCandidate(Relation relation) + { + return switch (relation) { + case AliasedRelation alias -> containsSemanticCandidate(alias.relation()); + case CommonTableReference _ -> false; + case JoinRelation join -> containsSemanticCandidate(join.left()) || containsSemanticCandidate(join.right()); + case HogQlQuery.PivotRelation pivot -> containsSemanticCandidate(pivot.input()) || + pivot.aggregations().stream().anyMatch(aggregation -> containsFunctionCall(aggregation.expression())); + case SubqueryRelation subquery -> containsSemanticCandidate(subquery.query()); + case TablePlaceholder _ -> false; + case HogQlQuery.TableReference table -> table.parts().size() == 1; + case UnnestRelation unnest -> unnest.expressions().stream().anyMatch(HogQlCompiler::containsFunctionCall); + case ValuesRelation _ -> false; + }; + } + + private static boolean containsFunctionCall(HogQlQuery query) + { + return query.with().stream().anyMatch(commonTable -> containsFunctionCall(commonTable.query())) || + switch (query.body()) { + case SelectQueryBody select -> select.projections().stream().anyMatch(HogQlCompiler::containsFunctionCall) || + select.from().map(HogQlCompiler::containsFunctionCall).orElse(false) || + select.where().map(HogQlCompiler::containsFunctionCall).orElse(false) || + select.groupBy().stream().anyMatch(HogQlCompiler::containsFunctionCall) || + select.having().map(HogQlCompiler::containsFunctionCall).orElse(false) || + select.windows().stream().map(WindowDefinition::specification).anyMatch(HogQlCompiler::containsFunctionCall) || + select.limitBy().map(limitBy -> + containsFunctionCall(limitBy.limit()) || + containsFunctionCall(limitBy.offset()) || + limitBy.partitionBy().stream().anyMatch(HogQlCompiler::containsFunctionCall)).orElse(false); + case SetOperation set -> containsFunctionCall(set.left()) || containsFunctionCall(set.right()); + } || + query.orderBy().stream().anyMatch(sortItem -> containsFunctionCall(sortItem.expression())) || + query.limit().map(HogQlCompiler::containsFunctionCall).orElse(false) || + query.offset().map(HogQlCompiler::containsFunctionCall).orElse(false); + } + + private static boolean containsFunctionCall(Relation relation) + { + return switch (relation) { + case AliasedRelation alias -> containsFunctionCall(alias.relation()); + case CommonTableReference _ -> false; + case JoinRelation join -> containsFunctionCall(join.left()) || + containsFunctionCall(join.right()) || + join.criteria().filter(JoinOn.class::isInstance) + .map(JoinOn.class::cast) + .map(JoinOn::expression) + .map(HogQlCompiler::containsFunctionCall) + .orElse(false); + case HogQlQuery.PivotRelation pivot -> containsFunctionCall(pivot.input()) || + pivot.aggregations().stream().anyMatch(aggregation -> containsFunctionCall(aggregation.expression())) || + pivot.pivotColumns().stream().anyMatch(HogQlCompiler::containsFunctionCall) || + pivot.valueGroups().stream() + .flatMap(group -> group.values().stream()) + .anyMatch(HogQlCompiler::containsFunctionCall) || + pivot.groupBy().stream().anyMatch(HogQlCompiler::containsFunctionCall); + case SubqueryRelation subquery -> containsFunctionCall(subquery.query()); + case TablePlaceholder _, HogQlQuery.TableReference _ -> false; + case UnnestRelation unnest -> unnest.expressions().stream().anyMatch(HogQlCompiler::containsFunctionCall); + case ValuesRelation values -> values.rows().stream() + .flatMap(List::stream) + .anyMatch(HogQlCompiler::containsFunctionCall); + }; + } + + private static boolean containsFunctionCall(Projection projection) + { + return switch (projection) { + case ColumnsList columns -> columns.expressions().stream().anyMatch(HogQlCompiler::containsFunctionCall); + case ColumnsRegex _ -> false; + case ExpressionProjection expression -> containsFunctionCall(expression.expression()); + case Star star -> star.replacements().stream().anyMatch(replacement -> containsFunctionCall(replacement.expression())); + }; + } + + private static boolean containsFunctionCall(Expression expression) + { + return switch (expression) { + case ArrayExpression array -> array.values().stream().anyMatch(HogQlCompiler::containsFunctionCall); + case BetweenExpression between -> containsFunctionCall(between.value()) || containsFunctionCall(between.min()) || containsFunctionCall(between.max()); + case BinaryExpression binary -> containsFunctionCall(binary.left()) || containsFunctionCall(binary.right()); + case CaseExpression caseExpression -> caseExpression.operand().map(HogQlCompiler::containsFunctionCall).orElse(false) || + caseExpression.whenClauses().stream().anyMatch(when -> containsFunctionCall(when.operand()) || containsFunctionCall(when.result())) || + caseExpression.defaultValue().map(HogQlCompiler::containsFunctionCall).orElse(false); + case CastExpression cast -> containsFunctionCall(cast.value()); + case ColumnReference _, Literal _, Placeholder _ -> false; + case FunctionCall _ -> true; + case InCohortExpression _ -> true; + case InExpression in -> containsFunctionCall(in.value()) || in.values().stream().anyMatch(HogQlCompiler::containsFunctionCall); + case InSubqueryExpression in -> containsFunctionCall(in.value()) || containsFunctionCall(in.query()); + case IntervalExpression interval -> containsFunctionCall(interval.value()); + case IsNullExpression isNull -> containsFunctionCall(isNull.value()); + case LambdaExpression lambda -> containsFunctionCall(lambda.body()); + case MemberAccessExpression memberAccess -> containsFunctionCall(memberAccess.base()); + case ScalarSubqueryExpression subquery -> containsSemanticCandidate(subquery.query()); + case SubscriptExpression subscript -> containsFunctionCall(subscript.base()) || containsFunctionCall(subscript.index()); + case TupleExpression tuple -> tuple.values().stream().anyMatch(HogQlCompiler::containsFunctionCall); + case UnaryExpression unary -> containsFunctionCall(unary.operand()); + }; + } + + private static boolean containsFunctionCall(Window window) + { + return switch (window) { + case WindowReference _ -> false; + case WindowSpecification specification -> specification.partitionBy().stream().anyMatch(HogQlCompiler::containsFunctionCall) || + specification.orderBy().stream().anyMatch(sortItem -> containsFunctionCall(sortItem.expression())) || + specification.frame().map(frame -> containsFunctionCall(frame.start().value()) || + frame.end().map(bound -> containsFunctionCall(bound.value())).orElse(false)).orElse(false); + }; + } + + private static boolean containsFunctionCall(Optional expression) + { + return expression.map(HogQlCompiler::containsFunctionCall).orElse(false); + } + + private record ResolvedQuery(HogQlQuery query, Optional pinnedSnapshot, OptionalLong exchangeRateGeneration) + { + private ResolvedQuery + { + query = requireNonNull(query, "query is null"); + pinnedSnapshot = requireNonNull(pinnedSnapshot, "pinnedSnapshot is null"); + exchangeRateGeneration = requireNonNull(exchangeRateGeneration, "exchangeRateGeneration is null"); + } + + public OptionalLong catalogGeneration() + { + return pinnedSnapshot + .map(snapshot -> OptionalLong.of(snapshot.generation())) + .orElseGet(OptionalLong::empty); + } + } + + private static void validateParameters(HogQlQuery query, Map parameters) + { + for (Map.Entry parameter : parameters.entrySet()) { + if (parameter.getKey() == null || parameter.getKey().isBlank()) { + throw bindingError(query.span(), "HogQL parameter binding name is empty"); + } + if (parameter.getValue() == null) { + throw bindingError(query.span(), "HogQL parameter binding has no typed value: " + parameter.getKey()); + } + } + } + + private static void validateQuery(HogQlQuery query) + { + query.with().forEach(commonTable -> validateQuery(commonTable.query())); + switch (query.body()) { + case SelectQueryBody select -> select.from().ifPresent(HogQlCompiler::validateRelation); + case SetOperation setOperation -> { + validateQuery(setOperation.left()); + validateQuery(setOperation.right()); + } + } + } + + private static void collectPlaceholders(Expression expression, List placeholders) + { + switch (expression) { + case ArrayExpression array -> array.values().forEach(value -> collectPlaceholders(value, placeholders)); + case BetweenExpression between -> { + collectPlaceholders(between.value(), placeholders); + collectPlaceholders(between.min(), placeholders); + collectPlaceholders(between.max(), placeholders); + } + case BinaryExpression binary -> { + collectPlaceholders(binary.left(), placeholders); + collectPlaceholders(binary.right(), placeholders); + } + case CaseExpression caseExpression -> { + caseExpression.operand().ifPresent(operand -> collectPlaceholders(operand, placeholders)); + caseExpression.whenClauses().forEach(when -> { + collectPlaceholders(when.operand(), placeholders); + collectPlaceholders(when.result(), placeholders); + }); + caseExpression.defaultValue().ifPresent(value -> collectPlaceholders(value, placeholders)); + } + case CastExpression cast -> collectPlaceholders(cast.value(), placeholders); + case ColumnReference _ -> {} + case FunctionCall function -> { + function.arguments().forEach(argument -> collectPlaceholders(argument, placeholders)); + function.orderBy().forEach(sortItem -> collectPlaceholders(sortItem.expression(), placeholders)); + function.filter().ifPresent(filter -> collectPlaceholders(filter, placeholders)); + function.window().ifPresent(window -> collectPlaceholders(window, placeholders)); + } + case InCohortExpression in -> { + collectPlaceholders(in.value(), placeholders); + collectPlaceholders(in.cohort(), placeholders); + } + case InExpression in -> { + collectPlaceholders(in.value(), placeholders); + in.values().forEach(value -> collectPlaceholders(value, placeholders)); + } + case InSubqueryExpression in -> { + collectPlaceholders(in.value(), placeholders); + collectPlaceholders(in.query(), placeholders); + } + case IntervalExpression interval -> collectPlaceholders(interval.value(), placeholders); + case IsNullExpression isNull -> collectPlaceholders(isNull.value(), placeholders); + case LambdaExpression lambda -> collectPlaceholders(lambda.body(), placeholders); + case Literal _ -> {} + case MemberAccessExpression memberAccess -> collectPlaceholders(memberAccess.base(), placeholders); + case Placeholder placeholder -> placeholders.add(placeholder); + case ScalarSubqueryExpression subquery -> collectPlaceholders(subquery.query(), placeholders); + case SubscriptExpression subscript -> { + collectPlaceholders(subscript.base(), placeholders); + collectPlaceholders(subscript.index(), placeholders); + } + case TupleExpression tuple -> tuple.values().forEach(value -> collectPlaceholders(value, placeholders)); + case UnaryExpression unary -> collectPlaceholders(unary.operand(), placeholders); + } + } + + private static void collectPlaceholders(Window window, List placeholders) + { + switch (window) { + case WindowReference _ -> {} + case WindowSpecification specification -> { + specification.partitionBy().forEach(expression -> collectPlaceholders(expression, placeholders)); + specification.orderBy().forEach(sortItem -> collectPlaceholders(sortItem.expression(), placeholders)); + specification.frame().ifPresent(frame -> { + frame.start().value().ifPresent(value -> collectPlaceholders(value, placeholders)); + frame.end().flatMap(HogQlQuery.FrameBound::value).ifPresent(value -> collectPlaceholders(value, placeholders)); + }); + } + } + } + + private static void validateRelation(Relation relation) + { + switch (relation) { + case AliasedRelation alias -> validateRelation(alias.relation()); + case CommonTableReference _ -> {} + case JoinRelation join -> { + validateRelation(join.left()); + validateRelation(join.right()); + } + case HogQlQuery.PivotRelation pivot -> validateRelation(pivot.input()); + case SubqueryRelation subquery -> validateQuery(subquery.query()); + case TablePlaceholder tablePlaceholder -> throw bindingError( + tablePlaceholder.span(), + "HogQL parameter placeholders are not supported in table positions: " + tablePlaceholder.placeholder().name()); + case HogQlQuery.TableReference _ -> {} + case UnnestRelation _ -> {} + case ValuesRelation _ -> {} + } + } + + private static void collectPlaceholders(Relation relation, List placeholders) + { + switch (relation) { + case AliasedRelation alias -> collectPlaceholders(alias.relation(), placeholders); + case CommonTableReference _ -> {} + case JoinRelation join -> { + collectPlaceholders(join.left(), placeholders); + collectPlaceholders(join.right(), placeholders); + join.criteria().ifPresent(criteria -> { + if (criteria instanceof JoinOn on) { + collectPlaceholders(on.expression(), placeholders); + } + }); + } + case HogQlQuery.PivotRelation pivot -> { + collectPlaceholders(pivot.input(), placeholders); + pivot.aggregations().forEach(aggregation -> collectPlaceholders(aggregation.expression(), placeholders)); + pivot.pivotColumns().forEach(expression -> collectPlaceholders(expression, placeholders)); + pivot.valueGroups().forEach(group -> group.values().forEach(expression -> collectPlaceholders(expression, placeholders))); + pivot.groupBy().forEach(expression -> collectPlaceholders(expression, placeholders)); + } + case SubqueryRelation subquery -> collectPlaceholders(subquery.query(), placeholders); + case TablePlaceholder _ -> {} + case HogQlQuery.TableReference _ -> {} + case UnnestRelation unnest -> unnest.expressions().forEach(expression -> collectPlaceholders(expression, placeholders)); + case ValuesRelation values -> values.rows().forEach(row -> row.forEach(expression -> collectPlaceholders(expression, placeholders))); + } + } + + private static TrinoException bindingError(SourceSpan span, String message) + { + return new TrinoException( + HOGQL_BINDING_ERROR, + Optional.of(new Location(span.startLine(), span.startColumn())), + message, + null); + } + + private static TrinoException unsupportedError(SourceSpan span, String message) + { + return new TrinoException( + io.trino.hogql.compiler.HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE, + Optional.of(new Location(span.startLine(), span.startColumn())), + message, + null); + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlErrorCode.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlErrorCode.java new file mode 100644 index 000000000000..dd1b9f2d5757 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlErrorCode.java @@ -0,0 +1,50 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.spi.ErrorCode; +import io.trino.spi.ErrorCodeSupplier; +import io.trino.spi.ErrorType; + +import static io.trino.spi.ErrorType.EXTERNAL; +import static io.trino.spi.ErrorType.INTERNAL_ERROR; +import static io.trino.spi.ErrorType.USER_ERROR; + +public enum HogQlErrorCode + implements ErrorCodeSupplier +{ + HOGQL_SYNTAX_ERROR(0, USER_ERROR), + HOGQL_BINDING_ERROR(1, USER_ERROR), + HOGQL_RESOLUTION_ERROR(2, USER_ERROR), + HOGQL_TYPE_ERROR(3, USER_ERROR), + HOGQL_UNSUPPORTED_FEATURE(4, USER_ERROR), + HOGQL_CATALOG_NOT_READY(5, EXTERNAL), + HOGQL_CATALOG_GENERATION_MISMATCH(6, EXTERNAL), + HOGQL_COMPILER_LIMIT_EXCEEDED(7, USER_ERROR), + HOGQL_COMPILER_INTERNAL_ERROR(8, INTERNAL_ERROR), + ; + + private final ErrorCode errorCode; + + HogQlErrorCode(int code, ErrorType type) + { + errorCode = new ErrorCode(code + 0x0521_0000, name(), type); + } + + @Override + public ErrorCode toErrorCode() + { + return errorCode; + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java new file mode 100644 index 000000000000..d337e978257d --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlFunctionResolver.java @@ -0,0 +1,1364 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotProvider; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCapabilityDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionImplementation; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionRewrite; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionSignature; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; +import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.AliasedRelation; +import io.trino.hogql.parser.tree.HogQlQuery.ArrayExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BetweenExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CaseExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CaseWhen; +import io.trino.hogql.parser.tree.HogQlQuery.CastExpression; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsList; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsRegex; +import io.trino.hogql.parser.tree.HogQlQuery.CommonTableExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CommonTableReference; +import io.trino.hogql.parser.tree.HogQlQuery.Expression; +import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; +import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; +import io.trino.hogql.parser.tree.HogQlQuery.Identifier; +import io.trino.hogql.parser.tree.HogQlQuery.InCohortExpression; +import io.trino.hogql.parser.tree.HogQlQuery.InExpression; +import io.trino.hogql.parser.tree.HogQlQuery.InSubqueryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IntervalExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; +import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; +import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; +import io.trino.hogql.parser.tree.HogQlQuery.JoinUsing; +import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.LambdaExpression; +import io.trino.hogql.parser.tree.HogQlQuery.LimitBy; +import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; +import io.trino.hogql.parser.tree.HogQlQuery.PivotAggregation; +import io.trino.hogql.parser.tree.HogQlQuery.PivotRelation; +import io.trino.hogql.parser.tree.HogQlQuery.PivotValueGroup; +import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; +import io.trino.hogql.parser.tree.HogQlQuery.Projection; +import io.trino.hogql.parser.tree.HogQlQuery.Relation; +import io.trino.hogql.parser.tree.HogQlQuery.ScalarSubqueryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.SelectQueryBody; +import io.trino.hogql.parser.tree.HogQlQuery.SetOperation; +import io.trino.hogql.parser.tree.HogQlQuery.SortItem; +import io.trino.hogql.parser.tree.HogQlQuery.Star; +import io.trino.hogql.parser.tree.HogQlQuery.StarReplacement; +import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; +import io.trino.hogql.parser.tree.HogQlQuery.SubscriptExpression; +import io.trino.hogql.parser.tree.HogQlQuery.TablePlaceholder; +import io.trino.hogql.parser.tree.HogQlQuery.TableReference; +import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; +import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.UnnestRelation; +import io.trino.hogql.parser.tree.HogQlQuery.ValuesRelation; +import io.trino.hogql.parser.tree.HogQlQuery.Window; +import io.trino.hogql.parser.tree.HogQlQuery.WindowDefinition; +import io.trino.hogql.parser.tree.HogQlQuery.WindowFrame; +import io.trino.hogql.parser.tree.HogQlQuery.WindowReference; +import io.trino.hogql.parser.tree.HogQlQuery.WindowSpecification; +import io.trino.spi.Location; +import io.trino.spi.TrinoException; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; + +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_RESOLUTION_ERROR; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE; +import static java.util.Objects.requireNonNull; + +final class HogQlFunctionResolver +{ + private static final String MATCHES_ACTION = "matchesaction"; + + private final Map functions; + private final boolean semanticCatalogAvailable; + private final Optional exchangeRateSnapshotProvider; + private OptionalLong exchangeRateGeneration = OptionalLong.empty(); + private ArrayList activeArrayJoins; + + private HogQlFunctionResolver( + Optional snapshot, + Optional exchangeRateSnapshotProvider, + boolean semanticCatalogAvailable) + { + Map functions = new LinkedHashMap<>(HogQlV0FunctionRegistry.functions()); + snapshot.stream() + .flatMap(pinned -> pinned.snapshot().functions().stream()) + .forEach(function -> functions.putIfAbsent(canonical(function.name()), function)); + this.functions = Map.copyOf(functions); + this.semanticCatalogAvailable = semanticCatalogAvailable; + this.exchangeRateSnapshotProvider = requireNonNull(exchangeRateSnapshotProvider, "exchangeRateSnapshotProvider is null"); + } + + public static HogQlQuery resolve(PinnedSnapshot snapshot, HogQlQuery query) + { + requireNonNull(snapshot, "snapshot is null"); + requireNonNull(query, "query is null"); + return resolve(snapshot, query, Optional.empty()).query(); + } + + static Resolution resolve( + PinnedSnapshot snapshot, + HogQlQuery query, + Optional exchangeRateSnapshotProvider) + { + requireNonNull(snapshot, "snapshot is null"); + requireNonNull(query, "query is null"); + return new HogQlFunctionResolver(Optional.of(snapshot), exchangeRateSnapshotProvider, true).resolveWithMetadata(query); + } + + public static HogQlQuery resolve(HogQlQuery query) + { + requireNonNull(query, "query is null"); + return resolve(query, Optional.empty()).query(); + } + + static Resolution resolve(HogQlQuery query, Optional exchangeRateSnapshotProvider) + { + requireNonNull(query, "query is null"); + return new HogQlFunctionResolver(Optional.empty(), exchangeRateSnapshotProvider, false).resolveWithMetadata(query); + } + + public static HogQlQuery resolveV0(HogQlQuery query) + { + requireNonNull(query, "query is null"); + return resolveV0(query, Optional.empty()).query(); + } + + static Resolution resolveV0(HogQlQuery query, Optional exchangeRateSnapshotProvider) + { + return resolveV0(query, exchangeRateSnapshotProvider, false); + } + + static Resolution resolveV0( + HogQlQuery query, + Optional exchangeRateSnapshotProvider, + boolean semanticCatalogAvailable) + { + requireNonNull(query, "query is null"); + return new HogQlFunctionResolver(Optional.empty(), exchangeRateSnapshotProvider, semanticCatalogAvailable).resolveWithMetadata(query); + } + + private Resolution resolveWithMetadata(HogQlQuery query) + { + HogQlQuery resolved = resolveQuery(query); + return new Resolution(resolved, exchangeRateGeneration); + } + + private HogQlQuery resolveQuery(HogQlQuery query) + { + List commonTables = query.with().stream() + .map(commonTable -> new CommonTableExpression( + commonTable.name(), + commonTable.columnAliases(), + resolveQuery(commonTable.query()), + commonTable.span())) + .toList(); + if (query.body() instanceof SelectQueryBody select) { + return resolveSelectQuery(query, commonTables, select); + } + SetOperation set = (SetOperation) query.body(); + return new HogQlQuery( + commonTables, + new SetOperation( + set.type(), + set.distinct(), + resolveQuery(set.left()), + resolveQuery(set.right()), + set.leftParenthesized(), + set.rightParenthesized(), + set.operatorSpan(), + set.span()), + resolveSortItems(query.orderBy()), + query.limit().map(this::resolveExpression), + query.offset().map(this::resolveExpression), + query.span()); + } + + private HogQlQuery resolveSelectQuery(HogQlQuery query, List commonTables, SelectQueryBody select) + { + ArrayList parentArrayJoins = activeArrayJoins; + activeArrayJoins = new ArrayList<>(); + try { + Optional from = select.from().map(this::resolveRelation); + List projections = select.projections().stream().map(this::resolveProjection).toList(); + Optional where = select.where().map(this::resolveExpression); + List groupBy = select.groupBy().stream().map(this::resolveExpression).toList(); + Optional having = select.having().map(this::resolveExpression); + List windows = select.windows().stream().map(this::resolveWindowDefinition).toList(); + Optional limitBy = select.limitBy().map(clause -> new LimitBy( + resolveExpression(clause.limit()), + clause.offset().map(this::resolveExpression), + clause.partitionBy().stream().map(this::resolveExpression).toList(), + clause.span())); + List orderBy = resolveSortItems(query.orderBy()); + Optional limit = query.limit().map(this::resolveExpression); + Optional offset = query.offset().map(this::resolveExpression); + for (UnnestRelation unnest : activeArrayJoins) { + from = Optional.of(from + .map(left -> new JoinRelation(HogQlQuery.JoinType.CROSS, left, unnest, Optional.empty(), unnest.span())) + .orElse(unnest)); + } + return new HogQlQuery( + commonTables, + new SelectQueryBody(select.distinct(), projections, from, where, groupBy, having, windows, limitBy, select.span()), + orderBy, + limit, + offset, + query.span()); + } + finally { + activeArrayJoins = parentArrayJoins; + } + } + + private Projection resolveProjection(Projection projection) + { + return switch (projection) { + case ColumnsList columns -> new ColumnsList(columns.expressions().stream().map(this::resolveExpression).toList(), columns.span()); + case ColumnsRegex columns -> columns; + case ExpressionProjection expression -> new ExpressionProjection(resolveExpression(expression.expression()), expression.alias()); + case Star star -> new Star( + star.qualifier(), + star.exclusions(), + star.replacements().stream() + .map(replacement -> new StarReplacement(resolveExpression(replacement.expression()), replacement.target(), replacement.span())) + .toList(), + star.span()); + }; + } + + private Relation resolveRelation(Relation relation) + { + return switch (relation) { + case AliasedRelation alias -> new AliasedRelation(resolveRelation(alias.relation()), alias.alias(), alias.columnAliases(), alias.span()); + case CommonTableReference commonTable -> commonTable; + case JoinRelation join -> new JoinRelation( + join.type(), + resolveRelation(join.left()), + resolveRelation(join.right()), + join.criteria().map(criteria -> switch (criteria) { + case JoinOn on -> new JoinOn(resolveExpression(on.expression()), on.span()); + case JoinUsing using -> using; + }), + join.span()); + case PivotRelation pivot -> new PivotRelation( + resolveRelation(pivot.input()), + pivot.aggregations().stream() + .map(aggregation -> new PivotAggregation( + resolveExpression(aggregation.expression()), + aggregation.alias(), + aggregation.span())) + .toList(), + pivot.pivotColumns().stream().map(this::resolveExpression).toList(), + pivot.valueGroups().stream() + .map(group -> new PivotValueGroup( + group.values().stream().map(this::resolveExpression).toList(), + group.alias(), + group.span())) + .toList(), + pivot.groupBy().stream().map(this::resolveExpression).toList(), + pivot.span()); + case SubqueryRelation subquery -> new SubqueryRelation(resolveQuery(subquery.query()), subquery.span()); + case TablePlaceholder placeholder -> placeholder; + case TableReference table -> table; + case UnnestRelation unnest -> new UnnestRelation( + unnest.expressions().stream().map(this::resolveExpression).toList(), + unnest.alias(), + unnest.columnAliases(), + unnest.span()); + case ValuesRelation values -> new ValuesRelation( + values.rows().stream() + .map(row -> row.stream().map(this::resolveExpression).toList()) + .toList(), + values.span()); + }; + } + + private List resolveSortItems(List sortItems) + { + return sortItems.stream() + .map(sortItem -> new SortItem(resolveExpression(sortItem.expression()), sortItem.direction(), sortItem.nullPlacement(), sortItem.span())) + .toList(); + } + + private Expression resolveExpression(Expression expression) + { + return switch (expression) { + case ArrayExpression array -> new ArrayExpression(array.values().stream().map(this::resolveExpression).toList(), array.span()); + case BetweenExpression between -> new BetweenExpression( + resolveExpression(between.value()), + resolveExpression(between.min()), + resolveExpression(between.max()), + between.negated(), + between.predicateSpan(), + between.span()); + case BinaryExpression binary -> new BinaryExpression( + binary.operator(), + resolveExpression(binary.left()), + resolveExpression(binary.right()), + binary.span()); + case CaseExpression caseExpression -> new CaseExpression( + caseExpression.operand().map(this::resolveExpression), + caseExpression.whenClauses().stream() + .map(when -> new CaseWhen(resolveExpression(when.operand()), resolveExpression(when.result()), when.span())) + .toList(), + caseExpression.defaultValue().map(this::resolveExpression), + caseExpression.span()); + case CastExpression cast -> new CastExpression(resolveExpression(cast.value()), cast.type(), cast.safe(), cast.typeDialect(), cast.span()); + case ColumnReference reference -> reference; + case FunctionCall function -> resolveFunction(function, function.window().isPresent()); + case InCohortExpression in -> new InCohortExpression( + resolveExpression(in.value()), + resolveExpression(in.cohort()), + in.negated(), + in.predicateSpan(), + in.span()); + case InExpression in -> new InExpression( + resolveExpression(in.value()), + in.values().stream().map(this::resolveExpression).toList(), + in.negated(), + in.predicateSpan(), + in.span()); + case InSubqueryExpression in -> new InSubqueryExpression( + resolveExpression(in.value()), + resolveQuery(in.query()), + in.negated(), + in.predicateSpan(), + in.span()); + case IntervalExpression interval -> new IntervalExpression(resolveExpression(interval.value()), interval.unit(), interval.span()); + case IsNullExpression isNull -> new IsNullExpression( + resolveExpression(isNull.value()), + isNull.negated(), + isNull.predicateSpan(), + isNull.span()); + case LambdaExpression lambda -> new LambdaExpression(lambda.arguments(), resolveExpression(lambda.body()), lambda.span()); + case Literal literal -> literal; + case MemberAccessExpression memberAccess -> new MemberAccessExpression( + resolveExpression(memberAccess.base()), + memberAccess.member(), + memberAccess.span()); + case Placeholder placeholder -> placeholder; + case ScalarSubqueryExpression subquery -> new ScalarSubqueryExpression(resolveQuery(subquery.query()), subquery.span()); + case SubscriptExpression subscript -> new SubscriptExpression( + resolveExpression(subscript.base()), + resolveExpression(subscript.index()), + subscript.span()); + case TupleExpression tuple -> new TupleExpression(tuple.values().stream().map(this::resolveExpression).toList(), tuple.span()); + case UnaryExpression unary -> new UnaryExpression(unary.operator(), resolveExpression(unary.operand()), unary.span()); + }; + } + + private Expression resolveFunction(FunctionCall function, boolean windowInvocation) + { + String name = function.name().value(); + if (function.nameParts().size() != 1) { + throw unsupportedError(function, "Qualified HogQL functions are not supported"); + } + if (canonical(name).equals("arrayjoin")) { + return resolveArrayJoin(function); + } + if (semanticCatalogAvailable && function.nameParts().size() == 1 && canonical(name).equals(MATCHES_ACTION)) { + return new FunctionCall( + function.nameParts(), + function.arguments().stream().map(this::resolveExpression).toList(), + function.distinct(), + resolveSortItems(function.orderBy()), + function.filter().map(this::resolveExpression), + function.nullTreatment(), + function.window().map(this::resolveWindow), + function.span()); + } + FunctionCapabilityDefinition capability = functions.get(canonical(name)); + if (capability == null) { + throw resolutionError(function, "Unknown HogQL function: " + name); + } + if (capability.implementation() == FunctionImplementation.REWRITE && function.nullTreatment().isPresent()) { + throw unsupportedError(function, "HogQL function " + name + " does not support null treatment"); + } + if (capability.kind() == FunctionKind.TABLE) { + throw unsupportedError(function, "HogQL table function " + name + " cannot be used as an expression"); + } + if (windowInvocation && !capability.supportsWindow()) { + throw unsupportedError(function, "HogQL function " + name + " does not support OVER"); + } + if (!windowInvocation && capability.kind() == FunctionKind.WINDOW) { + throw unsupportedError(function, "HogQL window function " + name + " requires an OVER clause"); + } + if (!matchesArity(capability.signatures(), function.arguments().size())) { + throw resolutionError(function, "HogQL function " + name + " does not accept " + function.arguments().size() + " arguments"); + } + if (function.distinct() && !capability.supportsDistinct()) { + throw unsupportedError(function, "HogQL function " + name + " does not support DISTINCT"); + } + if (!function.orderBy().isEmpty() && !capability.supportsOrderBy()) { + throw unsupportedError(function, "HogQL function " + name + " does not support ORDER BY"); + } + if (function.filter().isPresent() && !capability.supportsFilter()) { + throw unsupportedError(function, "HogQL function " + name + " does not support FILTER"); + } + List arguments = function.arguments().stream().map(this::resolveExpression).toList(); + if (capability.implementation() == FunctionImplementation.REWRITE) { + FunctionRewrite rewrite = capability.rewrite() + .orElseThrow(() -> unsupportedError(function, "HogQL function " + name + " has no compiler rewrite")); + return rewrite(function, rewrite, arguments); + } + return new FunctionCall( + capability.trinoName().stream() + .map(identifier -> identifier(identifier, function.span())) + .toList(), + arguments, + function.distinct(), + resolveSortItems(function.orderBy()), + function.filter().map(this::resolveExpression), + function.nullTreatment(), + function.window().map(this::resolveWindow), + function.span()); + } + + private Expression resolveArrayJoin(FunctionCall function) + { + if (function.arguments().size() != 1) { + throw resolutionError(function, "HogQL function " + function.name().value() + " does not accept " + function.arguments().size() + " arguments"); + } + if (function.distinct() || !function.orderBy().isEmpty() || function.filter().isPresent() || + function.nullTreatment().isPresent() || function.window().isPresent()) { + throw unsupportedError(function, "HogQL arrayJoin does not accept function modifiers"); + } + if (activeArrayJoins == null) { + throw unsupportedError(function, "HogQL arrayJoin requires a SELECT query context"); + } + int index = activeArrayJoins.size(); + Identifier alias = new Identifier("__hogql_array_join_" + index, false, function.span()); + Identifier column = new Identifier("__hogql_value_" + index, false, function.span()); + activeArrayJoins.add(new UnnestRelation( + List.of(resolveExpression(function.arguments().getFirst())), + alias, + List.of(column), + function.span())); + return new ColumnReference(List.of(alias, column), function.span()); + } + + private Expression rewrite(FunctionCall function, FunctionRewrite rewrite, List arguments) + { + HogQlQuery.SourceSpan span = function.span(); + return switch (rewrite) { + case IS_NULL -> new IsNullExpression(arguments.getFirst(), false, span, span); + case IS_NOT_NULL -> new IsNullExpression(arguments.getFirst(), true, span, span); + case CAST_DATE -> cast(arguments.getFirst(), "date", span); + case CAST_DOUBLE -> cast(arguments.getFirst(), "double", span); + case CAST_SMALLINT -> cast(arguments.getFirst(), "smallint", span); + case FLOAT_OR_ZERO -> coalesce(tryCast(arguments.getFirst(), "double", span), new Literal(HogQlQuery.LiteralKind.FLOAT, "0.0", span), span); + case FLOAT_OR_DEFAULT -> coalesce(tryCast(arguments.getFirst(), "double", span), cast(arguments.get(1), "double", span), span); + case DECIMAL_CAST -> tryCast(arguments.getFirst(), decimalType(function, arguments.get(1)), span); + case INT_DIV -> intDiv(arguments, span); + case ARRAY_ELEMENT -> call("element_at", arguments, span); + case ARRAY_FILTER -> call("filter", List.of(arguments.get(1), arguments.getFirst()), span); + case ARRAY_FIRST -> call( + "element_at", + List.of(call("filter", List.of(arguments.get(1), arguments.getFirst()), span), integerLiteral("1", span)), + span); + case ARRAY_MAP -> call("transform", List.of(arguments.get(1), arguments.getFirst()), span); + case ARRAY_SUM -> arraySum(arguments.getFirst(), span); + case RANGE -> range(arguments, span); + case TUPLE_ELEMENT -> new SubscriptExpression(arguments.getFirst(), arguments.get(1), span); + case SPLIT_CHAR -> call("split", List.of(arguments.get(1), arguments.getFirst()), span); + case HAS -> has(arguments, span); + case ASSUME_NOT_NULL -> arguments.getFirst(); + case EMPTY -> empty(arguments.getFirst(), true, span); + case NOT_EMPTY -> empty(arguments.getFirst(), false, span); + case EQUALS -> new BinaryExpression(HogQlQuery.BinaryOperator.EQUAL, arguments.getFirst(), arguments.get(1), span); + case PLUS -> new BinaryExpression(HogQlQuery.BinaryOperator.ADD, arguments.getFirst(), arguments.get(1), span); + case MINUS -> new BinaryExpression(HogQlQuery.BinaryOperator.SUBTRACT, arguments.getFirst(), arguments.get(1), span); + case NOT_EQUALS -> new BinaryExpression(HogQlQuery.BinaryOperator.NOT_EQUAL, arguments.getFirst(), arguments.get(1), span); + case MULTIPLY, MULTIPLY_DECIMAL -> new BinaryExpression(HogQlQuery.BinaryOperator.MULTIPLY, arguments.getFirst(), arguments.get(1), span); + case DIVIDE_DECIMAL -> new BinaryExpression(HogQlQuery.BinaryOperator.DIVIDE, arguments.getFirst(), arguments.get(1), span); + case IN_ARRAY -> call("contains", List.of(arguments.get(1), arguments.getFirst()), span); + case MAP_CONSTRUCTOR -> map(function, arguments); + case TUPLE -> new TupleExpression(arguments, span); + case SUBTRACT_MONTHS -> dateAdd( + "month", + List.of(arguments.getFirst(), new UnaryExpression(HogQlQuery.UnaryOperator.NEGATE, arguments.get(1), span)), + span); + case SUBTRACT_DAYS -> dateAdd( + "day", + List.of(arguments.getFirst(), new UnaryExpression(HogQlQuery.UnaryOperator.NEGATE, arguments.get(1), span)), + span); + case INTERVAL_MONTH -> new IntervalExpression(arguments.getFirst(), HogQlQuery.IntervalUnit.MONTH, span); + case START_WEEK -> startOfWeek(function, arguments); + case SPLIT_STRING -> call("split", List.of(arguments.get(1), arguments.getFirst()), span); + case CAST_BIGINT -> castBigint(arguments.getFirst(), span); + case CAST_TIMESTAMP -> arguments.size() == 1 + ? cast(arguments.getFirst(), "timestamp(0)", span) + : call("with_timezone", List.of(cast(arguments.getFirst(), "timestamp(0)", span), arguments.get(1)), span); + case CAST_VARCHAR -> cast(arguments.getFirst(), "varchar", span); + case DATE_TRUNC_DAY -> dateTrunc("day", arguments.getFirst(), span); + case DATE_TRUNC_HOUR -> dateTrunc("hour", arguments.getFirst(), span); + case DATE_TRUNC_MONTH -> dateTrunc("month", arguments.getFirst(), span); + case DATE_TRUNC_WEEK -> dateTrunc("week", arguments.getFirst(), span); + case COUNT_IF -> aggregate(function, "count", List.of(), false, arguments.getFirst(), span); + case SUM_IF -> aggregate(function, "sum", List.of(arguments.getFirst()), false, arguments.get(1), span); + case MAX_IF -> aggregate(function, "max", List.of(arguments.getFirst()), false, arguments.get(1), span); + case UNIQ_IF -> aggregate(function, "approx_distinct", List.of(arguments.getFirst()), false, arguments.get(1), span); + case UNIQ_EXACT -> aggregate(function, "count", List.of(arguments.getFirst()), true, null, span); + case GROUP_UNIQ_ARRAY -> aggregate(function, "array_agg", List.of(arguments.getFirst()), true, null, span); + case ARG_MAX_IF -> aggregate(function, "max_by", List.of(arguments.getFirst(), arguments.get(1)), false, arguments.get(2), span); + case ARG_MIN_IF -> aggregate(function, "min_by", List.of(arguments.getFirst(), arguments.get(1)), false, arguments.get(2), span); + case ANY_IF -> aggregate(function, "arbitrary", List.of(arguments.getFirst()), false, arguments.get(1), span); + case MIN_IF -> aggregate(function, "min", List.of(arguments.getFirst()), false, arguments.get(1), span); + case AVG_IF -> aggregate(function, "avg", List.of(arguments.getFirst()), false, arguments.get(1), span); + case GROUP_ARRAY_IF -> arguments.size() == 2 + ? aggregate(function, "array_agg", List.of(arguments.getFirst()), false, arguments.get(1), span) + : call( + "slice", + List.of( + aggregate(function, "array_agg", List.of(arguments.getFirst()), false, arguments.get(1), span), + integerLiteral("1", span), + arguments.get(2)), + span); + case UNIQ_EXACT_IF -> aggregate(function, "count", List.of(arguments.getFirst()), true, arguments.get(1), span); + case GROUP_UNIQ_ARRAY_IF -> aggregate(function, "array_agg", List.of(arguments.getFirst()), true, arguments.get(1), span); + case COUNT_DISTINCT -> aggregate(function, "count", List.of(arguments.getFirst()), true, null, span); + case CONVERT_CURRENCY -> convertCurrency(function, arguments); + case MULTI_IF -> multiIf(function, arguments); + case JSON_EXTRACT_STRING -> jsonExtractScalar(function, arguments, "varchar", new Literal(HogQlQuery.LiteralKind.STRING, "", span)); + case JSON_EXTRACT_INT -> jsonExtractScalar(function, arguments, "bigint", new Literal(HogQlQuery.LiteralKind.INTEGER, "0", span)); + case JSON_EXTRACT_FLOAT -> jsonExtractScalar(function, arguments, "double", new Literal(HogQlQuery.LiteralKind.FLOAT, "0.0", span)); + case JSON_EXTRACT_BOOL -> jsonExtractScalar(function, arguments, "boolean", new Literal(HogQlQuery.LiteralKind.BOOLEAN, "false", span)); + case JSON_EXTRACT_UINT -> jsonExtractScalar(function, arguments, "bigint", integerLiteral("0", span)); + case JSON_EXTRACT_ARRAY_RAW -> jsonExtractArrayRaw(function, arguments); + case JSON_EXTRACT_KEYS -> call( + "map_keys", + List.of(typedJsonMapFromJson(jsonValue(function, arguments), "json", span)), + span); + case JSON_EXTRACT_RAW -> coalesce( + call("json_format", List.of(call("json_extract", List.of(arguments.getFirst(), jsonPath(function, arguments.subList(1, arguments.size()))), span)), span), + new Literal(HogQlQuery.LiteralKind.STRING, "", span), + span); + case JSON_EXTRACT_TYPED -> jsonExtractTyped(function, arguments); + case JSON_KEYS_AND_VALUES -> jsonExtractKeysAndValues(function, arguments); + case JSON_KEYS_AND_VALUES_RAW -> jsonExtractKeysAndValuesRaw(function, arguments); + case JSON_LENGTH -> coalesce( + call("json_size", List.of(arguments.getFirst(), jsonPath(function, arguments.subList(1, arguments.size()))), span), + new Literal(HogQlQuery.LiteralKind.INTEGER, "0", span), + span); + case TODAY -> cast(call("now", List.of(), span), "date", span); + case INTERVAL_DAY -> new IntervalExpression(arguments.getFirst(), HogQlQuery.IntervalUnit.DAY, span); + case ADD_DAYS -> dateAdd("day", arguments, span); + case ADD_MONTHS -> dateAdd("month", arguments, span); + case DATE_ADD -> arguments.size() == 2 + ? new BinaryExpression(HogQlQuery.BinaryOperator.ADD, arguments.getFirst(), arguments.get(1), span) + : call("date_add", arguments, span); + case TO_UNIX_TIMESTAMP -> cast(call("to_unixtime", arguments, span), "bigint", span); + case PARSE_TIMESTAMP -> new CastExpression(arguments.getFirst(), new Identifier("timestamp(3)", false, span), true, span); + case NOT -> new UnaryExpression(HogQlQuery.UnaryOperator.NOT, arguments.getFirst(), span); + case AND -> logical(arguments, HogQlQuery.BinaryOperator.AND, span); + case OR -> logical(arguments, HogQlQuery.BinaryOperator.OR, span); + case GREATER -> new BinaryExpression(HogQlQuery.BinaryOperator.GREATER_THAN, arguments.getFirst(), arguments.get(1), span); + case GREATER_OR_EQUAL -> new BinaryExpression(HogQlQuery.BinaryOperator.GREATER_THAN_OR_EQUAL, arguments.getFirst(), arguments.get(1), span); + case LESS_OR_EQUAL -> new BinaryExpression(HogQlQuery.BinaryOperator.LESS_THAN_OR_EQUAL, arguments.getFirst(), arguments.get(1), span); + case LIKE -> new BinaryExpression(HogQlQuery.BinaryOperator.LIKE, arguments.getFirst(), arguments.get(1), span); + case REGEX_EXTRACT -> regexExtract(function, arguments); + case REGEX_EXTRACT_ALL -> regexExtractAll(function, arguments); + case REGEX_REPLACE_ALL -> regexReplaceAll(function, arguments); + case REGEX_REPLACE_ONE -> regexReplaceOne(function, arguments); + case ARRAY_SLICE -> call("slice", arguments, span); + case ARRAY_SORT -> arraySort(function, arguments); + case ARRAY_ENUMERATE -> arrayEnumerate(arguments.getFirst(), span); + case SUBTRACT_YEARS -> dateAdd( + "year", + List.of(arguments.getFirst(), new UnaryExpression(HogQlQuery.UnaryOperator.NEGATE, arguments.get(1), span)), + span); + case INT_OR_ZERO -> coalesce(tryCast(arguments.getFirst(), "bigint", span), integerLiteral("0", span), span); + case CAST_UUID -> tryCast(arguments.getFirst(), "uuid", span); + case TO_JSON_STRING -> call("json_format", List.of(cast(arguments.getFirst(), "json", span)), span); + case JSON_HAS -> new IsNullExpression( + call("json_extract", List.of(arguments.getFirst(), jsonPath(function, arguments.subList(1, arguments.size()))), span), + true, + span, + span); + case JSON_VALUE -> call("json_extract_scalar", arguments, span); + case SURVEY_RESPONSE -> surveyResponse(function, arguments); + case MD5 -> call("md5", List.of(call("to_utf8", List.of(cast(arguments.getFirst(), "varchar", span)), span)), span); + case MEDIAN_IF -> aggregate( + function, + "approx_percentile", + List.of(arguments.getFirst(), new Literal(HogQlQuery.LiteralKind.FLOAT, "0.5", span)), + false, + arguments.get(1), + span); + case QUANTILE, QUANTILE_EXACT -> aggregate(function, "approx_percentile", arguments, false, null, span); + case QUANTILE_IF -> aggregate( + function, + "approx_percentile", + List.of(arguments.getFirst(), arguments.get(2)), + false, + arguments.get(1), + span); + case DATE_PART -> datePart(function, arguments); + }; + } + + private static Expression datePart(FunctionCall function, List arguments) + { + Literal unit = stringLiteral(function, arguments.getFirst(), "date part"); + String trinoFunction = switch (unit.value().toLowerCase(Locale.ENGLISH)) { + case "year" -> "year"; + case "quarter" -> "quarter"; + case "month" -> "month"; + case "week" -> "week"; + case "day" -> "day"; + case "dow", "dayofweek" -> "day_of_week"; + case "doy", "dayofyear" -> "day_of_year"; + case "hour" -> "hour"; + case "minute" -> "minute"; + case "second" -> "second"; + default -> throw unsupportedError(function, "Unsupported HogQL date part: " + unit.value()); + }; + return call(trinoFunction, List.of(arguments.get(1)), function.span()); + } + + private static Expression logical(List arguments, HogQlQuery.BinaryOperator operator, HogQlQuery.SourceSpan span) + { + Expression result = arguments.getFirst(); + for (int index = 1; index < arguments.size(); index++) { + result = new BinaryExpression(operator, result, arguments.get(index), span); + } + return result; + } + + private static Expression regexExtract(FunctionCall function, List arguments) + { + Literal pattern = stringLiteral(function, arguments.get(1), "regular expression"); + Literal group = new Literal( + HogQlQuery.LiteralKind.INTEGER, + hasCapturingGroup(pattern.value()) ? "1" : "0", + function.span()); + Expression extracted = call("regexp_extract", List.of(arguments.getFirst(), pattern, group), function.span()); + return coalesce(extracted, new Literal(HogQlQuery.LiteralKind.STRING, "", function.span()), function.span()); + } + + private static Expression regexExtractAll(FunctionCall function, List arguments) + { + Literal pattern = stringLiteral(function, arguments.get(1), "regular expression"); + Literal group = integerLiteral(hasCapturingGroup(pattern.value()) ? "1" : "0", function.span()); + return call("regexp_extract_all", List.of(arguments.getFirst(), pattern, group), function.span()); + } + + private static Expression regexReplaceAll(FunctionCall function, List arguments) + { + Literal pattern = stringLiteral(function, arguments.get(1), "regular expression"); + Literal replacement = stringLiteral(function, arguments.get(2), "regular expression replacement"); + String trinoReplacement = regexReplacement(replacement.value(), 0); + return call( + "regexp_replace", + List.of(arguments.getFirst(), pattern, new Literal(HogQlQuery.LiteralKind.STRING, trinoReplacement, replacement.span())), + function.span()); + } + + private static Expression regexReplaceOne(FunctionCall function, List arguments) + { + Literal pattern = stringLiteral(function, arguments.get(1), "regular expression"); + Literal replacement = stringLiteral(function, arguments.get(2), "regular expression replacement"); + Literal firstPattern = new Literal( + HogQlQuery.LiteralKind.STRING, + "(?s)^(.*?)(" + pattern.value() + ")", + pattern.span()); + Literal firstReplacement = new Literal( + HogQlQuery.LiteralKind.STRING, + "$1" + regexReplacement(replacement.value(), 2), + replacement.span()); + return call("regexp_replace", List.of(arguments.getFirst(), firstPattern, firstReplacement), function.span()); + } + + private static String regexReplacement(String replacement, int groupOffset) + { + StringBuilder result = new StringBuilder(replacement.length()); + for (int index = 0; index < replacement.length(); index++) { + char current = replacement.charAt(index); + if (current == '\\' && index + 1 < replacement.length() && Character.isDigit(replacement.charAt(index + 1))) { + result.append('$').append(Character.digit(replacement.charAt(++index), 10) + groupOffset); + } + else { + result.append(current); + } + } + return result.toString(); + } + + private static Literal stringLiteral(FunctionCall function, Expression expression, String description) + { + if (expression instanceof Literal literal && literal.kind() == HogQlQuery.LiteralKind.STRING) { + return literal; + } + throw unsupportedError(function, "HogQL " + description + " must be a string literal"); + } + + private static boolean hasCapturingGroup(String pattern) + { + boolean escaped = false; + boolean characterClass = false; + for (int index = 0; index < pattern.length(); index++) { + char current = pattern.charAt(index); + if (escaped) { + escaped = false; + continue; + } + if (current == '\\') { + escaped = true; + continue; + } + if (current == '[') { + characterClass = true; + continue; + } + if (current == ']' && characterClass) { + characterClass = false; + continue; + } + if (current != '(' || characterClass) { + continue; + } + if (index + 1 >= pattern.length() || pattern.charAt(index + 1) != '?') { + return true; + } + if (index + 2 < pattern.length() && pattern.charAt(index + 2) == '<' && + (index + 3 >= pattern.length() || (pattern.charAt(index + 3) != '=' && pattern.charAt(index + 3) != '!'))) { + return true; + } + } + return false; + } + + private static FunctionCall dateAdd(String unit, List arguments, HogQlQuery.SourceSpan span) + { + return call( + "date_add", + List.of(new Literal(HogQlQuery.LiteralKind.STRING, unit, span), arguments.get(1), arguments.getFirst()), + span); + } + + private FunctionCall aggregate(FunctionCall source, String name, List arguments, boolean distinct, Expression filter, HogQlQuery.SourceSpan span) + { + return new FunctionCall( + List.of(new Identifier(name, false, span)), + arguments, + distinct, + List.of(), + Optional.ofNullable(filter), + Optional.empty(), + source.window().map(this::resolveWindow), + span); + } + + private static CaseExpression multiIf(FunctionCall function, List arguments) + { + if (arguments.size() % 2 == 0) { + throw resolutionError(function, "HogQL function " + function.name().value() + " requires condition/result pairs followed by a default value"); + } + List clauses = new ArrayList<>((arguments.size() - 1) / 2); + for (int index = 0; index < arguments.size() - 1; index += 2) { + clauses.add(new CaseWhen(arguments.get(index), arguments.get(index + 1), function.span())); + } + return new CaseExpression(Optional.empty(), clauses, Optional.of(arguments.getLast()), function.span()); + } + + private static Expression jsonExtractScalar(FunctionCall function, List arguments, String type, Literal defaultValue) + { + HogQlQuery.SourceSpan span = function.span(); + Expression extracted = call( + "json_extract_scalar", + List.of(arguments.getFirst(), jsonPath(function, arguments.subList(1, arguments.size()))), + span); + if (!type.equals("varchar")) { + extracted = new CastExpression(extracted, new Identifier(type, false, span), true, span); + } + return coalesce(extracted, defaultValue, span); + } + + private static Expression surveyResponse(FunctionCall function, List arguments) + { + HogQlQuery.SourceSpan span = function.span(); + int questionIndex = surveyQuestionIndex(function, arguments.getFirst()); + if (!(arguments.get(1) instanceof Literal questionId) || questionId.kind() != HogQlQuery.LiteralKind.STRING || questionId.value().isEmpty()) { + throw unsupportedError(function, "HogQL getSurveyResponse question ID must be a non-empty string literal"); + } + Expression properties = new ColumnReference(List.of(new Identifier("properties", false, span)), span); + Literal empty = new Literal(HogQlQuery.LiteralKind.STRING, "", span); + Expression idResponse = call( + "nullif", + List.of(jsonExtractScalar(function, List.of(properties, stringLiteral("$survey_response_" + questionId.value(), span)), "varchar", empty), empty), + span); + String indexKey = questionIndex == 0 ? "$survey_response" : "$survey_response_" + questionIndex; + Expression indexResponse = call( + "nullif", + List.of(jsonExtractScalar(function, List.of(properties, stringLiteral(indexKey, span)), "varchar", empty), empty), + span); + return call("coalesce", List.of(idResponse, indexResponse), span); + } + + private static int surveyQuestionIndex(FunctionCall function, Expression expression) + { + if (!(expression instanceof Literal literal) || + (literal.kind() != HogQlQuery.LiteralKind.INTEGER && literal.kind() != HogQlQuery.LiteralKind.STRING)) { + throw unsupportedError(function, "HogQL getSurveyResponse question index must be an integer literal"); + } + try { + return Integer.parseInt(literal.value()); + } + catch (NumberFormatException _) { + throw resolutionError(function, "HogQL getSurveyResponse question index is outside the supported range"); + } + } + + private static Expression jsonExtractTyped(FunctionCall function, List arguments) + { + String type = jsonType(function, arguments.get(1)); + return switch (type) { + case "Map(String, Float64)" -> typedJsonMap(arguments.getFirst(), "double", function.span()); + default -> throw unsupportedError(function, "Unsupported HogQL JSON extraction type: " + type); + }; + } + + private static Expression jsonExtractKeysAndValues(FunctionCall function, List arguments) + { + String type = jsonType(function, arguments.get(1)); + if (!type.equals("Float64")) { + throw unsupportedError(function, "Unsupported HogQL JSON key/value type: " + type); + } + return call("map_entries", List.of(typedJsonMap(arguments.getFirst(), "double", function.span())), function.span()); + } + + private static Expression jsonExtractKeysAndValuesRaw(FunctionCall function, List arguments) + { + HogQlQuery.SourceSpan span = function.span(); + Identifier key = new Identifier("key", false, span); + Identifier item = new Identifier("value", false, span); + LambdaExpression formatValue = new LambdaExpression( + List.of(key, item), + call("json_format", List.of(new ColumnReference(List.of(item), span)), span), + span); + Expression formatted = call( + "transform_values", + List.of(typedJsonMapFromJson(jsonValue(function, arguments), "json", span), formatValue), + span); + return call("map_entries", List.of(formatted), span); + } + + private static Expression jsonExtractArrayRaw(FunctionCall function, List arguments) + { + HogQlQuery.SourceSpan span = function.span(); + String type = "array(json)"; + Expression converted = tryCast(jsonValue(function, arguments), type, span); + Expression empty = cast(new ArrayExpression(List.of(), span), type, span); + Expression values = coalesce(converted, empty, span); + Identifier item = new Identifier("_hogql_json_item", false, span); + LambdaExpression format = new LambdaExpression( + List.of(item), + call("json_format", List.of(new ColumnReference(List.of(item), span)), span), + span); + return call("transform", List.of(values, format), span); + } + + private static Expression typedJsonMap(Expression value, String valueType, HogQlQuery.SourceSpan span) + { + return typedJsonMapFromJson(call("json_parse", List.of(value), span), valueType, span); + } + + private static Expression typedJsonMapFromJson(Expression parsed, String valueType, HogQlQuery.SourceSpan span) + { + String type = "map(varchar," + valueType + ")"; + Expression converted = new CastExpression(parsed, new Identifier(type, false, span), true, span); + Expression empty = new CastExpression( + call("map", List.of(new ArrayExpression(List.of(), span), new ArrayExpression(List.of(), span)), span), + new Identifier(type, false, span), + false, + span); + return coalesce(converted, empty, span); + } + + private static Expression jsonValue(FunctionCall function, List arguments) + { + if (arguments.size() == 1) { + return call("json_parse", List.of(arguments.getFirst()), function.span()); + } + return call( + "json_extract", + List.of(arguments.getFirst(), jsonPath(function, arguments.subList(1, arguments.size()))), + function.span()); + } + + private static String jsonType(FunctionCall function, Expression expression) + { + if (expression instanceof Literal literal && literal.kind() == HogQlQuery.LiteralKind.STRING) { + return literal.value(); + } + throw unsupportedError(function, "HogQL JSON extraction type must be a string literal"); + } + + private static Expression jsonPath(FunctionCall function, List segments) + { + StringBuilder path = new StringBuilder("$"); + List pathParts = new ArrayList<>(); + for (Expression segment : segments) { + if (!(segment instanceof Literal literal)) { + pathParts.add(stringLiteral(path.toString(), function.span())); + path.setLength(0); + pathParts.add(stringLiteral("[", function.span())); + pathParts.add(call("json_format", List.of(cast(segment, "json", function.span())), function.span())); + pathParts.add(stringLiteral("]", function.span())); + continue; + } + switch (literal.kind()) { + case STRING -> path.append("[\"") + .append(literal.value().replace("\\", "\\\\").replace("\"", "\\\"")) + .append("\"]"); + case INTEGER -> path.append('[').append(literal.value()).append(']'); + default -> throw unsupportedError(function, "HogQL JSON path segments must be string or integer literals"); + } + } + if (pathParts.isEmpty()) { + return stringLiteral(path.toString(), function.span()); + } + if (!path.isEmpty()) { + pathParts.add(stringLiteral(path.toString(), function.span())); + } + return call("concat", pathParts, function.span()); + } + + private static Expression map(FunctionCall function, List arguments) + { + if (arguments.isEmpty()) { + return call("map", List.of(), function.span()); + } + if (arguments.getFirst() instanceof LambdaExpression) { + return call("map", arguments, function.span()); + } + if (arguments.size() % 2 != 0) { + throw resolutionError(function, "HogQL function map requires key/value argument pairs"); + } + List keys = new ArrayList<>(arguments.size() / 2); + List values = new ArrayList<>(arguments.size() / 2); + for (int index = 0; index < arguments.size(); index += 2) { + keys.add(arguments.get(index)); + values.add(arguments.get(index + 1)); + } + return call( + "map", + List.of(new ArrayExpression(keys, function.span()), new ArrayExpression(values, function.span())), + function.span()); + } + + private static FunctionCall coalesce(Expression value, Expression defaultValue, HogQlQuery.SourceSpan span) + { + return call("coalesce", List.of(value, defaultValue), span); + } + + private Expression convertCurrency(FunctionCall function, List arguments) + { + if (exchangeRateGeneration.isEmpty()) { + HogQlExchangeRateSnapshotProvider provider = exchangeRateSnapshotProvider + .orElseThrow(() -> resolutionError(function, "HogQL function convertCurrency requires an exchange-rate snapshot")); + exchangeRateGeneration = OptionalLong.of(provider.pin(OptionalLong.empty()).generation()); + } + + HogQlQuery.SourceSpan span = function.span(); + Expression effectiveDate = cast( + arguments.size() == 4 ? arguments.get(3) : call("now", List.of(), span), + "date", + span); + return call( + "hogql_convert_currency", + List.of( + integerLiteral(Long.toString(exchangeRateGeneration.orElseThrow()), span), + cast(arguments.get(0), "varchar", span), + cast(arguments.get(1), "varchar", span), + cast(arguments.get(2), "decimal(38,10)", span), + effectiveDate), + span); + } + + private static FunctionCall call(String name, List arguments, HogQlQuery.SourceSpan span) + { + return new FunctionCall( + new Identifier(name, false, span), + arguments, + false, + List.of(), + Optional.empty(), + span); + } + + private static Expression castBigint(Expression value, HogQlQuery.SourceSpan span) + { + if (value instanceof CastExpression cast && canonical(cast.type().value()).equals("date")) { + return call( + "date_diff", + List.of( + stringLiteral("day", span), + cast(stringLiteral("1970-01-01", span), "date", span), + value), + span); + } + return cast(value, "bigint", span); + } + + private static CastExpression cast(Expression value, String type, HogQlQuery.SourceSpan span) + { + return new CastExpression(value, new Identifier(type, false, span), false, span); + } + + private static CastExpression tryCast(Expression value, String type, HogQlQuery.SourceSpan span) + { + return new CastExpression(value, new Identifier(type, false, span), true, span); + } + + private static String decimalType(FunctionCall function, Expression scaleExpression) + { + if (!(scaleExpression instanceof Literal literal) || literal.kind() != HogQlQuery.LiteralKind.INTEGER) { + throw unsupportedError(function, "HogQL decimal scale must be an integer literal"); + } + int scale; + try { + scale = Integer.parseInt(literal.value()); + } + catch (NumberFormatException _) { + throw resolutionError(function, "HogQL decimal scale is outside the supported range"); + } + if (scale < 0 || scale > 18) { + throw resolutionError(function, "HogQL Decimal64 scale must be between 0 and 18"); + } + return "decimal(18," + scale + ")"; + } + + private static Expression intDiv(List arguments, HogQlQuery.SourceSpan span) + { + Expression dividend = cast(arguments.getFirst(), "bigint", span); + Expression divisor = cast(arguments.get(1), "bigint", span); + Literal zero = new Literal(HogQlQuery.LiteralKind.INTEGER, "0", span); + Literal one = new Literal(HogQlQuery.LiteralKind.INTEGER, "1", span); + Expression hasRemainder = new BinaryExpression( + HogQlQuery.BinaryOperator.NOT_EQUAL, + new BinaryExpression(HogQlQuery.BinaryOperator.MODULO, dividend, divisor, span), + zero, + span); + Expression signsDiffer = new BinaryExpression( + HogQlQuery.BinaryOperator.OR, + new BinaryExpression( + HogQlQuery.BinaryOperator.AND, + new BinaryExpression(HogQlQuery.BinaryOperator.LESS_THAN, dividend, zero, span), + new BinaryExpression(HogQlQuery.BinaryOperator.GREATER_THAN, divisor, zero, span), + span), + new BinaryExpression( + HogQlQuery.BinaryOperator.AND, + new BinaryExpression(HogQlQuery.BinaryOperator.GREATER_THAN, dividend, zero, span), + new BinaryExpression(HogQlQuery.BinaryOperator.LESS_THAN, divisor, zero, span), + span), + span); + Expression adjustment = call( + "if", + List.of(new BinaryExpression(HogQlQuery.BinaryOperator.AND, hasRemainder, signsDiffer, span), one, zero), + span); + return new BinaryExpression( + HogQlQuery.BinaryOperator.SUBTRACT, + new BinaryExpression(HogQlQuery.BinaryOperator.DIVIDE, dividend, divisor, span), + adjustment, + span); + } + + private static Expression arraySum(Expression array, HogQlQuery.SourceSpan span) + { + Identifier sum = new Identifier("_hogql_sum", false, span); + Identifier item = new Identifier("_hogql_item", false, span); + Expression sumReference = new ColumnReference(List.of(sum), span); + LambdaExpression add = new LambdaExpression( + List.of(sum, item), + new BinaryExpression( + HogQlQuery.BinaryOperator.ADD, + sumReference, + new ColumnReference(List.of(item), span), + span), + span); + LambdaExpression finish = new LambdaExpression(List.of(sum), sumReference, span); + return call("reduce", List.of(array, integerLiteral("0", span), add, finish), span); + } + + private static Literal integerLiteral(String value, HogQlQuery.SourceSpan span) + { + return new Literal(HogQlQuery.LiteralKind.INTEGER, value, span); + } + + private static Literal stringLiteral(String value, HogQlQuery.SourceSpan span) + { + return new Literal(HogQlQuery.LiteralKind.STRING, value, span); + } + + private static Expression range(List arguments, HogQlQuery.SourceSpan span) + { + Expression start = arguments.size() == 1 ? integerLiteral("0", span) : arguments.getFirst(); + Expression end = arguments.getLast(); + Expression isEmpty = new BinaryExpression(HogQlQuery.BinaryOperator.LESS_THAN_OR_EQUAL, end, start, span); + Expression empty = cast(new ArrayExpression(List.of(), span), "array(bigint)", span); + Expression sequence = call( + "sequence", + List.of(start, new BinaryExpression(HogQlQuery.BinaryOperator.SUBTRACT, end, integerLiteral("1", span), span)), + span); + return call("if", List.of(isEmpty, empty, sequence), span); + } + + private static Expression arrayEnumerate(Expression array, HogQlQuery.SourceSpan span) + { + Expression size = call("cardinality", List.of(array), span); + Expression empty = cast(new ArrayExpression(List.of(), span), "array(bigint)", span); + Expression sequence = call("sequence", List.of(integerLiteral("1", span), size), span); + return call( + "if", + List.of( + new BinaryExpression(HogQlQuery.BinaryOperator.EQUAL, size, integerLiteral("0", span), span), + empty, + sequence), + span); + } + + private static Expression arraySort(FunctionCall function, List arguments) + { + if (arguments.size() == 1) { + return call("array_sort", arguments, function.span()); + } + if (!(arguments.getFirst() instanceof LambdaExpression lambda) || lambda.arguments().size() != 1) { + throw unsupportedError(function, "HogQL arraySort key must be a single-argument lambda"); + } + HogQlQuery.SourceSpan span = function.span(); + Identifier left = new Identifier("__hogql_array_sort_left", false, span); + Identifier right = new Identifier("__hogql_array_sort_right", false, span); + String parameter = canonical(lambda.arguments().getFirst().value()); + Expression leftKey = substituteArraySortParameter(function, lambda.body(), parameter, new ColumnReference(List.of(left), span)); + Expression rightKey = substituteArraySortParameter(function, lambda.body(), parameter, new ColumnReference(List.of(right), span)); + Expression comparator = new CaseExpression( + Optional.empty(), + List.of( + new CaseWhen( + new BinaryExpression(HogQlQuery.BinaryOperator.LESS_THAN, leftKey, rightKey, span), + new UnaryExpression(HogQlQuery.UnaryOperator.NEGATE, integerLiteral("1", span), span), + span), + new CaseWhen( + new BinaryExpression(HogQlQuery.BinaryOperator.GREATER_THAN, leftKey, rightKey, span), + integerLiteral("1", span), + span)), + Optional.of(integerLiteral("0", span)), + span); + return call( + "array_sort", + List.of( + arguments.get(1), + new LambdaExpression(List.of(left, right), comparator, span)), + span); + } + + private static Expression substituteArraySortParameter(FunctionCall function, Expression expression, String parameter, Expression replacement) + { + return switch (expression) { + case ColumnReference reference -> reference.parts().size() == 1 && canonical(reference.parts().getFirst().value()).equals(parameter) + ? replacement + : reference; + case SubscriptExpression subscript -> new SubscriptExpression( + substituteArraySortParameter(function, subscript.base(), parameter, replacement), + substituteArraySortParameter(function, subscript.index(), parameter, replacement), + subscript.span()); + case MemberAccessExpression member -> new MemberAccessExpression( + substituteArraySortParameter(function, member.base(), parameter, replacement), + member.member(), + member.span()); + case CastExpression cast -> new CastExpression( + substituteArraySortParameter(function, cast.value(), parameter, replacement), + cast.type(), + cast.safe(), + cast.typeDialect(), + cast.span()); + case UnaryExpression unary -> new UnaryExpression( + unary.operator(), + substituteArraySortParameter(function, unary.operand(), parameter, replacement), + unary.span()); + case BinaryExpression binary -> new BinaryExpression( + binary.operator(), + substituteArraySortParameter(function, binary.left(), parameter, replacement), + substituteArraySortParameter(function, binary.right(), parameter, replacement), + binary.span()); + case Literal literal -> literal; + default -> throw unsupportedError(function, "HogQL arraySort key expression is outside the supported subset"); + }; + } + + private static Expression has(List arguments, HogQlQuery.SourceSpan span) + { + Expression array = arguments.getFirst(); + Expression value = arguments.get(1); + Identifier item = new Identifier("_hogql_item", false, span); + LambdaExpression isNull = new LambdaExpression( + List.of(item), + new IsNullExpression(new ColumnReference(List.of(item), span), false, span, span), + span); + Expression findNull = call("any_match", List.of(array, isNull), span); + Expression contains = coalesce( + call("contains", List.of(array, value), span), + new Literal(HogQlQuery.LiteralKind.BOOLEAN, "false", span), + span); + return call( + "if", + List.of(new IsNullExpression(value, false, span, span), findNull, contains), + span); + } + + private static Expression empty(Expression value, boolean expectedEmpty, HogQlQuery.SourceSpan span) + { + Expression isEmpty = coalesce( + call("hogql_empty", List.of(value), span), + new Literal(HogQlQuery.LiteralKind.BOOLEAN, "true", span), + span); + if (expectedEmpty) { + return isEmpty; + } + return new UnaryExpression(HogQlQuery.UnaryOperator.NOT, isEmpty, span); + } + + private static Expression startOfWeek(FunctionCall function, List arguments) + { + HogQlQuery.SourceSpan span = function.span(); + String mode = "0"; + if (arguments.size() == 2) { + Expression modeExpression = arguments.get(1); + if (!(modeExpression instanceof Literal literal) || literal.kind() != HogQlQuery.LiteralKind.INTEGER) { + throw unsupportedError(function, "HogQL toStartOfWeek mode must be an integer literal"); + } + mode = literal.value(); + } + if (mode.equals("1") || mode.equals("3")) { + return dateTrunc("week", arguments.getFirst(), span); + } + if (!mode.equals("0")) { + throw unsupportedError(function, "HogQL toStartOfWeek mode must be 0, 1, or 3"); + } + Expression value = arguments.getFirst(); + Expression shifted = call("date_add", List.of( + new Literal(HogQlQuery.LiteralKind.STRING, "day", span), + integerLiteral("1", span), + value), span); + Expression monday = dateTrunc("week", shifted, span); + return call("date_add", List.of( + new Literal(HogQlQuery.LiteralKind.STRING, "day", span), + integerLiteral("-1", span), + monday), span); + } + + private static FunctionCall dateTrunc(String unit, Expression value, HogQlQuery.SourceSpan span) + { + return new FunctionCall( + new Identifier("date_trunc", false, span), + List.of(new Literal(HogQlQuery.LiteralKind.STRING, unit, span), value), + false, + List.of(), + Optional.empty(), + span); + } + + private WindowDefinition resolveWindowDefinition(WindowDefinition definition) + { + return new WindowDefinition(definition.name(), (WindowSpecification) resolveWindow(definition.specification()), definition.span()); + } + + private Window resolveWindow(Window window) + { + return switch (window) { + case WindowReference reference -> reference; + case WindowSpecification specification -> new WindowSpecification( + specification.partitionBy().stream().map(this::resolveExpression).toList(), + resolveSortItems(specification.orderBy()), + specification.frame().map(this::resolveWindowFrame), + specification.span()); + }; + } + + private WindowFrame resolveWindowFrame(WindowFrame frame) + { + return new WindowFrame( + frame.type(), + new HogQlQuery.FrameBound( + frame.start().type(), + frame.start().value().map(this::resolveExpression), + frame.start().span()), + frame.end().map(bound -> new HogQlQuery.FrameBound( + bound.type(), + bound.value().map(this::resolveExpression), + bound.span())), + frame.span()); + } + + private static boolean matchesArity(List signatures, int arity) + { + return signatures.stream().anyMatch(signature -> signature.variadic() + ? arity >= Math.max(0, signature.argumentTypes().size() - 1) + : arity == signature.argumentTypes().size()); + } + + private static Identifier identifier(PhysicalIdentifier identifier, HogQlQuery.SourceSpan span) + { + return new Identifier(identifier.value(), identifier.delimited(), span); + } + + private static String canonical(String value) + { + return value.toLowerCase(Locale.ENGLISH); + } + + private static TrinoException resolutionError(FunctionCall function, String message) + { + return error(HOGQL_RESOLUTION_ERROR, function, message); + } + + private static TrinoException unsupportedError(FunctionCall function, String message) + { + return error(HOGQL_UNSUPPORTED_FEATURE, function, message); + } + + private static TrinoException error(HogQlErrorCode errorCode, FunctionCall function, String message) + { + return new TrinoException( + errorCode, + Optional.of(new Location(function.span().startLine(), function.span().startColumn())), + message, + null); + } + + record Resolution(HogQlQuery query, OptionalLong exchangeRateGeneration) + { + Resolution + { + query = requireNonNull(query, "query is null"); + exchangeRateGeneration = requireNonNull(exchangeRateGeneration, "exchangeRateGeneration is null"); + } + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlLimitByRewriter.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlLimitByRewriter.java new file mode 100644 index 000000000000..e41be08b0cd7 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlLimitByRewriter.java @@ -0,0 +1,355 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.ArrayExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BetweenExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator; +import io.trino.hogql.parser.tree.HogQlQuery.CaseExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CaseWhen; +import io.trino.hogql.parser.tree.HogQlQuery.CastExpression; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; +import io.trino.hogql.parser.tree.HogQlQuery.Expression; +import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; +import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; +import io.trino.hogql.parser.tree.HogQlQuery.Identifier; +import io.trino.hogql.parser.tree.HogQlQuery.InCohortExpression; +import io.trino.hogql.parser.tree.HogQlQuery.InExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IntervalExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; +import io.trino.hogql.parser.tree.HogQlQuery.LimitBy; +import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; +import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; +import io.trino.hogql.parser.tree.HogQlQuery.Projection; +import io.trino.hogql.parser.tree.HogQlQuery.ScalarSubqueryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.SelectQueryBody; +import io.trino.hogql.parser.tree.HogQlQuery.SortItem; +import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; +import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; +import io.trino.hogql.parser.tree.HogQlQuery.SubscriptExpression; +import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; +import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.WindowSpecification; +import io.trino.spi.Location; +import io.trino.spi.TrinoException; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE; + +final class HogQlLimitByRewriter +{ + private static final String BASE_ALIAS = "__hogql_limit_by_base"; + private static final String RANKED_ALIAS = "__hogql_limit_by_ranked"; + private static final String ROW_NUMBER = "__hogql_limit_by_row_number"; + + private HogQlLimitByRewriter() {} + + public static HogQlQuery rewrite(HogQlQuery query) + { + if (!(query.body() instanceof SelectQueryBody select) || select.limitBy().isEmpty()) { + return query; + } + LimitBy limitBy = select.limitBy().orElseThrow(); + List outputs = outputs(select.projections(), limitBy.span()); + Map outputsByName = outputsByName(outputs); + + List baseProjections = outputs.stream() + .map(output -> new ExpressionProjection(output.expression(), Optional.of(output.innerName()))) + .map(Projection.class::cast) + .toList(); + HogQlQuery baseQuery = new HogQlQuery( + List.of(), + new SelectQueryBody( + select.distinct(), + baseProjections, + select.from(), + select.where(), + select.groupBy(), + select.having(), + select.windows(), + Optional.empty(), + select.span()), + List.of(), + Optional.empty(), + Optional.empty(), + query.span()); + + Identifier baseAlias = identifier(BASE_ALIAS, limitBy.span()); + List rankedProjections = new ArrayList<>(); + outputs.forEach(output -> rankedProjections.add(new ExpressionProjection( + column(baseAlias, output.innerName(), output.expression().span()), + Optional.of(output.innerName())))); + WindowSpecification rankingWindow = new WindowSpecification( + limitBy.partitionBy().stream() + .map(expression -> remap(expression, outputsByName, baseAlias)) + .toList(), + query.orderBy().stream() + .map(sortItem -> remap(sortItem, outputsByName, baseAlias)) + .toList(), + Optional.empty(), + limitBy.span()); + FunctionCall rowNumber = new FunctionCall( + List.of(identifier("row_number", limitBy.span())), + List.of(), + false, + List.of(), + Optional.empty(), + Optional.empty(), + Optional.of(rankingWindow), + limitBy.span()); + rankedProjections.add(new ExpressionProjection(rowNumber, Optional.of(identifier(ROW_NUMBER, limitBy.span())))); + HogQlQuery rankedQuery = new HogQlQuery( + List.of(), + new SelectQueryBody( + false, + rankedProjections, + Optional.of(new HogQlQuery.AliasedRelation(new SubqueryRelation(baseQuery, query.span()), baseAlias, query.span())), + Optional.empty(), + List.of(), + Optional.empty(), + List.of(), + Optional.empty(), + select.span()), + List.of(), + Optional.empty(), + Optional.empty(), + query.span()); + + Identifier rankedAlias = identifier(RANKED_ALIAS, limitBy.span()); + List outerProjections = outputs.stream() + .map(output -> new ExpressionProjection( + column(rankedAlias, output.innerName(), output.expression().span()), + Optional.of(output.outputName()))) + .map(Projection.class::cast) + .toList(); + Expression rowNumberReference = column(rankedAlias, identifier(ROW_NUMBER, limitBy.span()), limitBy.span()); + Expression upperBound = limitBy.offset() + .map(offset -> new BinaryExpression(BinaryOperator.ADD, offset, limitBy.limit(), limitBy.span())) + .orElse(limitBy.limit()); + Expression filter = new BinaryExpression(BinaryOperator.LESS_THAN_OR_EQUAL, rowNumberReference, upperBound, limitBy.span()); + if (limitBy.offset().isPresent()) { + filter = new BinaryExpression( + BinaryOperator.AND, + new BinaryExpression(BinaryOperator.GREATER_THAN, rowNumberReference, limitBy.offset().orElseThrow(), limitBy.span()), + filter, + limitBy.span()); + } + List outerOrderBy = query.orderBy().stream() + .map(sortItem -> remap(sortItem, outputsByName, rankedAlias)) + .toList(); + return new HogQlQuery( + query.with(), + new SelectQueryBody( + false, + outerProjections, + Optional.of(new HogQlQuery.AliasedRelation(new SubqueryRelation(rankedQuery, query.span()), rankedAlias, query.span())), + Optional.of(filter), + List.of(), + Optional.empty(), + List.of(), + Optional.empty(), + select.span()), + outerOrderBy, + query.limit(), + query.offset(), + query.span()); + } + + private static List outputs(List projections, SourceSpan span) + { + List outputs = new ArrayList<>(); + for (Projection projection : projections) { + if (!(projection instanceof ExpressionProjection expression)) { + throw unsupported(span, "HogQL LIMIT BY requires explicit output columns"); + } + Identifier outputName = expression.alias() + .orElseGet(() -> { + if (expression.expression() instanceof ColumnReference reference) { + return reference.parts().getLast(); + } + throw unsupported(expression.span(), "HogQL LIMIT BY expressions require output aliases"); + }); + outputs.add(new OutputColumn( + expression.expression(), + outputName, + identifier("__hogql_limit_by_column_" + outputs.size(), expression.span()))); + } + return List.copyOf(outputs); + } + + private static Map outputsByName(List outputs) + { + Map byName = new HashMap<>(); + Set duplicates = new HashSet<>(); + for (OutputColumn output : outputs) { + String name = canonical(output.outputName().value()); + if (byName.putIfAbsent(name, output) != null) { + duplicates.add(name); + } + } + duplicates.forEach(byName::remove); + return Map.copyOf(byName); + } + + private static SortItem remap(SortItem sortItem, Map outputs, Identifier relationAlias) + { + return new SortItem( + remap(sortItem.expression(), outputs, relationAlias), + sortItem.direction(), + sortItem.nullPlacement(), + sortItem.span()); + } + + private static Expression remap(Expression expression, Map outputs, Identifier relationAlias) + { + return switch (expression) { + case ArrayExpression array -> new ArrayExpression(array.values().stream().map(value -> remap(value, outputs, relationAlias)).toList(), array.span()); + case BetweenExpression between -> new BetweenExpression( + remap(between.value(), outputs, relationAlias), + remap(between.min(), outputs, relationAlias), + remap(between.max(), outputs, relationAlias), + between.negated(), + between.predicateSpan(), + between.span()); + case BinaryExpression binary -> new BinaryExpression( + binary.operator(), + remap(binary.left(), outputs, relationAlias), + remap(binary.right(), outputs, relationAlias), + binary.span()); + case CaseExpression caseExpression -> new CaseExpression( + caseExpression.operand().map(value -> remap(value, outputs, relationAlias)), + caseExpression.whenClauses().stream() + .map(when -> new CaseWhen( + remap(when.operand(), outputs, relationAlias), + remap(when.result(), outputs, relationAlias), + when.span())) + .toList(), + caseExpression.defaultValue().map(value -> remap(value, outputs, relationAlias)), + caseExpression.span()); + case CastExpression cast -> new CastExpression(remap(cast.value(), outputs, relationAlias), cast.type(), cast.safe(), cast.typeDialect(), cast.span()); + case ColumnReference reference -> remap(reference, outputs, relationAlias); + case FunctionCall function -> new FunctionCall( + function.nameParts(), + function.arguments().stream().map(argument -> remap(argument, outputs, relationAlias)).toList(), + function.distinct(), + function.orderBy().stream().map(item -> remap(item, outputs, relationAlias)).toList(), + function.filter().map(filter -> remap(filter, outputs, relationAlias)), + function.nullTreatment(), + function.window().map(_ -> { + throw unsupported(function.span(), "HogQL LIMIT BY does not support window expressions in its keys or ordering"); + }), + function.span()); + case InCohortExpression in -> new InCohortExpression( + remap(in.value(), outputs, relationAlias), + remap(in.cohort(), outputs, relationAlias), + in.negated(), + in.predicateSpan(), + in.span()); + case InExpression in -> new InExpression( + remap(in.value(), outputs, relationAlias), + in.values().stream().map(value -> remap(value, outputs, relationAlias)).toList(), + in.negated(), + in.predicateSpan(), + in.span()); + case HogQlQuery.InSubqueryExpression in -> throw unsupported(in.span(), "HogQL LIMIT BY does not support subqueries in its keys or ordering"); + case IntervalExpression interval -> new IntervalExpression(remap(interval.value(), outputs, relationAlias), interval.unit(), interval.span()); + case IsNullExpression isNull -> new IsNullExpression( + remap(isNull.value(), outputs, relationAlias), + isNull.negated(), + isNull.predicateSpan(), + isNull.span()); + case HogQlQuery.LambdaExpression lambda -> throw unsupported(lambda.span(), "HogQL LIMIT BY does not support lambdas in its keys or ordering"); + case Literal literal -> literal; + case MemberAccessExpression member -> new MemberAccessExpression(remap(member.base(), outputs, relationAlias), member.member(), member.span()); + case Placeholder placeholder -> placeholder; + case ScalarSubqueryExpression subquery -> throw unsupported(subquery.span(), "HogQL LIMIT BY does not support subqueries in its keys or ordering"); + case SubscriptExpression subscript -> new SubscriptExpression( + remap(subscript.base(), outputs, relationAlias), + remap(subscript.index(), outputs, relationAlias), + subscript.span()); + case TupleExpression tuple -> new TupleExpression(tuple.values().stream().map(value -> remap(value, outputs, relationAlias)).toList(), tuple.span()); + case UnaryExpression unary -> new UnaryExpression(unary.operator(), remap(unary.operand(), outputs, relationAlias), unary.span()); + }; + } + + private static Expression remap(ColumnReference reference, Map outputs, Identifier relationAlias) + { + if (reference.parts().size() == 1) { + OutputColumn output = outputs.get(canonical(reference.parts().getFirst().value())); + if (output != null) { + return column(relationAlias, output.innerName(), reference.span()); + } + } + Optional exact = outputs.values().stream() + .filter(output -> output.expression() instanceof ColumnReference) + .filter(output -> sameReference(reference, (ColumnReference) output.expression())) + .findFirst(); + if (exact.isPresent()) { + return column(relationAlias, exact.orElseThrow().innerName(), reference.span()); + } + throw unsupported(reference.span(), "HogQL LIMIT BY keys and ordering must reference projected outputs"); + } + + private static boolean sameReference(ColumnReference left, ColumnReference right) + { + if (left.parts().size() != right.parts().size()) { + return false; + } + for (int index = 0; index < left.parts().size(); index++) { + Identifier leftPart = left.parts().get(index); + Identifier rightPart = right.parts().get(index); + if (leftPart.delimited() != rightPart.delimited() || !canonical(leftPart.value()).equals(canonical(rightPart.value()))) { + return false; + } + } + return true; + } + + private static ColumnReference column(Identifier relation, Identifier column, SourceSpan span) + { + return new ColumnReference(List.of(relation, column), span); + } + + private static Identifier identifier(String value, SourceSpan span) + { + return new Identifier(value, false, span); + } + + private static String canonical(String value) + { + return value.toLowerCase(Locale.ENGLISH); + } + + private static TrinoException unsupported(SourceSpan span, String message) + { + return new TrinoException( + HOGQL_UNSUPPORTED_FEATURE, + Optional.of(new Location(span.startLine(), span.startColumn())), + message, + null); + } + + private record OutputColumn(Expression expression, Identifier outputName, Identifier innerName) {} +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlModifierBinding.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlModifierBinding.java new file mode 100644 index 000000000000..6a8adaa74862 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlModifierBinding.java @@ -0,0 +1,38 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; + +import java.util.List; +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +public record HogQlModifierBinding(String modifierName, Optional> sessionProperty, HogQlTypedValue value) +{ + public HogQlModifierBinding + { + modifierName = requireNonNull(modifierName, "modifierName is null"); + if (modifierName.isBlank()) { + throw new IllegalArgumentException("modifier name is empty"); + } + sessionProperty = requireNonNull(sessionProperty, "sessionProperty is null") + .map(property -> List.copyOf(requireNonNull(property, "session property is null"))); + if (sessionProperty.isPresent() && sessionProperty.orElseThrow().isEmpty()) { + throw new IllegalArgumentException("session property name is empty"); + } + value = requireNonNull(value, "value is null"); + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlModifierResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlModifierResolver.java new file mode 100644 index 000000000000..ed60b988c5b8 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlModifierResolver.java @@ -0,0 +1,177 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.trino.hogql.compiler.HogQlTypedValue.ArrayValue; +import io.trino.hogql.compiler.HogQlTypedValue.BooleanValue; +import io.trino.hogql.compiler.HogQlTypedValue.NullValue; +import io.trino.hogql.compiler.HogQlTypedValue.NumberValue; +import io.trino.hogql.compiler.HogQlTypedValue.ObjectValue; +import io.trino.hogql.compiler.HogQlTypedValue.StringValue; +import io.trino.hogql.compiler.HogQlTypedValue.Value; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticModifierDefault; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.TypedLiteral; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; +import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; +import io.trino.spi.Location; +import io.trino.spi.TrinoException; + +import java.util.ArrayList; +import java.util.Base64; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HexFormat; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; + +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_BINDING_ERROR; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_COMPILER_INTERNAL_ERROR; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE; +import static java.util.Objects.requireNonNull; + +final class HogQlModifierResolver +{ + private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); + + private HogQlModifierResolver() {} + + public static List resolve( + PinnedSnapshot pinnedSnapshot, + Map suppliedModifiers, + SourceSpan querySpan) + { + requireNonNull(pinnedSnapshot, "pinnedSnapshot is null"); + requireNonNull(suppliedModifiers, "suppliedModifiers is null"); + requireNonNull(querySpan, "querySpan is null"); + + Map definitions = new HashMap<>(); + for (SemanticModifierDefault definition : pinnedSnapshot.snapshot().modifierDefaults()) { + definitions.put(canonical(definition.name()), definition); + } + + Map suppliedByCanonicalName = new LinkedHashMap<>(); + suppliedModifiers.entrySet().stream() + .sorted(Comparator.comparing(entry -> canonical(entry.getKey()))) + .forEach(entry -> { + String name = canonical(entry.getKey()); + if (suppliedByCanonicalName.put(name, entry.getValue()) != null) { + throw bindingError(querySpan, "Duplicate HogQL modifier: " + entry.getKey()); + } + SemanticModifierDefault definition = definitions.get(name); + if (definition == null) { + throw bindingError(querySpan, "Unknown HogQL modifier: " + entry.getKey()); + } + if (!definition.defaultValue().typeSignature().equalsIgnoreCase(entry.getValue().type())) { + throw bindingError(querySpan, "HogQL modifier has an incompatible type: " + definition.name()); + } + }); + + List bindings = new ArrayList<>(); + for (SemanticModifierDefault definition : pinnedSnapshot.snapshot().modifierDefaults()) { + Optional supplied = Optional.ofNullable(suppliedByCanonicalName.get(canonical(definition.name()))); + switch (definition.behavior()) { + case TRINO_SESSION_PROPERTY -> bindings.add(new HogQlModifierBinding( + definition.name(), + Optional.of(definition.sessionProperty()), + supplied.orElseGet(() -> typedValue(definition.defaultValue())))); + case COMPILER -> { + if (supplied.isPresent()) { + throw unsupportedError(querySpan, "HogQL compiler modifier is not implemented: " + definition.name()); + } + } + case SAFE_NOOP -> supplied.ifPresent(value -> bindings.add(new HogQlModifierBinding( + definition.name(), + Optional.empty(), + value))); + case UNSUPPORTED -> { + if (supplied.isPresent()) { + throw unsupportedError(querySpan, "HogQL modifier is not supported: " + definition.name()); + } + } + } + } + return List.copyOf(bindings); + } + + private static HogQlTypedValue typedValue(TypedLiteral literal) + { + Value value = switch (literal.encoding()) { + case NULL -> NullValue.NULL; + case STRING -> new StringValue(literal.value()); + case BOOLEAN -> new BooleanValue(Boolean.parseBoolean(literal.value())); + case INTEGER, DECIMAL, FLOAT -> new NumberValue(literal.value()); + case JSON -> jsonValue(literal.value()); + case BASE64 -> new StringValue(HexFormat.of().formatHex(Base64.getDecoder().decode(literal.value()))); + }; + return new HogQlTypedValue(literal.typeSignature(), value); + } + + private static Value jsonValue(String json) + { + try { + return jsonValue(JSON_MAPPER.readTree(json)); + } + catch (JsonProcessingException _) { + throw new TrinoException(HOGQL_COMPILER_INTERNAL_ERROR, "Invalid HogQL modifier default"); + } + } + + private static Value jsonValue(JsonNode node) + { + if (node.isNull()) { + return NullValue.NULL; + } + if (node.isBoolean()) { + return new BooleanValue(node.booleanValue()); + } + if (node.isNumber()) { + return new NumberValue(node.asText()); + } + if (node.isTextual()) { + return new StringValue(node.textValue()); + } + if (node.isArray()) { + List values = new ArrayList<>(node.size()); + node.forEach(element -> values.add(jsonValue(element))); + return new ArrayValue(values); + } + if (node.isObject()) { + Map values = new LinkedHashMap<>(); + node.properties().forEach(entry -> values.put(entry.getKey(), jsonValue(entry.getValue()))); + return new ObjectValue(values); + } + throw new TrinoException(HOGQL_COMPILER_INTERNAL_ERROR, "Invalid HogQL modifier default"); + } + + private static String canonical(String name) + { + return requireNonNull(name, "name is null").toLowerCase(Locale.ENGLISH); + } + + private static TrinoException bindingError(SourceSpan span, String message) + { + return new TrinoException(HOGQL_BINDING_ERROR, Optional.of(new Location(span.startLine(), span.startColumn())), message, null); + } + + private static TrinoException unsupportedError(SourceSpan span, String message) + { + return new TrinoException(HOGQL_UNSUPPORTED_FEATURE, Optional.of(new Location(span.startLine(), span.startColumn())), message, null); + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java new file mode 100644 index 000000000000..7d5b67ec7090 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlProjectionDemand.java @@ -0,0 +1,456 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.ArrayExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BetweenExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CaseExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CastExpression; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsList; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsRegex; +import io.trino.hogql.parser.tree.HogQlQuery.Expression; +import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; +import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; +import io.trino.hogql.parser.tree.HogQlQuery.Identifier; +import io.trino.hogql.parser.tree.HogQlQuery.InCohortExpression; +import io.trino.hogql.parser.tree.HogQlQuery.InExpression; +import io.trino.hogql.parser.tree.HogQlQuery.InSubqueryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IntervalExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; +import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; +import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; +import io.trino.hogql.parser.tree.HogQlQuery.JoinUsing; +import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.LambdaExpression; +import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; +import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; +import io.trino.hogql.parser.tree.HogQlQuery.Projection; +import io.trino.hogql.parser.tree.HogQlQuery.Relation; +import io.trino.hogql.parser.tree.HogQlQuery.ScalarSubqueryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.SelectQueryBody; +import io.trino.hogql.parser.tree.HogQlQuery.Star; +import io.trino.hogql.parser.tree.HogQlQuery.SubscriptExpression; +import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; +import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.UnnestRelation; +import io.trino.hogql.parser.tree.HogQlQuery.Window; +import io.trino.hogql.parser.tree.HogQlQuery.WindowFrame; +import io.trino.hogql.parser.tree.HogQlQuery.WindowReference; +import io.trino.hogql.parser.tree.HogQlQuery.WindowSpecification; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import static java.util.Objects.requireNonNull; + +final class HogQlProjectionDemand +{ + private final boolean all; + private final Set unqualified; + private final Set allQualifiers; + private final Map> qualified; + + private HogQlProjectionDemand(boolean all, Set unqualified, Set allQualifiers, Map> qualified) + { + this.all = all; + this.unqualified = Set.copyOf(requireNonNull(unqualified, "unqualified is null")); + this.allQualifiers = Set.copyOf(requireNonNull(allQualifiers, "allQualifiers is null")); + requireNonNull(qualified, "qualified is null"); + Map> copied = new HashMap<>(); + qualified.forEach((key, value) -> copied.put(key, Set.copyOf(value))); + this.qualified = Map.copyOf(copied); + } + + public static HogQlProjectionDemand collect(HogQlQuery query) + { + Builder builder = new Builder(); + if (query.body() instanceof SelectQueryBody select) { + select.projections().forEach(projection -> collect(projection, builder)); + select.where().ifPresent(expression -> collect(expression, builder)); + select.groupBy().forEach(expression -> collect(expression, builder)); + select.having().ifPresent(expression -> collect(expression, builder)); + select.windows().forEach(window -> collect(window.specification(), builder)); + select.limitBy().ifPresent(limitBy -> { + collect(limitBy.limit(), builder); + limitBy.offset().ifPresent(expression -> collect(expression, builder)); + limitBy.partitionBy().forEach(expression -> collect(expression, builder)); + }); + select.from().ifPresent(relation -> collectJoinCriteria(relation, builder)); + } + query.orderBy().forEach(sortItem -> collect(sortItem.expression(), builder)); + query.limit().ifPresent(expression -> collect(expression, builder)); + query.offset().ifPresent(expression -> collect(expression, builder)); + return builder.build(); + } + + public static HogQlProjectionDemand collect(HogQlQuery query, RequiredOutputs requiredOutputs) + { + requireNonNull(requiredOutputs, "requiredOutputs is null"); + if (requiredOutputs.all()) { + return collect(query); + } + Builder builder = collectNonProjection(query).toBuilder(); + if (query.body() instanceof SelectQueryBody select) { + select.projections().forEach(projection -> collectRequiredProjection(projection, requiredOutputs, builder)); + } + return builder.build(); + } + + private static void collectRequiredProjection(Projection projection, RequiredOutputs requiredOutputs, Builder builder) + { + switch (projection) { + case ExpressionProjection expression -> { + String name = expression.alias() + .map(Identifier::value) + .orElseGet(() -> expression.expression() instanceof ColumnReference reference ? reference.parts().getLast().value() : ""); + if (requiredOutputs.includes(name)) { + collect(expression.expression(), builder); + } + } + case ColumnsList columns -> columns.expressions().stream() + .filter(ColumnReference.class::isInstance) + .map(ColumnReference.class::cast) + .filter(reference -> requiredOutputs.includes(reference.parts().getLast().value())) + .forEach(builder::add); + case ColumnsRegex _ -> requiredOutputs.names().forEach(builder::addUnqualified); + case Star star -> requiredOutputs.names().forEach(name -> { + if (star.qualifier().isEmpty()) { + builder.addUnqualified(name); + } + else { + builder.addQualified(star.qualifier().getFirst().value(), name); + } + }); + } + } + + public static HogQlProjectionDemand collectNonProjection(HogQlQuery query) + { + Builder builder = new Builder(); + if (query.body() instanceof SelectQueryBody select) { + select.where().ifPresent(expression -> collect(expression, builder)); + select.groupBy().forEach(expression -> collect(expression, builder)); + select.having().ifPresent(expression -> collect(expression, builder)); + select.windows().forEach(window -> collect(window.specification(), builder)); + select.limitBy().ifPresent(limitBy -> { + collect(limitBy.limit(), builder); + limitBy.offset().ifPresent(expression -> collect(expression, builder)); + limitBy.partitionBy().forEach(expression -> collect(expression, builder)); + }); + select.from().ifPresent(relation -> collectJoinCriteria(relation, builder)); + } + query.orderBy().forEach(sortItem -> collect(sortItem.expression(), builder)); + query.limit().ifPresent(expression -> collect(expression, builder)); + query.offset().ifPresent(expression -> collect(expression, builder)); + return builder.build(); + } + + public static RequiredOutputs collectOrderingOutputs(HogQlQuery query) + { + Builder builder = new Builder(); + if (query.body() instanceof SelectQueryBody select) { + select.limitBy().ifPresent(limitBy -> limitBy.partitionBy().forEach(expression -> collect(expression, builder))); + } + query.orderBy().forEach(sortItem -> collect(sortItem.expression(), builder)); + return builder.build().asRequiredOutputs(); + } + + public static HogQlProjectionDemand collect(Expression expression) + { + Builder builder = new Builder(); + collect(expression, builder); + return builder.build(); + } + + public static HogQlProjectionDemand column(String qualifier, String name) + { + Builder builder = new Builder(); + builder.addQualified(qualifier, name); + return builder.build(); + } + + public static HogQlProjectionDemand preserveAll() + { + return new HogQlProjectionDemand(true, Set.of(), Set.of(), Map.of()); + } + + public static HogQlProjectionDemand preserveNone() + { + return new HogQlProjectionDemand(false, Set.of(), Set.of(), Map.of()); + } + + public RequiredOutputs forAlias(Identifier alias) + { + String qualifier = canonical(alias.value()); + if (all || allQualifiers.contains(qualifier)) { + return RequiredOutputs.allOutputs(); + } + Set names = new HashSet<>(unqualified); + names.addAll(qualified.getOrDefault(qualifier, Set.of())); + return new RequiredOutputs(false, names); + } + + public RequiredOutputs unqualified() + { + if (all) { + return RequiredOutputs.allOutputs(); + } + return new RequiredOutputs(false, unqualified); + } + + private RequiredOutputs asRequiredOutputs() + { + if (all || !allQualifiers.isEmpty()) { + return RequiredOutputs.allOutputs(); + } + Set names = new HashSet<>(unqualified); + qualified.values().forEach(names::addAll); + return new RequiredOutputs(false, names); + } + + public HogQlProjectionDemand merge(HogQlProjectionDemand other) + { + requireNonNull(other, "other is null"); + Map> mergedQualified = new HashMap<>(); + qualified.forEach((key, value) -> mergedQualified.put(key, new HashSet<>(value))); + other.qualified.forEach((key, value) -> mergedQualified.computeIfAbsent(key, _ -> new HashSet<>()).addAll(value)); + Set mergedUnqualified = new HashSet<>(unqualified); + mergedUnqualified.addAll(other.unqualified); + Set mergedAllQualifiers = new HashSet<>(allQualifiers); + mergedAllQualifiers.addAll(other.allQualifiers); + return new HogQlProjectionDemand(all || other.all, mergedUnqualified, mergedAllQualifiers, mergedQualified); + } + + private Builder toBuilder() + { + Builder builder = new Builder(); + builder.all = all; + builder.unqualified.addAll(unqualified); + builder.allQualifiers.addAll(allQualifiers); + qualified.forEach((key, value) -> builder.qualified.put(key, new HashSet<>(value))); + return builder; + } + + private static void collect(Projection projection, Builder builder) + { + switch (projection) { + case ColumnsList columns -> columns.expressions().forEach(expression -> collect(expression, builder)); + case ColumnsRegex _ -> builder.all = true; + case ExpressionProjection expression -> collect(expression.expression(), builder); + case Star star -> { + if (star.qualifier().isEmpty()) { + builder.all = true; + } + else { + builder.allQualifiers.add(canonical(star.qualifier().getFirst().value())); + } + star.replacements().forEach(replacement -> collect(replacement.expression(), builder)); + } + } + } + + private static void collectJoinCriteria(Relation relation, Builder builder) + { + switch (relation) { + case HogQlQuery.AliasedRelation alias -> collectJoinCriteria(alias.relation(), builder); + case HogQlQuery.CommonTableReference _, HogQlQuery.SubqueryRelation _, HogQlQuery.TablePlaceholder _, HogQlQuery.TableReference _, HogQlQuery.ValuesRelation _ -> {} + case JoinRelation join -> { + collectJoinCriteria(join.left(), builder); + collectJoinCriteria(join.right(), builder); + join.criteria().ifPresent(criteria -> { + switch (criteria) { + case JoinOn on -> collect(on.expression(), builder); + case JoinUsing using -> using.columns().forEach(builder::addUnqualified); + } + }); + } + case HogQlQuery.PivotRelation pivot -> { + collectJoinCriteria(pivot.input(), builder); + pivot.aggregations().forEach(aggregation -> collect(aggregation.expression(), builder)); + pivot.pivotColumns().forEach(column -> collect(column, builder)); + pivot.valueGroups().forEach(group -> group.values().forEach(value -> collect(value, builder))); + pivot.groupBy().forEach(expression -> collect(expression, builder)); + } + case UnnestRelation unnest -> unnest.expressions().forEach(expression -> collect(expression, builder)); + } + } + + private static void collect(Expression expression, Builder builder) + { + switch (expression) { + case ArrayExpression array -> array.values().forEach(value -> collect(value, builder)); + case BetweenExpression between -> { + collect(between.value(), builder); + collect(between.min(), builder); + collect(between.max(), builder); + } + case BinaryExpression binary -> { + collect(binary.left(), builder); + collect(binary.right(), builder); + } + case CaseExpression caseExpression -> { + caseExpression.operand().ifPresent(value -> collect(value, builder)); + caseExpression.whenClauses().forEach(when -> { + collect(when.operand(), builder); + collect(when.result(), builder); + }); + caseExpression.defaultValue().ifPresent(value -> collect(value, builder)); + } + case CastExpression cast -> collect(cast.value(), builder); + case ColumnReference reference -> builder.add(reference); + case FunctionCall function -> { + function.arguments().forEach(argument -> collect(argument, builder)); + function.orderBy().forEach(sortItem -> collect(sortItem.expression(), builder)); + function.filter().ifPresent(filter -> collect(filter, builder)); + function.window().ifPresent(window -> collect(window, builder)); + } + case InCohortExpression cohort -> collect(cohort.value(), builder); + case InExpression in -> { + collect(in.value(), builder); + in.values().forEach(value -> collect(value, builder)); + } + case InSubqueryExpression in -> { + collect(in.value(), builder); + builder.all = true; + } + case IntervalExpression interval -> collect(interval.value(), builder); + case IsNullExpression isNull -> collect(isNull.value(), builder); + case LambdaExpression lambda -> builder.withSuppressed( + lambda.arguments().stream().map(Identifier::value).map(HogQlProjectionDemand::canonical).collect(java.util.stream.Collectors.toSet()), + () -> collect(lambda.body(), builder)); + case Literal _, Placeholder _ -> {} + case MemberAccessExpression member -> collect(member.base(), builder); + case ScalarSubqueryExpression _ -> builder.all = true; + case SubscriptExpression subscript -> { + collect(subscript.base(), builder); + collect(subscript.index(), builder); + } + case TupleExpression tuple -> tuple.values().forEach(value -> collect(value, builder)); + case UnaryExpression unary -> collect(unary.operand(), builder); + } + } + + private static void collect(Window window, Builder builder) + { + switch (window) { + case WindowReference _ -> {} + case WindowSpecification specification -> { + specification.partitionBy().forEach(expression -> collect(expression, builder)); + specification.orderBy().forEach(sortItem -> collect(sortItem.expression(), builder)); + specification.frame().ifPresent(frame -> collect(frame, builder)); + } + } + } + + private static void collect(WindowFrame frame, Builder builder) + { + frame.start().value().ifPresent(value -> collect(value, builder)); + frame.end().flatMap(HogQlQuery.FrameBound::value).ifPresent(value -> collect(value, builder)); + } + + private static String canonical(String value) + { + return value.toLowerCase(java.util.Locale.ENGLISH); + } + + public record RequiredOutputs(boolean all, Set names) + { + public RequiredOutputs + { + names = Set.copyOf(requireNonNull(names, "names is null")); + } + + public static RequiredOutputs allOutputs() + { + return new RequiredOutputs(true, Set.of()); + } + + public boolean includes(String name) + { + return all || names.contains(canonical(name)); + } + + public RequiredOutputs merge(RequiredOutputs other) + { + requireNonNull(other, "other is null"); + Set merged = new HashSet<>(names); + merged.addAll(other.names); + return new RequiredOutputs(all || other.all, merged); + } + } + + private static final class Builder + { + private boolean all; + private final Set unqualified = new HashSet<>(); + private final Set allQualifiers = new HashSet<>(); + private final Map> qualified = new HashMap<>(); + private final Set suppressed = new HashSet<>(); + + public void add(ColumnReference reference) + { + if (reference.parts().size() == 1) { + String name = canonical(reference.parts().getFirst().value()); + if (!suppressed.contains(name)) { + unqualified.add(name); + } + return; + } + String qualifier = canonical(reference.parts().getFirst().value()); + if (suppressed.contains(qualifier)) { + return; + } + String field = canonical(reference.parts().get(1).value()); + qualified.computeIfAbsent(qualifier, _ -> new HashSet<>()).add(field); + } + + public void withSuppressed(Set names, Runnable action) + { + Set previous = Set.copyOf(suppressed); + suppressed.addAll(names); + try { + action.run(); + } + finally { + suppressed.clear(); + suppressed.addAll(previous); + } + } + + public void addUnqualified(Identifier identifier) + { + unqualified.add(canonical(identifier.value())); + } + + public void addUnqualified(String name) + { + unqualified.add(canonical(name)); + } + + public void addQualified(String qualifier, String name) + { + qualified.computeIfAbsent(canonical(qualifier), _ -> new HashSet<>()).add(canonical(name)); + } + + public HogQlProjectionDemand build() + { + return new HogQlProjectionDemand(all, unqualified, allQualifiers, qualified); + } + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSelectAliasRewriter.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSelectAliasRewriter.java new file mode 100644 index 000000000000..1ce4ced56d71 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSelectAliasRewriter.java @@ -0,0 +1,360 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.AliasedRelation; +import io.trino.hogql.parser.tree.HogQlQuery.ArrayExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BetweenExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CaseExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CaseWhen; +import io.trino.hogql.parser.tree.HogQlQuery.CastExpression; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsList; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsRegex; +import io.trino.hogql.parser.tree.HogQlQuery.CommonTableExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CommonTableReference; +import io.trino.hogql.parser.tree.HogQlQuery.Expression; +import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; +import io.trino.hogql.parser.tree.HogQlQuery.FrameBound; +import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; +import io.trino.hogql.parser.tree.HogQlQuery.Identifier; +import io.trino.hogql.parser.tree.HogQlQuery.InCohortExpression; +import io.trino.hogql.parser.tree.HogQlQuery.InExpression; +import io.trino.hogql.parser.tree.HogQlQuery.InSubqueryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IntervalExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; +import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; +import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; +import io.trino.hogql.parser.tree.HogQlQuery.JoinUsing; +import io.trino.hogql.parser.tree.HogQlQuery.LambdaExpression; +import io.trino.hogql.parser.tree.HogQlQuery.LimitBy; +import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; +import io.trino.hogql.parser.tree.HogQlQuery.PivotAggregation; +import io.trino.hogql.parser.tree.HogQlQuery.PivotRelation; +import io.trino.hogql.parser.tree.HogQlQuery.PivotValueGroup; +import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; +import io.trino.hogql.parser.tree.HogQlQuery.Projection; +import io.trino.hogql.parser.tree.HogQlQuery.Relation; +import io.trino.hogql.parser.tree.HogQlQuery.ScalarSubqueryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.SelectQueryBody; +import io.trino.hogql.parser.tree.HogQlQuery.SetOperation; +import io.trino.hogql.parser.tree.HogQlQuery.SortItem; +import io.trino.hogql.parser.tree.HogQlQuery.Star; +import io.trino.hogql.parser.tree.HogQlQuery.StarReplacement; +import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; +import io.trino.hogql.parser.tree.HogQlQuery.SubscriptExpression; +import io.trino.hogql.parser.tree.HogQlQuery.TablePlaceholder; +import io.trino.hogql.parser.tree.HogQlQuery.TableReference; +import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; +import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.UnnestRelation; +import io.trino.hogql.parser.tree.HogQlQuery.ValuesRelation; +import io.trino.hogql.parser.tree.HogQlQuery.Window; +import io.trino.hogql.parser.tree.HogQlQuery.WindowDefinition; +import io.trino.hogql.parser.tree.HogQlQuery.WindowFrame; +import io.trino.hogql.parser.tree.HogQlQuery.WindowReference; +import io.trino.hogql.parser.tree.HogQlQuery.WindowSpecification; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +final class HogQlSelectAliasRewriter +{ + private HogQlSelectAliasRewriter() {} + + public static HogQlQuery rewrite(HogQlQuery query) + { + List commonTables = query.with().stream() + .map(commonTable -> new CommonTableExpression( + commonTable.name(), + commonTable.columnAliases(), + rewrite(commonTable.query()), + commonTable.span())) + .toList(); + return switch (query.body()) { + case SelectQueryBody select -> rewriteSelect(query, commonTables, select); + case SetOperation set -> new HogQlQuery( + commonTables, + new SetOperation( + set.type(), + set.distinct(), + rewrite(set.left()), + rewrite(set.right()), + set.leftParenthesized(), + set.rightParenthesized(), + set.operatorSpan(), + set.span()), + query.orderBy().stream().map(item -> rewriteSortItem(item, Map.of())).toList(), + query.limit().map(expression -> rewriteExpression(expression, Map.of())), + query.offset().map(expression -> rewriteExpression(expression, Map.of())), + query.span()); + }; + } + + private static HogQlQuery rewriteSelect(HogQlQuery query, List commonTables, SelectQueryBody select) + { + Map aliases = new HashMap<>(); + Set ambiguous = new HashSet<>(); + List projections = select.projections().stream() + .map(projection -> rewriteProjection(projection, aliases, ambiguous)) + .toList(); + Map visibleAliases = Map.copyOf(aliases); + return new HogQlQuery( + commonTables, + new SelectQueryBody( + select.distinct(), + projections, + select.from().map(HogQlSelectAliasRewriter::rewriteRelation), + select.where().map(expression -> rewriteExpression(expression, visibleAliases)), + select.groupBy().stream().map(expression -> rewriteExpression(expression, visibleAliases)).toList(), + select.having().map(expression -> rewriteExpression(expression, visibleAliases)), + select.windows().stream().map(window -> rewriteWindowDefinition(window, visibleAliases)).toList(), + select.limitBy().map(limitBy -> new LimitBy( + rewriteExpression(limitBy.limit(), visibleAliases), + limitBy.offset().map(expression -> rewriteExpression(expression, visibleAliases)), + limitBy.partitionBy().stream().map(expression -> rewriteExpression(expression, visibleAliases)).toList(), + limitBy.span())), + select.span()), + query.orderBy().stream().map(item -> rewriteSortItem(item, visibleAliases)).toList(), + query.limit().map(expression -> rewriteExpression(expression, visibleAliases)), + query.offset().map(expression -> rewriteExpression(expression, visibleAliases)), + query.span()); + } + + private static Projection rewriteProjection(Projection projection, Map aliases, Set ambiguous) + { + return switch (projection) { + case ColumnsList columns -> new ColumnsList( + columns.expressions().stream().map(expression -> rewriteExpression(expression, aliases)).toList(), + columns.span()); + case ColumnsRegex columns -> columns; + case ExpressionProjection expression -> { + Expression rewritten = rewriteExpression(expression.expression(), aliases); + expression.alias().ifPresent(alias -> registerAlias(alias, rewritten, aliases, ambiguous)); + yield new ExpressionProjection(rewritten, expression.alias()); + } + case Star star -> new Star( + star.qualifier(), + star.exclusions(), + star.replacements().stream() + .map(replacement -> new StarReplacement( + rewriteExpression(replacement.expression(), aliases), + replacement.target(), + replacement.span())) + .toList(), + star.span()); + }; + } + + private static void registerAlias(Identifier alias, Expression expression, Map aliases, Set ambiguous) + { + AliasKey key = AliasKey.of(alias); + if (!ambiguous.add(key)) { + aliases.remove(key); + return; + } + aliases.put(key, expression); + } + + private static Relation rewriteRelation(Relation relation) + { + return switch (relation) { + case AliasedRelation alias -> new AliasedRelation(rewriteRelation(alias.relation()), alias.alias(), alias.columnAliases(), alias.span()); + case CommonTableReference commonTable -> commonTable; + case JoinRelation join -> new JoinRelation( + join.type(), + rewriteRelation(join.left()), + rewriteRelation(join.right()), + join.criteria().map(criteria -> switch (criteria) { + case JoinOn on -> new JoinOn(rewriteExpression(on.expression(), Map.of()), on.span()); + case JoinUsing using -> using; + }), + join.span()); + case PivotRelation pivot -> new PivotRelation( + rewriteRelation(pivot.input()), + pivot.aggregations().stream() + .map(aggregation -> new PivotAggregation( + rewriteExpression(aggregation.expression(), Map.of()), + aggregation.alias(), + aggregation.span())) + .toList(), + pivot.pivotColumns().stream().map(expression -> rewriteExpression(expression, Map.of())).toList(), + pivot.valueGroups().stream() + .map(group -> new PivotValueGroup( + group.values().stream().map(expression -> rewriteExpression(expression, Map.of())).toList(), + group.alias(), + group.span())) + .toList(), + pivot.groupBy().stream().map(expression -> rewriteExpression(expression, Map.of())).toList(), + pivot.span()); + case SubqueryRelation subquery -> new SubqueryRelation(rewrite(subquery.query()), subquery.span()); + case TablePlaceholder table -> table; + case TableReference table -> table; + case UnnestRelation unnest -> new UnnestRelation( + unnest.expressions().stream().map(expression -> rewriteExpression(expression, Map.of())).toList(), + unnest.alias(), + unnest.columnAliases(), + unnest.span()); + case ValuesRelation values -> new ValuesRelation( + values.rows().stream() + .map(row -> row.stream().map(expression -> rewriteExpression(expression, Map.of())).toList()) + .toList(), + values.span()); + }; + } + + private static Expression rewriteExpression(Expression expression, Map aliases) + { + return switch (expression) { + case ArrayExpression array -> new ArrayExpression(array.values().stream().map(value -> rewriteExpression(value, aliases)).toList(), array.span()); + case BetweenExpression between -> new BetweenExpression( + rewriteExpression(between.value(), aliases), + rewriteExpression(between.min(), aliases), + rewriteExpression(between.max(), aliases), + between.negated(), + between.predicateSpan(), + between.span()); + case BinaryExpression binary -> new BinaryExpression( + binary.operator(), + rewriteExpression(binary.left(), aliases), + rewriteExpression(binary.right(), aliases), + binary.span()); + case CaseExpression caseExpression -> new CaseExpression( + caseExpression.operand().map(value -> rewriteExpression(value, aliases)), + caseExpression.whenClauses().stream() + .map(when -> new CaseWhen( + rewriteExpression(when.operand(), aliases), + rewriteExpression(when.result(), aliases), + when.span())) + .toList(), + caseExpression.defaultValue().map(value -> rewriteExpression(value, aliases)), + caseExpression.span()); + case CastExpression cast -> new CastExpression(rewriteExpression(cast.value(), aliases), cast.type(), cast.safe(), cast.typeDialect(), cast.span()); + case ColumnReference reference -> alias(reference, aliases).orElse(reference); + case FunctionCall function -> new FunctionCall( + function.nameParts(), + function.arguments().stream().map(argument -> rewriteExpression(argument, aliases)).toList(), + function.distinct(), + function.orderBy().stream().map(item -> rewriteSortItem(item, aliases)).toList(), + function.filter().map(filter -> rewriteExpression(filter, aliases)), + function.nullTreatment(), + function.window().map(window -> rewriteWindow(window, aliases)), + function.span()); + case InCohortExpression in -> new InCohortExpression( + rewriteExpression(in.value(), aliases), + rewriteExpression(in.cohort(), aliases), + in.negated(), + in.predicateSpan(), + in.span()); + case InExpression in -> new InExpression( + rewriteExpression(in.value(), aliases), + in.values().stream().map(value -> rewriteExpression(value, aliases)).toList(), + in.negated(), + in.predicateSpan(), + in.span()); + case InSubqueryExpression in -> new InSubqueryExpression( + rewriteExpression(in.value(), aliases), + rewrite(in.query()), + in.negated(), + in.predicateSpan(), + in.span()); + case IntervalExpression interval -> new IntervalExpression(rewriteExpression(interval.value(), aliases), interval.unit(), interval.span()); + case IsNullExpression isNull -> new IsNullExpression( + rewriteExpression(isNull.value(), aliases), + isNull.negated(), + isNull.predicateSpan(), + isNull.span()); + case LambdaExpression lambda -> { + Map visibleAliases = new HashMap<>(aliases); + lambda.arguments().forEach(argument -> visibleAliases.remove(AliasKey.of(argument))); + yield new LambdaExpression(lambda.arguments(), rewriteExpression(lambda.body(), visibleAliases), lambda.span()); + } + case Literal literal -> literal; + case MemberAccessExpression member -> new MemberAccessExpression(rewriteExpression(member.base(), aliases), member.member(), member.span()); + case Placeholder placeholder -> placeholder; + case ScalarSubqueryExpression subquery -> new ScalarSubqueryExpression(rewrite(subquery.query()), subquery.span()); + case SubscriptExpression subscript -> new SubscriptExpression( + rewriteExpression(subscript.base(), aliases), + rewriteExpression(subscript.index(), aliases), + subscript.span()); + case TupleExpression tuple -> new TupleExpression(tuple.values().stream().map(value -> rewriteExpression(value, aliases)).toList(), tuple.span()); + case UnaryExpression unary -> new UnaryExpression(unary.operator(), rewriteExpression(unary.operand(), aliases), unary.span()); + }; + } + + private static Optional alias(ColumnReference reference, Map aliases) + { + if (reference.parts().size() != 1) { + return Optional.empty(); + } + return Optional.ofNullable(aliases.get(AliasKey.of(reference.parts().getFirst()))); + } + + private static SortItem rewriteSortItem(SortItem item, Map aliases) + { + return new SortItem(rewriteExpression(item.expression(), aliases), item.direction(), item.nullPlacement(), item.span()); + } + + private static WindowDefinition rewriteWindowDefinition(WindowDefinition definition, Map aliases) + { + return new WindowDefinition(definition.name(), rewriteWindowSpecification(definition.specification(), aliases), definition.span()); + } + + private static Window rewriteWindow(Window window, Map aliases) + { + return switch (window) { + case WindowReference reference -> reference; + case WindowSpecification specification -> rewriteWindowSpecification(specification, aliases); + }; + } + + private static WindowSpecification rewriteWindowSpecification(WindowSpecification window, Map aliases) + { + return new WindowSpecification( + window.partitionBy().stream().map(expression -> rewriteExpression(expression, aliases)).toList(), + window.orderBy().stream().map(item -> rewriteSortItem(item, aliases)).toList(), + window.frame().map(frame -> rewriteWindowFrame(frame, aliases)), + window.span()); + } + + private static WindowFrame rewriteWindowFrame(WindowFrame frame, Map aliases) + { + return new WindowFrame( + frame.type(), + rewriteFrameBound(frame.start(), aliases), + frame.end().map(bound -> rewriteFrameBound(bound, aliases)), + frame.span()); + } + + private static FrameBound rewriteFrameBound(FrameBound bound, Map aliases) + { + return new FrameBound(bound.type(), bound.value().map(value -> rewriteExpression(value, aliases)), bound.span()); + } + + private record AliasKey(String value, boolean delimited) + { + private static AliasKey of(Identifier identifier) + { + return new AliasKey(identifier.delimited() ? identifier.value() : identifier.value().toLowerCase(Locale.ENGLISH), identifier.delimited()); + } + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticCatalogContext.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticCatalogContext.java new file mode 100644 index 000000000000..2197b13c41a2 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticCatalogContext.java @@ -0,0 +1,30 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider; + +import static java.util.Objects.requireNonNull; + +public record HogQlSemanticCatalogContext( + PhysicalIdentifier catalog, + HogQlSemanticCatalogSnapshotProvider snapshotProvider) +{ + public HogQlSemanticCatalogContext + { + catalog = requireNonNull(catalog, "catalog is null"); + snapshotProvider = requireNonNull(snapshotProvider, "snapshotProvider is null"); + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java new file mode 100644 index 000000000000..dc151963eae0 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlSemanticResolver.java @@ -0,0 +1,2758 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.HogQlProjectionDemand.RequiredOutputs; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ActionReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ArgumentReferenceRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.CastRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.CohortReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionArgument; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FieldReferenceRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCallRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCapabilityDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionImplementation; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.JoinKey; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LazyProjectionDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LazyTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.MaterializedViewReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.OperatorRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PredicateRepresentation; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyLookupRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ReferencedField; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationMembershipRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationMembershipRepresentation; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipJoinSide; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SavedQueryReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ScopedFieldReferenceRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticOperator; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.TypedLiteral; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.VirtualTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; +import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.AliasedRelation; +import io.trino.hogql.parser.tree.HogQlQuery.ArrayExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BetweenExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CaseExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CaseWhen; +import io.trino.hogql.parser.tree.HogQlQuery.CastExpression; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsList; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsRegex; +import io.trino.hogql.parser.tree.HogQlQuery.CommonTableExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CommonTableReference; +import io.trino.hogql.parser.tree.HogQlQuery.Expression; +import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; +import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; +import io.trino.hogql.parser.tree.HogQlQuery.Identifier; +import io.trino.hogql.parser.tree.HogQlQuery.InCohortExpression; +import io.trino.hogql.parser.tree.HogQlQuery.InExpression; +import io.trino.hogql.parser.tree.HogQlQuery.InSubqueryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IntervalExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; +import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; +import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; +import io.trino.hogql.parser.tree.HogQlQuery.JoinUsing; +import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.LambdaExpression; +import io.trino.hogql.parser.tree.HogQlQuery.LimitBy; +import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; +import io.trino.hogql.parser.tree.HogQlQuery.PivotAggregation; +import io.trino.hogql.parser.tree.HogQlQuery.PivotRelation; +import io.trino.hogql.parser.tree.HogQlQuery.PivotValueGroup; +import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; +import io.trino.hogql.parser.tree.HogQlQuery.Projection; +import io.trino.hogql.parser.tree.HogQlQuery.Relation; +import io.trino.hogql.parser.tree.HogQlQuery.ScalarSubqueryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.SelectQueryBody; +import io.trino.hogql.parser.tree.HogQlQuery.SetOperation; +import io.trino.hogql.parser.tree.HogQlQuery.SortItem; +import io.trino.hogql.parser.tree.HogQlQuery.Star; +import io.trino.hogql.parser.tree.HogQlQuery.StarReplacement; +import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; +import io.trino.hogql.parser.tree.HogQlQuery.SubscriptExpression; +import io.trino.hogql.parser.tree.HogQlQuery.TablePlaceholder; +import io.trino.hogql.parser.tree.HogQlQuery.TableReference; +import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; +import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.UnnestRelation; +import io.trino.hogql.parser.tree.HogQlQuery.ValuesRelation; +import io.trino.hogql.parser.tree.HogQlQuery.Window; +import io.trino.hogql.parser.tree.HogQlQuery.WindowDefinition; +import io.trino.hogql.parser.tree.HogQlQuery.WindowFrame; +import io.trino.hogql.parser.tree.HogQlQuery.WindowReference; +import io.trino.hogql.parser.tree.HogQlQuery.WindowSpecification; +import io.trino.re2j.Pattern; +import io.trino.re2j.PatternSyntaxException; +import io.trino.spi.Location; +import io.trino.spi.TrinoException; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static io.airlift.slice.Slices.utf8Slice; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_COMPILER_LIMIT_EXCEEDED; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_RESOLUTION_ERROR; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE; +import static java.util.Objects.requireNonNull; + +final class HogQlSemanticResolver +{ + private static final String MATCHES_ACTION = "matchesaction"; + private static final String CTE_RELATION_PREFIX = "\0cte:"; + private static final String DERIVED_RELATION_PREFIX = "\0derived:"; + + private final PinnedSnapshot snapshot; + private final ExpansionBudget expansionBudget; + private final Optional requiredOutputs; + private final List outerBindingScopes; + private final Map commonTableBindings; + private final Map relationshipPaths = new LinkedHashMap<>(); + private List bindings = List.of(); + private Set localRelationQualifiers = Set.of(); + private boolean allRelationsLogical; + private Optional expandedRelation = Optional.empty(); + private final Set lambdaArguments = new HashSet<>(); + private int generatedRelationId; + + private HogQlSemanticResolver(PinnedSnapshot snapshot) + { + this(snapshot, new ExpansionBudget(), Optional.empty()); + } + + private HogQlSemanticResolver(PinnedSnapshot snapshot, ExpansionBudget expansionBudget) + { + this(snapshot, expansionBudget, Optional.empty()); + } + + private HogQlSemanticResolver(PinnedSnapshot snapshot, ExpansionBudget expansionBudget, Optional requiredOutputs) + { + this(snapshot, expansionBudget, requiredOutputs, List.of()); + } + + private HogQlSemanticResolver( + PinnedSnapshot snapshot, + ExpansionBudget expansionBudget, + Optional requiredOutputs, + List outerBindingScopes) + { + this(snapshot, expansionBudget, requiredOutputs, outerBindingScopes, Map.of()); + } + + private HogQlSemanticResolver( + PinnedSnapshot snapshot, + ExpansionBudget expansionBudget, + Optional requiredOutputs, + List outerBindingScopes, + Map commonTableBindings) + { + this.snapshot = requireNonNull(snapshot, "snapshot is null"); + this.expansionBudget = requireNonNull(expansionBudget, "expansionBudget is null"); + this.requiredOutputs = requireNonNull(requiredOutputs, "requiredOutputs is null"); + this.outerBindingScopes = List.copyOf(requireNonNull(outerBindingScopes, "outerBindingScopes is null")); + this.commonTableBindings = Map.copyOf(requireNonNull(commonTableBindings, "commonTableBindings is null")); + } + + public static Optional resolve(PinnedSnapshot snapshot, HogQlQuery query) + { + requireNonNull(snapshot, "snapshot is null"); + requireNonNull(query, "query is null"); + HogQlSemanticResolver resolver = new HogQlSemanticResolver(snapshot); + HogQlQuery resolved = resolver.resolveNestedQuery(query); + return resolved.equals(query) ? Optional.empty() : Optional.of(new ResolvedQuery(resolved)); + } + + private HogQlQuery resolveNestedQuery(HogQlQuery query) + { + if (!query.with().isEmpty()) { + return resolveWithQuery(query); + } + List commonTables = List.of(); + if (query.body() instanceof SetOperation setOperation) { + Optional branchDemand = requiredOutputs.map(outputs -> { + SetOperationOutputs inferredOutputs = inferSetOperationOutputs(setOperation, commonTableBindings); + RequiredOutputs effectiveOutputs = query.orderBy().isEmpty() ? outputs : RequiredOutputs.allOutputs(); + return remapSetOperationDemand(inferredOutputs, effectiveOutputs); + }); + SetOperation resolved = new SetOperation( + setOperation.type(), + setOperation.distinct(), + new HogQlSemanticResolver(snapshot, expansionBudget, branchDemand.map(SetBranchDemand::left), outerBindingScopes, commonTableBindings).resolveNestedQuery(setOperation.left()), + new HogQlSemanticResolver(snapshot, expansionBudget, branchDemand.map(SetBranchDemand::right), outerBindingScopes, commonTableBindings).resolveNestedQuery(setOperation.right()), + setOperation.leftParenthesized(), + setOperation.rightParenthesized(), + setOperation.operatorSpan(), + setOperation.span()); + return new HogQlQuery(commonTables, resolved, query.orderBy(), query.limit(), query.offset(), query.span()); + } + SelectQueryBody select = (SelectQueryBody) query.body(); + localRelationQualifiers = select.from().map(HogQlSemanticResolver::relationQualifiers).orElse(Set.of()); + HogQlProjectionDemand projectionDemand = requiredOutputs + .map(outputs -> HogQlProjectionDemand.collect(query, outputs)) + .orElseGet(() -> HogQlProjectionDemand.collect(query)); + Optional relation = select.from().map(value -> resolveRelation(value, projectionDemand)); + bindings = relation.map(ResolvedRelation::bindings).orElse(List.of()); + allRelationsLogical = relation.map(ResolvedRelation::allLogical).orElse(false); + expandedRelation = relation.map(ResolvedRelation::relation); + if (bindings.isEmpty() && outerBindingScopes.isEmpty()) { + return new HogQlQuery( + commonTables, + new SelectQueryBody( + select.distinct(), + select.projections(), + relation.map(ResolvedRelation::relation), + select.where(), + select.groupBy(), + select.having(), + select.windows(), + select.limitBy(), + select.span()), + query.orderBy(), + query.limit(), + query.offset(), + query.span()); + } + return resolveQuery(query, commonTables); + } + + private HogQlQuery resolveWithQuery(HogQlQuery query) + { + Map inferredBindings = new LinkedHashMap<>(commonTableBindings); + List analyses = new ArrayList<>(); + for (CommonTableExpression commonTable : query.with()) { + CteAnalysis analysis = inferCommonTable(commonTable, inferredBindings); + analyses.add(analysis); + inferredBindings.put(canonical(commonTable.name().value()), analysis.binding()); + } + + HogQlQuery body = new HogQlQuery(List.of(), query.body(), query.orderBy(), query.limit(), query.offset(), query.span()); + Map demands = new HashMap<>(); + if (body.body() instanceof SetOperation) { + collectQueryOutputDemands(body, requiredOutputs.orElseGet(RequiredOutputs::allOutputs), inferredBindings, demands); + } + else { + collectCommonTableDemands(body, HogQlProjectionDemand.collect(body), inferredBindings, demands); + } + for (int index = analyses.size() - 1; index >= 0; index--) { + CteAnalysis analysis = analyses.get(index); + String name = canonical(analysis.commonTable().name().value()); + RequiredOutputs externalDemand = demands.getOrDefault(name, new RequiredOutputs(false, Set.of())); + RequiredOutputs sourceDemand = remapAliasedOutputDemand( + analysis.outputs(), + analysis.commonTable().columnAliases(), + externalDemand).sourceOutputs(); + collectQueryOutputDemands(analysis.commonTable().query(), sourceDemand, inferredBindings, demands); + } + + Map resolvedBindings = new LinkedHashMap<>(commonTableBindings); + List resolvedCommonTables = new ArrayList<>(); + for (CteAnalysis analysis : analyses) { + CommonTableExpression commonTable = analysis.commonTable(); + RequiredOutputs externalDemand = demands.getOrDefault(canonical(commonTable.name().value()), new RequiredOutputs(false, Set.of())); + AliasedOutputDemand demand = remapAliasedOutputDemand(analysis.outputs(), commonTable.columnAliases(), externalDemand); + HogQlQuery resolved = new HogQlSemanticResolver( + snapshot, + expansionBudget, + Optional.of(demand.sourceOutputs()), + List.of(), + resolvedBindings) + .resolveNestedQuery(commonTable.query()); + resolvedCommonTables.add(new CommonTableExpression(commonTable.name(), demand.outputAliases(), resolved, commonTable.span())); + resolvedBindings.put(canonical(commonTable.name().value()), analysis.binding()); + } + + HogQlQuery resolvedBody = new HogQlSemanticResolver( + snapshot, + expansionBudget, + requiredOutputs, + outerBindingScopes, + resolvedBindings) + .resolveNestedQuery(body); + return new HogQlQuery( + resolvedCommonTables, + resolvedBody.body(), + resolvedBody.orderBy(), + resolvedBody.limit(), + resolvedBody.offset(), + query.span()); + } + + private CteAnalysis inferCommonTable(CommonTableExpression commonTable, Map availableBindings) + { + List outputs = inferQueryOutputs(commonTable.query(), commonTable.columnAliases(), commonTable.span(), availableBindings); + TableBinding binding = inferredBinding(CTE_RELATION_PREFIX, commonTable.name(), outputs); + return new CteAnalysis(commonTable, binding, outputs); + } + + private List inferQueryOutputs( + HogQlQuery query, + List columnAliases, + HogQlQuery.SourceSpan span, + Map availableBindings) + { + if (!query.with().isEmpty()) { + throw cteInferenceError(span, "HogQL inferred relation outputs do not support branch-local WITH clauses"); + } + List outputs = switch (query.body()) { + case SelectQueryBody select -> inferSelectOutputs(select, availableBindings, !columnAliases.isEmpty()); + case SetOperation setOperation -> inferSetOperationOutputs(setOperation, availableBindings).outputs(); + }; + outputs = new ArrayList<>(outputs); + if (!columnAliases.isEmpty()) { + if (columnAliases.size() != outputs.size()) { + throw cteInferenceError(span, "HogQL inferred relation column alias count does not match its output count"); + } + for (int index = 0; index < outputs.size(); index++) { + CteOutput output = outputs.get(index); + outputs.set(index, new CteOutput(columnAliases.get(index).value(), output.sourceName(), output.sourceDemand())); + } + } + Set names = new HashSet<>(); + for (CteOutput output : outputs) { + if (!names.add(canonical(output.name()))) { + throw cteInferenceError(span, "HogQL inferred relation output names must be unique: " + output.name()); + } + } + return List.copyOf(outputs); + } + + private List inferSelectOutputs( + SelectQueryBody select, + Map availableBindings, + boolean hasColumnAliases) + { + List sourceBindings = select.from() + .map(relation -> inferRelationBindings(relation, availableBindings)) + .orElse(List.of()); + List outputs = new ArrayList<>(); + for (Projection projection : select.projections()) { + outputs.addAll(inferProjectionOutputs(projection, sourceBindings, hasColumnAliases)); + } + return List.copyOf(outputs); + } + + private SetOperationOutputs inferSetOperationOutputs(SetOperation setOperation, Map availableBindings) + { + List left = inferQueryOutputs(setOperation.left(), List.of(), setOperation.left().span(), availableBindings); + List right = inferQueryOutputs(setOperation.right(), List.of(), setOperation.right().span(), availableBindings); + if (left.size() != right.size()) { + throw cteInferenceError(setOperation.operatorSpan(), "HogQL set operation branches have incompatible output arity"); + } + List outputs = java.util.stream.IntStream.range(0, left.size()) + .mapToObj(index -> new CteOutput( + left.get(index).name(), + Optional.of(left.get(index).name()), + left.get(index).sourceDemand().merge(right.get(index).sourceDemand()))) + .toList(); + return new SetOperationOutputs(outputs, left, right); + } + + private static SetBranchDemand remapSetOperationDemand(SetOperationOutputs outputs, RequiredOutputs demand) + { + if (demand.all()) { + return new SetBranchDemand(RequiredOutputs.allOutputs(), RequiredOutputs.allOutputs()); + } + Set left = new HashSet<>(); + Set right = new HashSet<>(); + for (int index = 0; index < outputs.outputs().size(); index++) { + if (demand.includes(outputs.outputs().get(index).name())) { + left.add(canonical(outputs.left().get(index).name())); + right.add(canonical(outputs.right().get(index).name())); + } + } + return new SetBranchDemand(new RequiredOutputs(false, left), new RequiredOutputs(false, right)); + } + + private static AliasedOutputDemand remapAliasedOutputDemand( + List outputs, + List columnAliases, + RequiredOutputs demand) + { + if (columnAliases.isEmpty() || demand.all()) { + return new AliasedOutputDemand(demand, columnAliases); + } + List selectedIndexes = java.util.stream.IntStream.range(0, outputs.size()) + .filter(index -> demand.includes(outputs.get(index).name())) + .boxed() + .toList(); + if (selectedIndexes.stream().anyMatch(index -> outputs.get(index).sourceName().isEmpty())) { + return new AliasedOutputDemand(RequiredOutputs.allOutputs(), columnAliases); + } + Set sourceNames = selectedIndexes.stream() + .map(index -> canonical(outputs.get(index).sourceName().orElseThrow())) + .collect(java.util.stream.Collectors.toSet()); + boolean overlapsUnselectedOutput = java.util.stream.IntStream.range(0, outputs.size()) + .filter(index -> !selectedIndexes.contains(index)) + .mapToObj(index -> outputs.get(index).sourceName()) + .flatMap(Optional::stream) + .map(HogQlSemanticResolver::canonical) + .anyMatch(sourceNames::contains); + if (overlapsUnselectedOutput) { + return new AliasedOutputDemand(RequiredOutputs.allOutputs(), columnAliases); + } + List selectedAliases = selectedIndexes.stream() + .map(columnAliases::get) + .toList(); + return new AliasedOutputDemand(new RequiredOutputs(false, sourceNames), selectedAliases); + } + + private static TableBinding inferredBinding(String relationPrefix, Identifier name, List outputs) + { + List fields = outputs.stream() + .map(output -> new BoundField(output.name(), new PhysicalIdentifier(output.name(), true), true, Optional.empty())) + .toList(); + return new TableBinding( + relationPrefix + canonical(name.value()), + canonical(name.value()), + new PhysicalIdentifier(name.value(), name.delimited()), + List.of(new PhysicalIdentifier(name.value(), name.delimited())), + false, + fields, + TableBinding.fieldMap(fields)); + } + + private List inferRelationBindings(Relation relation, Map availableBindings) + { + return switch (relation) { + case AliasedRelation alias -> { + if (alias.relation() instanceof SubqueryRelation subquery) { + List outputs = inferQueryOutputs(subquery.query(), alias.columnAliases(), alias.span(), availableBindings); + yield List.of(inferredBinding(DERIVED_RELATION_PREFIX, alias.alias(), outputs)); + } + yield inferRelationBindings(alias.relation(), availableBindings).stream() + .map(binding -> binding.withAlias(alias.alias())) + .toList(); + } + case CommonTableReference commonTable -> Optional.ofNullable(availableBindings.get(canonical(commonTable.name().value()))) + .map(List::of) + .orElse(List.of()); + case JoinRelation join -> { + List bindings = new ArrayList<>(inferRelationBindings(join.left(), availableBindings)); + bindings.addAll(inferRelationBindings(join.right(), availableBindings)); + yield List.copyOf(bindings); + } + case PivotRelation _ -> List.of(); + case TableReference table -> table.parts().size() == 1 + ? snapshot.logicalTable(table.parts().getFirst().value()) + .map(definition -> resolveLogicalTable(definition, table.span()).bindings()) + .orElse(List.of()) + : List.of(); + case SubqueryRelation subquery -> { + List outputs = inferQueryOutputs(subquery.query(), List.of(), subquery.span(), availableBindings); + Identifier anonymousName = new Identifier("__hogql_derived_" + subquery.span().startOffset(), true, subquery.span()); + yield List.of(inferredBinding(DERIVED_RELATION_PREFIX, anonymousName, outputs)); + } + case TablePlaceholder _, UnnestRelation _, ValuesRelation _ -> List.of(); + }; + } + + private List inferProjectionOutputs(Projection projection, List sourceBindings, boolean hasColumnAliases) + { + return switch (projection) { + case ExpressionProjection expression -> { + Optional name = expression.alias().map(Identifier::value); + if (name.isEmpty() && expression.expression() instanceof ColumnReference reference) { + name = Optional.of(reference.parts().getLast().value()); + } + if (name.isEmpty() && !hasColumnAliases) { + throw cteInferenceError(expression.span(), "Cannot infer HogQL CTE output name; add a projection alias or CTE column alias list"); + } + yield List.of(new CteOutput(name.orElse("__hogql_cte_output"), name, HogQlProjectionDemand.collect(expression.expression()))); + } + case ColumnsList columns -> columns.expressions().stream() + .map(expression -> { + if (!(expression instanceof ColumnReference reference)) { + throw cteInferenceError(expression.span(), "Cannot infer HogQL CTE COLUMNS output name"); + } + String name = reference.parts().getLast().value(); + return new CteOutput(name, Optional.of(name), HogQlProjectionDemand.collect(expression)); + }) + .toList(); + case ColumnsRegex columns -> inferRegexOutputs(columns, sourceBindings); + case Star star -> inferStarOutputs(star, sourceBindings); + }; + } + + private List inferRegexOutputs(ColumnsRegex columns, List sourceBindings) + { + Pattern pattern; + try { + pattern = Pattern.compile(columns.pattern()); + } + catch (PatternSyntaxException _) { + throw semanticEntityError(HOGQL_RESOLUTION_ERROR, columns.patternSpan(), "Invalid HogQL COLUMNS regex: " + columns.pattern()); + } + List outputs = sourceBindings.stream() + .flatMap(binding -> binding.orderedFields().stream() + .filter(BoundField::starVisible) + .filter(field -> pattern.find(utf8Slice(field.name()))) + .map(field -> new CteOutput(field.name(), Optional.of(field.name()), HogQlProjectionDemand.column(binding.qualifier(), field.name())))) + .toList(); + if (outputs.isEmpty()) { + throw cteInferenceError(columns.patternSpan(), "No HogQL CTE fields matched COLUMNS regex: " + columns.pattern()); + } + return outputs; + } + + private List inferStarOutputs(Star star, List sourceBindings) + { + Optional lazyStar = inferLazyStar(star, sourceBindings); + if (lazyStar.isPresent()) { + return lazyStar.orElseThrow().definition().projections().stream() + .filter(LazyProjectionDefinition::starVisible) + .filter(projection -> star.exclusions().stream().noneMatch(exclusion -> matchesIdentifier(exclusion.parts().getLast(), projection.name()))) + .map(projection -> new CteOutput( + projection.name(), + Optional.of(projection.name()), + star.replacements().stream() + .filter(replacement -> matchesIdentifier(replacement.target(), projection.name())) + .findFirst() + .map(replacement -> HogQlProjectionDemand.collect(replacement.expression())) + .orElseGet(HogQlProjectionDemand::preserveNone))) + .toList(); + } + List starBindings = star.qualifier().isEmpty() + ? sourceBindings + : sourceBindings.stream().filter(binding -> matchesStarQualifier(binding, star.qualifier())).toList(); + if (starBindings.isEmpty()) { + throw cteInferenceError(star.span(), "Cannot infer HogQL CTE star outputs from an unknown relation schema"); + } + Set exclusions = resolveStarExclusions(star, starBindings); + Map replacements = resolveStarReplacements(star, starBindings, exclusions); + return starBindings.stream() + .flatMap(binding -> binding.orderedFields().stream() + .filter(BoundField::starVisible) + .filter(field -> !exclusions.contains(new StarField(binding, field))) + .map(field -> { + StarReplacement replacement = replacements.get(new StarField(binding, field)); + HogQlProjectionDemand demand = replacement == null + ? HogQlProjectionDemand.column(binding.qualifier(), field.name()) + : HogQlProjectionDemand.collect(replacement.expression()); + return new CteOutput(field.name(), Optional.of(field.name()), demand); + })) + .toList(); + } + + private Optional inferLazyStar(Star star, List sourceBindings) + { + if (star.qualifier().isEmpty() || star.qualifier().size() > 2) { + return Optional.empty(); + } + List matches; + if (star.qualifier().size() == 1) { + String lazyName = star.qualifier().getFirst().value(); + matches = sourceBindings.stream() + .flatMap(binding -> lazyTable(binding, lazyName).stream().map(definition -> new LazyStar(binding, definition))) + .toList(); + } + else { + Identifier owner = star.qualifier().getFirst(); + String lazyName = star.qualifier().getLast().value(); + matches = sourceBindings.stream() + .filter(binding -> matchesStarQualifier(binding, List.of(owner))) + .flatMap(binding -> lazyTable(binding, lazyName).stream().map(definition -> new LazyStar(binding, definition))) + .toList(); + } + if (matches.size() > 1) { + throw cteInferenceError(star.span(), "Ambiguous HogQL CTE lazy star qualifier: " + starQualifier(star)); + } + return matches.stream().findFirst(); + } + + private void collectCommonTableDemands( + HogQlQuery query, + HogQlProjectionDemand demand, + Map availableBindings, + Map demands) + { + if (query.body() instanceof SelectQueryBody select) { + select.from().ifPresent(relation -> collectCommonTableDemands(relation, demand, availableBindings, demands)); + } + } + + private void collectQueryOutputDemands( + HogQlQuery query, + RequiredOutputs outputDemand, + Map availableBindings, + Map demands) + { + if (query.body() instanceof SetOperation setOperation) { + SetOperationOutputs outputs = inferSetOperationOutputs(setOperation, availableBindings); + RequiredOutputs effectiveDemand = query.orderBy().isEmpty() ? outputDemand : RequiredOutputs.allOutputs(); + SetBranchDemand branchDemand = remapSetOperationDemand(outputs, effectiveDemand); + collectQueryOutputDemands(setOperation.left(), branchDemand.left(), availableBindings, demands); + collectQueryOutputDemands(setOperation.right(), branchDemand.right(), availableBindings, demands); + return; + } + List outputs = inferQueryOutputs(query, List.of(), query.span(), availableBindings); + HogQlProjectionDemand sourceDemand = HogQlProjectionDemand.collectNonProjection(query); + for (CteOutput output : outputs) { + if (outputDemand.includes(output.name())) { + sourceDemand = sourceDemand.merge(output.sourceDemand()); + } + } + collectCommonTableDemands(query, sourceDemand, availableBindings, demands); + } + + private void collectCommonTableDemands( + Relation relation, + HogQlProjectionDemand demand, + Map availableBindings, + Map demands) + { + switch (relation) { + case AliasedRelation alias -> { + if (alias.relation() instanceof SubqueryRelation subquery) { + List outputs = inferQueryOutputs(subquery.query(), alias.columnAliases(), alias.span(), availableBindings); + RequiredOutputs sourceDemand = remapAliasedOutputDemand( + outputs, + alias.columnAliases(), + demand.forAlias(alias.alias())).sourceOutputs(); + collectQueryOutputDemands(subquery.query(), sourceDemand, availableBindings, demands); + } + else if (alias.relation() instanceof CommonTableReference commonTable) { + addCommonTableDemand(commonTable, demand.forAlias(alias.alias()), availableBindings, demands); + } + else { + collectCommonTableDemands(alias.relation(), demand, availableBindings, demands); + } + } + case CommonTableReference commonTable -> addCommonTableDemand(commonTable, demand.forAlias(commonTable.name()), availableBindings, demands); + case JoinRelation join -> { + collectCommonTableDemands(join.left(), demand, availableBindings, demands); + collectCommonTableDemands(join.right(), demand, availableBindings, demands); + } + case PivotRelation pivot -> collectCommonTableDemands( + pivot.input(), + HogQlProjectionDemand.preserveAll(), + availableBindings, + demands); + case SubqueryRelation _, TablePlaceholder _, TableReference _, UnnestRelation _, ValuesRelation _ -> {} + } + } + + private static void addCommonTableDemand( + CommonTableReference commonTable, + RequiredOutputs demand, + Map availableBindings, + Map demands) + { + String name = canonical(commonTable.name().value()); + TableBinding binding = availableBindings.get(name); + if (binding == null || !binding.relationName().startsWith(CTE_RELATION_PREFIX)) { + return; + } + demands.merge(name, demand, RequiredOutputs::merge); + } + + private HogQlQuery resolveQuery(HogQlQuery query, List commonTables) + { + Optional projectionOutputs = requiredOutputs + .map(outputs -> outputs.merge(HogQlProjectionDemand.collectOrderingOutputs(query))); + List projections = new ArrayList<>(); + query.projections().forEach(projection -> projections.addAll(resolveProjection(projection, projectionOutputs))); + if (projections.isEmpty() && requiredOutputs.isPresent()) { + projections.add(new ExpressionProjection( + new Literal(HogQlQuery.LiteralKind.INTEGER, "1", query.span()), + Optional.of(new Identifier("__hogql_pruned", true, query.span())))); + } + Optional where = query.where().map(this::resolveExpression); + List groupBy = query.groupBy().stream().map(this::resolveExpression).toList(); + Optional having = query.having().map(this::resolveExpression); + List windows = query.windows().stream().map(this::resolveWindowDefinition).toList(); + Optional limitBy = query.limitBy().map(clause -> new LimitBy( + resolveExpression(clause.limit()), + clause.offset().map(this::resolveExpression), + clause.partitionBy().stream().map(this::resolveExpression).toList(), + clause.span())); + List orderBy = resolveSortItems(query.orderBy()); + Optional limit = query.limit().map(this::resolveExpression); + Optional offset = query.offset().map(this::resolveExpression); + return new HogQlQuery( + commonTables, + new SelectQueryBody( + query.distinct(), + projections, + expandedRelation, + where, + groupBy, + having, + windows, + limitBy, + query.body().span()), + orderBy, + limit, + offset, + query.span()); + } + + private List resolveProjection(Projection projection, Optional projectionOutputs) + { + return switch (projection) { + case ColumnsList columns -> resolveColumnsList(columns, projectionOutputs); + case ColumnsRegex columns -> resolveColumnsRegex(columns, projectionOutputs); + case Star star -> resolveStar(star, projectionOutputs); + case ExpressionProjection expressionProjection -> { + Optional outputName = expressionProjection.alias().map(Identifier::value); + if (outputName.isEmpty() && expressionProjection.expression() instanceof ColumnReference reference) { + outputName = Optional.of(reference.parts().getLast().value()); + } + if (outputName.isPresent() && !projectionDemanded(outputName.orElseThrow(), projectionOutputs)) { + yield List.of(); + } + Expression resolved = resolveExpression(expressionProjection.expression()); + Optional alias = expressionProjection.alias(); + if (alias.isEmpty() && expressionProjection.expression() instanceof ColumnReference reference) { + alias = Optional.of(reference.parts().getLast()); + } + yield List.of(new ExpressionProjection(resolved, alias)); + } + }; + } + + private List resolveColumnsList(ColumnsList columns, Optional projectionOutputs) + { + return columns.expressions().stream() + .flatMap(expression -> resolveProjection(new ExpressionProjection(expression, Optional.empty()), projectionOutputs).stream()) + .toList(); + } + + private List resolveColumnsRegex(ColumnsRegex columns, Optional projectionOutputs) + { + if (!allRelationsLogical) { + throw unsupportedColumns(columns.span()); + } + Pattern pattern; + try { + pattern = Pattern.compile(columns.pattern()); + } + catch (PatternSyntaxException _) { + throw semanticEntityError(HOGQL_RESOLUTION_ERROR, columns.patternSpan(), "Invalid HogQL COLUMNS regex: " + columns.pattern()); + } + List projections = bindings.stream() + .flatMap(binding -> binding.orderedFields().stream() + .filter(BoundField::starVisible) + .filter(field -> pattern.find(utf8Slice(field.name()))) + .filter(field -> projectionDemanded(field.name(), projectionOutputs)) + .map(field -> new ExpressionProjection( + resolveBoundField(binding, field, binding.starQualifier(bindings.size()), columns.span(), expansionBudget), + Optional.of(new Identifier(field.name(), true, columns.span()))))) + .map(Projection.class::cast) + .toList(); + if (projections.isEmpty()) { + throw semanticEntityError(HOGQL_RESOLUTION_ERROR, columns.patternSpan(), "No HogQL fields matched COLUMNS regex: " + columns.pattern()); + } + return projections; + } + + private List resolveStar(Star star, Optional projectionOutputs) + { + Optional> lazyStar = resolveLazyStar(star, projectionOutputs); + if (lazyStar.isPresent()) { + return lazyStar.orElseThrow(); + } + + List starBindings; + boolean qualified = !star.qualifier().isEmpty(); + if (qualified) { + starBindings = bindings.stream() + .filter(binding -> matchesStarQualifier(binding, star.qualifier())) + .toList(); + if (starBindings.size() > 1) { + throw starResolutionError(star.qualifier().getFirst(), "Ambiguous HogQL star qualifier: " + starQualifier(star)); + } + if (starBindings.isEmpty()) { + if (allRelationsLogical) { + throw starResolutionError(star.qualifier().getFirst(), "Unknown HogQL star qualifier: " + starQualifier(star)); + } + return List.of(star); + } + } + else { + if (!allRelationsLogical) { + return List.of(star); + } + starBindings = bindings; + } + + Set exclusions = resolveStarExclusions(star, starBindings); + Map replacements = resolveStarReplacements(star, starBindings, exclusions); + return starBindings.stream() + .flatMap(binding -> binding.orderedFields().stream() + .filter(BoundField::starVisible) + .filter(field -> !exclusions.contains(new StarField(binding, field))) + .filter(field -> projectionDemanded(field.name(), projectionOutputs)) + .map(field -> { + StarReplacement replacement = replacements.get(new StarField(binding, field)); + Expression expression = replacement == null + ? resolveBoundField( + binding, + field, + qualified ? Optional.of(binding.outputQualifier()) : binding.starQualifier(bindings.size()), + star.span(), + expansionBudget) + : resolveExpression(replacement.expression()); + return new ExpressionProjection(expression, Optional.of(new Identifier(field.name(), true, star.span()))); + })) + .map(Projection.class::cast) + .toList(); + } + + private Optional> resolveLazyStar(Star star, Optional projectionOutputs) + { + if (star.qualifier().isEmpty() || star.qualifier().size() > 2) { + return Optional.empty(); + } + + List matches; + if (star.qualifier().size() == 1) { + String lazyName = star.qualifier().getFirst().value(); + matches = bindings.stream() + .flatMap(binding -> lazyTable(binding, lazyName) + .map(definition -> java.util.stream.Stream.of(new LazyStar(binding, definition))) + .orElseGet(java.util.stream.Stream::empty)) + .toList(); + } + else { + Identifier owner = star.qualifier().getFirst(); + String lazyName = star.qualifier().getLast().value(); + matches = bindings.stream() + .filter(binding -> matchesStarQualifier(binding, List.of(owner))) + .flatMap(binding -> lazyTable(binding, lazyName) + .map(definition -> java.util.stream.Stream.of(new LazyStar(binding, definition))) + .orElseGet(java.util.stream.Stream::empty)) + .toList(); + } + if (matches.isEmpty()) { + return Optional.empty(); + } + if (matches.size() > 1) { + throw starResolutionError(star.qualifier().getFirst(), "Ambiguous HogQL star qualifier: " + starQualifier(star)); + } + + LazyStar lazyStar = matches.getFirst(); + List visible = lazyStar.definition().projections().stream() + .filter(LazyProjectionDefinition::starVisible) + .toList(); + Set exclusions = new HashSet<>(); + for (ColumnReference exclusion : star.exclusions()) { + Identifier name = exclusion.parts().getLast(); + LazyProjectionDefinition matched = visible.stream() + .filter(projection -> matchesIdentifier(name, projection.name())) + .findFirst() + .orElseThrow(() -> starResolutionError(exclusion.span(), "Unknown HogQL star exclusion: " + identifierPath(exclusion.parts()))); + if (!exclusions.add(canonical(matched.name()))) { + throw starResolutionError(exclusion.span(), "Duplicate HogQL star exclusion: " + identifierPath(exclusion.parts())); + } + } + Map replacements = new HashMap<>(); + for (StarReplacement replacement : star.replacements()) { + LazyProjectionDefinition matched = visible.stream() + .filter(projection -> !exclusions.contains(canonical(projection.name()))) + .filter(projection -> matchesIdentifier(replacement.target(), projection.name())) + .findFirst() + .orElseThrow(() -> starResolutionError(replacement.target(), "Unknown HogQL star replacement: " + replacement.target().value())); + if (replacements.putIfAbsent(canonical(matched.name()), replacement) != null) { + throw starResolutionError(replacement.target(), "Duplicate HogQL star replacement: " + replacement.target().value()); + } + } + + return Optional.of(visible.stream() + .filter(projection -> !exclusions.contains(canonical(projection.name()))) + .filter(projection -> projectionDemanded(projection.name(), projectionOutputs)) + .map(projection -> { + StarReplacement replacement = replacements.get(canonical(projection.name())); + Expression expression = replacement == null + ? resolveLazyProjection(lazyStar.binding(), lazyStar.definition(), projection, star.span()) + : resolveExpression(replacement.expression()); + return new ExpressionProjection(expression, Optional.of(new Identifier(projection.name(), true, star.span()))); + }) + .map(Projection.class::cast) + .toList()); + } + + private Expression resolveLazyProjection( + TableBinding owner, + LazyTableDefinition definition, + LazyProjectionDefinition projection, + HogQlQuery.SourceSpan span) + { + TableBinding terminal = ensureRelationshipPath(owner, definition.relationshipPath(), span); + return expandRecipe( + terminal, + projection.recipe(), + Optional.of(terminal.outputQualifier()), + span, + expansionBudget); + } + + private static boolean projectionDemanded(String name, Optional projectionOutputs) + { + return projectionOutputs.map(outputs -> outputs.includes(name)).orElse(true); + } + + private static Map resolveStarReplacements(Star star, List starBindings, Set exclusions) + { + Map replacements = new HashMap<>(); + for (StarReplacement replacement : star.replacements()) { + List matchedFields = starBindings.stream() + .flatMap(binding -> binding.orderedFields().stream() + .filter(BoundField::starVisible) + .map(field -> new StarField(binding, field))) + .filter(field -> !exclusions.contains(field)) + .filter(field -> matchesIdentifier(replacement.target(), field.field().name())) + .distinct() + .toList(); + if (matchedFields.isEmpty()) { + throw starResolutionError(replacement.target(), "Unknown HogQL star replacement: " + replacement.target().value()); + } + if (matchedFields.stream().anyMatch(replacements::containsKey)) { + throw starResolutionError(replacement.target(), "Duplicate HogQL star replacement: " + replacement.target().value()); + } + matchedFields.forEach(field -> replacements.put(field, replacement)); + } + return replacements; + } + + private static Set resolveStarExclusions(Star star, List starBindings) + { + Set exclusions = new HashSet<>(); + for (ColumnReference exclusion : star.exclusions()) { + List parts = exclusion.parts(); + Identifier fieldName = parts.getLast(); + List exclusionBindings = parts.size() == 1 ? starBindings : starBindings.stream() + .filter(binding -> matchesStarQualifier(binding, parts.subList(0, parts.size() - 1))) + .toList(); + if (parts.size() > 1 && exclusionBindings.size() > 1) { + throw starResolutionError(exclusion.span(), "Ambiguous HogQL star exclusion qualifier: " + identifierPath(parts.subList(0, parts.size() - 1))); + } + List matchedFields = exclusionBindings.stream() + .flatMap(binding -> binding.orderedFields().stream() + .filter(BoundField::starVisible) + .filter(field -> matchesIdentifier(fieldName, field.name())) + .map(field -> new StarField(binding, field))) + .distinct() + .toList(); + if (matchedFields.isEmpty()) { + throw starResolutionError(exclusion.span(), "Unknown HogQL star exclusion: " + identifierPath(parts)); + } + if (matchedFields.stream().anyMatch(exclusions::contains)) { + throw starResolutionError(exclusion.span(), "Duplicate HogQL star exclusion: " + identifierPath(parts)); + } + exclusions.addAll(matchedFields); + } + return exclusions; + } + + private static boolean matchesStarQualifier(TableBinding binding, List qualifier) + { + if (binding.aliased()) { + return qualifier.size() == 1 && matchesIdentifier(qualifier.getFirst(), binding.outputQualifier()); + } + if (qualifier.size() == 1 && canonicalIdentifier(qualifier.getFirst().value(), qualifier.getFirst().delimited()).equals(canonical(binding.relationName()))) { + return true; + } + if (qualifier.size() > binding.physicalQualifier().size()) { + return false; + } + int offset = binding.physicalQualifier().size() - qualifier.size(); + for (int index = 0; index < qualifier.size(); index++) { + if (!matchesIdentifier(qualifier.get(index), binding.physicalQualifier().get(offset + index))) { + return false; + } + } + return !qualifier.isEmpty(); + } + + private static boolean matchesIdentifier(Identifier identifier, String value) + { + return identifier.delimited() ? identifier.value().equals(value) : canonical(identifier.value()).equals(canonical(value)); + } + + private static boolean matchesIdentifier(Identifier identifier, PhysicalIdentifier value) + { + return canonicalIdentifier(identifier.value(), identifier.delimited()).equals(canonicalIdentifier(value.value(), value.delimited())); + } + + private static String canonicalIdentifier(String value, boolean delimited) + { + return delimited ? value : canonical(value); + } + + private static String starQualifier(Star star) + { + return identifierPath(star.qualifier()); + } + + private static String identifierPath(List identifiers) + { + return String.join(".", identifiers.stream().map(Identifier::value).toList()); + } + + private static Set relationQualifiers(Relation relation) + { + return switch (relation) { + case AliasedRelation alias -> Set.of(canonical(alias.alias().value())); + case CommonTableReference commonTable -> Set.of(canonical(commonTable.name().value())); + case JoinRelation join -> { + Set qualifiers = new HashSet<>(relationQualifiers(join.left())); + qualifiers.addAll(relationQualifiers(join.right())); + yield Set.copyOf(qualifiers); + } + case PivotRelation _ -> Set.of(); + case SubqueryRelation _ -> Set.of(); + case TablePlaceholder _ -> Set.of(); + case TableReference table -> Set.of(canonical(table.parts().getLast().value())); + case UnnestRelation unnest -> Set.of(canonical(unnest.alias().value())); + case ValuesRelation _ -> Set.of(); + }; + } + + private ResolvedRelation resolveRelation(Relation relation) + { + return resolveRelation(relation, HogQlProjectionDemand.preserveAll()); + } + + private ResolvedRelation resolveRelation(Relation relation, HogQlProjectionDemand projectionDemand) + { + return switch (relation) { + case AliasedRelation alias -> { + ResolvedRelation child; + if (alias.relation() instanceof SubqueryRelation subquery) { + List outputs = inferQueryOutputs(subquery.query(), alias.columnAliases(), alias.span(), commonTableBindings); + AliasedOutputDemand demand = remapAliasedOutputDemand(outputs, alias.columnAliases(), projectionDemand.forAlias(alias.alias())); + child = resolveSubquery(subquery, demand.sourceOutputs()); + yield new ResolvedRelation( + new AliasedRelation(child.relation(), alias.alias(), demand.outputAliases(), alias.span()), + List.of(inferredBinding(DERIVED_RELATION_PREFIX, alias.alias(), outputs)), + true); + } + else { + child = resolveRelation(alias.relation(), HogQlProjectionDemand.preserveAll()); + } + List aliasedBindings = child.bindings().stream() + .map(binding -> binding.withAlias(alias.alias())) + .toList(); + yield new ResolvedRelation( + new AliasedRelation(child.relation(), alias.alias(), alias.columnAliases(), alias.span()), + aliasedBindings, + child.allLogical()); + } + case CommonTableReference commonTable -> Optional.ofNullable(commonTableBindings.get(canonical(commonTable.name().value()))) + .map(binding -> new ResolvedRelation(commonTable, List.of(binding), true)) + .orElseGet(() -> new ResolvedRelation(commonTable, List.of(), false)); + case JoinRelation join -> { + ResolvedRelation left = resolveRelation(join.left(), projectionDemand); + List bindingsBeforeRight = bindings; + boolean allRelationsLogicalBeforeRight = allRelationsLogical; + if (join.right() instanceof UnnestRelation) { + bindings = left.bindings(); + allRelationsLogical = left.allLogical(); + } + ResolvedRelation right; + try { + right = resolveRelation(join.right(), projectionDemand); + } + finally { + bindings = bindingsBeforeRight; + allRelationsLogical = allRelationsLogicalBeforeRight; + } + List joinBindings = new ArrayList<>(left.bindings()); + joinBindings.addAll(right.bindings()); + List previousBindings = bindings; + boolean previousAllRelationsLogical = allRelationsLogical; + bindings = List.copyOf(joinBindings); + allRelationsLogical = left.allLogical() && right.allLogical(); + Optional criteria = join.criteria().map(value -> switch (value) { + case JoinOn on -> new JoinOn(resolveExpression(on.expression()), on.span()); + case JoinUsing using -> resolveJoinUsing(using, left.bindings(), right.bindings()); + }); + bindings = previousBindings; + allRelationsLogical = previousAllRelationsLogical; + yield new ResolvedRelation( + new JoinRelation(join.type(), left.relation(), right.relation(), criteria, join.span()), + joinBindings, + left.allLogical() && right.allLogical()); + } + case PivotRelation pivot -> { + ResolvedRelation input = resolveRelation(pivot.input(), HogQlProjectionDemand.preserveAll()); + List previousBindings = bindings; + boolean previousAllRelationsLogical = allRelationsLogical; + bindings = input.bindings(); + allRelationsLogical = input.allLogical(); + List aggregations = pivot.aggregations().stream() + .map(aggregation -> new PivotAggregation( + resolveExpression(aggregation.expression()), + aggregation.alias(), + aggregation.span())) + .toList(); + List pivotColumns = pivot.pivotColumns().stream() + .map(this::resolveExpression) + .toList(); + List valueGroups = pivot.valueGroups().stream() + .map(group -> new PivotValueGroup( + group.values().stream().map(this::resolveExpression).toList(), + group.alias(), + group.span())) + .toList(); + List groupBy = pivot.groupBy().stream() + .map(this::resolveExpression) + .toList(); + bindings = previousBindings; + allRelationsLogical = previousAllRelationsLogical; + yield new ResolvedRelation( + new PivotRelation(input.relation(), aggregations, pivotColumns, valueGroups, groupBy, pivot.span()), + List.of(), + false); + } + case SubqueryRelation subquery -> resolveSubquery(subquery, projectionDemand.unqualified()); + case TablePlaceholder placeholder -> new ResolvedRelation(placeholder, List.of(), false); + case TableReference table -> resolveTable(table); + case UnnestRelation unnest -> new ResolvedRelation( + new UnnestRelation( + unnest.expressions().stream().map(this::resolveExpression).toList(), + unnest.alias(), + unnest.columnAliases(), + unnest.span()), + List.of(), + false); + case ValuesRelation values -> new ResolvedRelation(values, List.of(), false); + }; + } + + private ResolvedRelation resolveSubquery(SubqueryRelation subquery, RequiredOutputs requiredOutputs) + { + return new ResolvedRelation( + new SubqueryRelation( + new HogQlSemanticResolver(snapshot, expansionBudget, Optional.of(requiredOutputs), List.of(), commonTableBindings).resolveNestedQuery(subquery.query()), + subquery.span()), + List.of(), + false); + } + + private JoinUsing resolveJoinUsing(JoinUsing using, List leftBindings, List rightBindings) + { + List columns = using.columns().stream() + .map(column -> { + List leftFields = matchingFields(leftBindings, column.value()); + List rightFields = matchingFields(rightBindings, column.value()); + if (leftFields.size() != 1 || rightFields.size() != 1) { + throw resolutionError(new ColumnReference(List.of(column), column.span()), column.value()); + } + PhysicalIdentifier left = leftFields.getFirst().physicalColumn(); + PhysicalIdentifier right = rightFields.getFirst().physicalColumn(); + if (!left.equals(right)) { + throw incompatibleUsingResolutionError(column); + } + return new Identifier(left.value(), left.delimited(), column.span()); + }) + .toList(); + return new JoinUsing(columns, using.span()); + } + + private static List matchingFields(List tableBindings, String name) + { + return tableBindings.stream() + .map(TableBinding::fields) + .map(fields -> fields.get(canonical(name))) + .filter(field -> field != null) + .toList(); + } + + private ResolvedRelation resolveTable(TableReference table) + { + if (table.parts().size() != 1) { + return new ResolvedRelation(table, List.of(), false); + } + String name = table.parts().getFirst().value(); + return resolveSemanticRelation(name, table.span(), new RelationExpansionBudget()) + .orElseGet(() -> new ResolvedRelation(table, List.of(), false)); + } + + private Optional resolveSemanticRelation(String name, HogQlQuery.SourceSpan span, RelationExpansionBudget budget) + { + budget.enter(span); + try { + Optional logicalTable = snapshot.logicalTable(name); + if (logicalTable.isPresent()) { + return Optional.of(resolveLogicalTable(logicalTable.orElseThrow(), span)); + } + Optional materializedView = snapshot.snapshot().materializedViews().stream() + .filter(view -> canonical(view.name()).equals(canonical(name))) + .findFirst(); + if (materializedView.isPresent()) { + return Optional.of(resolveMaterializedView(materializedView.orElseThrow(), span)); + } + Optional virtualTable = snapshot.snapshot().virtualTables().stream() + .filter(table -> canonical(table.name()).equals(canonical(name))) + .findFirst(); + if (virtualTable.isPresent()) { + return Optional.of(resolveVirtualTable(virtualTable.orElseThrow(), span, budget)); + } + Optional savedQuery = snapshot.snapshot().savedQueries().stream() + .filter(query -> canonical(query.name()).equals(canonical(name))) + .findFirst(); + if (savedQuery.isPresent()) { + return Optional.of(resolveSavedQuery(savedQuery.orElseThrow(), span, budget)); + } + return Optional.empty(); + } + finally { + budget.exit(); + } + } + + private ResolvedRelation resolveLogicalTable(LogicalTableDefinition definition, HogQlQuery.SourceSpan span) + { + TableReference physicalTable = new TableReference( + List.of( + identifier(definition.physicalTable().catalog(), span), + identifier(definition.physicalTable().schema(), span), + identifier(definition.physicalTable().table(), span)), + span); + List fields = new ArrayList<>(); + definition.fields().forEach(field -> fields.add(new BoundField(field.name(), field.physicalColumn(), field.starVisible(), Optional.empty()))); + snapshot.snapshot().expressionFields().stream() + .filter(field -> canonical(field.table()).equals(canonical(definition.name()))) + .forEach(field -> fields.add(new BoundField( + field.name(), + new PhysicalIdentifier(field.name(), true), + field.starVisible(), + Optional.of(field)))); + return new ResolvedRelation( + physicalTable, + List.of(tableBinding( + definition, + canonical(definition.name()), + definition.physicalTable().table(), + List.of(definition.physicalTable().catalog(), definition.physicalTable().schema(), definition.physicalTable().table()), + false, + fields)), + true); + } + + private ResolvedRelation resolveMaterializedView(MaterializedViewReference definition, HogQlQuery.SourceSpan span) + { + TableReference physicalTable = new TableReference( + List.of( + identifier(definition.physicalView().catalog(), span), + identifier(definition.physicalView().schema(), span), + identifier(definition.physicalView().table(), span)), + span); + List fields = definition.fields().stream() + .map(field -> referencedField(field, Optional.empty())) + .toList(); + return new ResolvedRelation( + physicalTable, + List.of(new TableBinding( + definition.name(), + canonical(definition.name()), + new PhysicalIdentifier(definition.physicalView().table().value(), definition.physicalView().table().delimited()), + List.of(definition.physicalView().catalog(), definition.physicalView().schema(), definition.physicalView().table()), + false, + fields, + TableBinding.fieldMap(fields))), + true); + } + + private ResolvedRelation resolveVirtualTable(VirtualTableDefinition definition, HogQlQuery.SourceSpan span, RelationExpansionBudget budget) + { + ResolvedRelation source = resolveRelationReference(definition.source(), span, budget); + List projections = definition.projections().stream() + .map(projection -> new ProjectedField(projection.name(), projection.sourceField(), projection.starVisible())) + .toList(); + return projectRelation(definition.name(), source, projections, span); + } + + private ResolvedRelation resolveSavedQuery(SavedQueryReference definition, HogQlQuery.SourceSpan span, RelationExpansionBudget budget) + { + ResolvedRelation source = resolveRelationReference(definition.target(), span, budget); + List projections = definition.fields().stream() + .map(field -> new ProjectedField(field.name(), field.name(), field.starVisible())) + .toList(); + return projectRelation(definition.name(), source, projections, span); + } + + private ResolvedRelation resolveRelationReference(RelationReference reference, HogQlQuery.SourceSpan span, RelationExpansionBudget budget) + { + ResolvedRelation relation = resolveSemanticRelation(reference.name(), span, budget) + .orElseThrow(() -> expansionError(span, "HogQL semantic relation references an unavailable target")); + RelationKind actualKind = semanticRelationKind(reference.name()); + if (actualKind != reference.kind()) { + throw expansionError(span, "HogQL semantic relation target kind does not match the catalog"); + } + return relation; + } + + private RelationKind semanticRelationKind(String name) + { + if (snapshot.logicalTable(name).isPresent()) { + return RelationKind.LOGICAL_TABLE; + } + if (snapshot.snapshot().virtualTables().stream().anyMatch(table -> canonical(table.name()).equals(canonical(name)))) { + return RelationKind.VIRTUAL_TABLE; + } + if (snapshot.snapshot().savedQueries().stream().anyMatch(query -> canonical(query.name()).equals(canonical(name)))) { + return RelationKind.SAVED_QUERY; + } + return RelationKind.MATERIALIZED_VIEW; + } + + private ResolvedRelation projectRelation(String relationName, ResolvedRelation source, List projectedFields, HogQlQuery.SourceSpan span) + { + List previousBindings = bindings; + boolean previousAllRelationsLogical = allRelationsLogical; + bindings = source.bindings(); + allRelationsLogical = source.allLogical(); + List projections; + try { + projections = projectedFields.stream() + .map(field -> new ExpressionProjection( + resolveColumn(new ColumnReference(List.of(new Identifier(field.sourceField(), true, span)), span)), + Optional.of(new Identifier(field.name(), true, span)))) + .map(Projection.class::cast) + .toList(); + } + finally { + bindings = previousBindings; + allRelationsLogical = previousAllRelationsLogical; + } + HogQlQuery projectedQuery = new HogQlQuery( + List.of(), + false, + projections, + Optional.of(source.relation()), + Optional.empty(), + List.of(), + Optional.empty(), + List.of(), + Optional.empty(), + Optional.empty(), + span); + List fields = projectedFields.stream() + .map(field -> new BoundField( + field.name(), + new PhysicalIdentifier(field.name(), true), + field.starVisible(), + Optional.empty())) + .toList(); + return new ResolvedRelation( + new SubqueryRelation(projectedQuery, span), + List.of(new TableBinding( + relationName, + canonical(relationName), + new PhysicalIdentifier(relationName, true), + List.of(), + false, + fields, + TableBinding.fieldMap(fields))), + true); + } + + private static BoundField referencedField(ReferencedField field, Optional expression) + { + return new BoundField(field.name(), new PhysicalIdentifier(field.name(), true), field.starVisible(), expression); + } + + private List resolveSortItems(List sortItems) + { + return sortItems.stream() + .map(sortItem -> new SortItem(resolveExpression(sortItem.expression()), sortItem.direction(), sortItem.nullPlacement(), sortItem.span())) + .toList(); + } + + private HogQlSemanticResolver correlatedSubqueryResolver() + { + List visibleOuterScopes = new ArrayList<>(); + currentBindingScope().ifPresent(visibleOuterScopes::add); + visibleOuterScopes.addAll(outerBindingScopes); + return new HogQlSemanticResolver(snapshot, expansionBudget, Optional.empty(), visibleOuterScopes, commonTableBindings); + } + + private Optional currentBindingScope() + { + if (bindings.isEmpty() && localRelationQualifiers.isEmpty()) { + return Optional.empty(); + } + return Optional.of(new BindingScope(bindings, localRelationQualifiers, allRelationsLogical)); + } + + private List visibleBindingScopes() + { + List scopes = new ArrayList<>(); + currentBindingScope().ifPresent(scopes::add); + scopes.addAll(outerBindingScopes); + return List.copyOf(scopes); + } + + private Expression resolveExpression(Expression expression) + { + return switch (expression) { + case ArrayExpression array -> new ArrayExpression(array.values().stream().map(this::resolveExpression).toList(), array.span()); + case BetweenExpression between -> new BetweenExpression( + resolveExpression(between.value()), + resolveExpression(between.min()), + resolveExpression(between.max()), + between.negated(), + between.predicateSpan(), + between.span()); + case BinaryExpression binary -> new BinaryExpression( + binary.operator(), + resolveExpression(binary.left()), + resolveExpression(binary.right()), + binary.span()); + case CaseExpression caseExpression -> new CaseExpression( + caseExpression.operand().map(this::resolveExpression), + caseExpression.whenClauses().stream() + .map(when -> new CaseWhen(resolveExpression(when.operand()), resolveExpression(when.result()), when.span())) + .toList(), + caseExpression.defaultValue().map(this::resolveExpression), + caseExpression.span()); + case CastExpression cast -> new CastExpression(resolveExpression(cast.value()), cast.type(), cast.safe(), cast.typeDialect(), cast.span()); + case ColumnReference reference -> resolveColumn(reference); + case FunctionCall function -> resolveFunctionExpression(function); + case InCohortExpression in -> resolveCohortExpression(in); + case InExpression in -> new InExpression( + resolveExpression(in.value()), + in.values().stream().map(this::resolveExpression).toList(), + in.negated(), + in.predicateSpan(), + in.span()); + case InSubqueryExpression in -> new InSubqueryExpression( + resolveExpression(in.value()), + correlatedSubqueryResolver().resolveNestedQuery(in.query()), + in.negated(), + in.predicateSpan(), + in.span()); + case IntervalExpression interval -> new IntervalExpression(resolveExpression(interval.value()), interval.unit(), interval.span()); + case IsNullExpression isNull -> new IsNullExpression( + resolveExpression(isNull.value()), + isNull.negated(), + isNull.predicateSpan(), + isNull.span()); + case LambdaExpression lambda -> resolveLambda(lambda); + case Literal literal -> literal; + case MemberAccessExpression memberAccess -> resolveMemberAccess(memberAccess); + case Placeholder placeholder -> placeholder; + case ScalarSubqueryExpression subquery -> new ScalarSubqueryExpression( + correlatedSubqueryResolver().resolveNestedQuery(subquery.query()), + subquery.span()); + case SubscriptExpression subscript -> resolveSubscript(subscript); + case TupleExpression tuple -> new TupleExpression(tuple.values().stream().map(this::resolveExpression).toList(), tuple.span()); + case UnaryExpression unary -> new UnaryExpression(unary.operator(), resolveExpression(unary.operand()), unary.span()); + }; + } + + private LambdaExpression resolveLambda(LambdaExpression lambda) + { + Set previous = Set.copyOf(lambdaArguments); + lambda.arguments().stream() + .map(Identifier::value) + .map(HogQlSemanticResolver::canonical) + .forEach(lambdaArguments::add); + try { + return new LambdaExpression(lambda.arguments(), resolveExpression(lambda.body()), lambda.span()); + } + finally { + lambdaArguments.clear(); + lambdaArguments.addAll(previous); + } + } + + private Expression resolveFunctionExpression(FunctionCall function) + { + if (function.nameParts().size() == 1 && canonical(function.name().value()).equals(MATCHES_ACTION)) { + return resolveActionExpression(function); + } + return new FunctionCall( + function.nameParts(), + function.arguments().stream().map(this::resolveExpression).toList(), + function.distinct(), + resolveSortItems(function.orderBy()), + function.filter().map(this::resolveExpression), + function.nullTreatment(), + function.window().map(this::resolveWindow), + function.span()); + } + + private Expression resolveActionExpression(FunctionCall function) + { + if (function.arguments().size() != 1) { + throw semanticEntityError(HOGQL_RESOLUTION_ERROR, function.span(), "HogQL matchesAction requires exactly one argument"); + } + if (function.distinct() || !function.orderBy().isEmpty() || function.filter().isPresent() || function.nullTreatment().isPresent() || function.window().isPresent()) { + throw semanticEntityError(HOGQL_UNSUPPORTED_FEATURE, function.span(), "HogQL matchesAction does not support invocation modifiers"); + } + EntityLookup lookup = entityLookup(function.arguments().getFirst(), "action"); + List matches = snapshot.snapshot().actions().stream() + .filter(action -> lookup.matches(action.name(), action.actionId())) + .toList(); + ActionReference action = requireEntity(matches, "action", function.span()); + TableBinding binding = requireEntityBinding(action.table(), "action", function.span()); + return switch (action.representation()) { + case PredicateRepresentation predicate -> expandRecipe( + binding, + predicate.predicate(), + Optional.of(binding.outputQualifier()), + function.span(), + expansionBudget); + case RelationMembershipRepresentation membership -> membershipExpression(binding, membership.relation(), false, function.span()); + }; + } + + private Expression resolveCohortExpression(InCohortExpression in) + { + EntityLookup lookup = entityLookup(in.cohort(), "cohort"); + List matches = snapshot.snapshot().cohorts().stream() + .filter(cohort -> lookup.matches(cohort.name(), cohort.cohortId())) + .toList(); + CohortReference cohort = requireEntity(matches, "cohort", in.span()); + TableBinding binding = requireEntityBinding(cohort.table(), "cohort", in.span()); + return switch (cohort.representation()) { + case PredicateRepresentation predicate -> { + requireCohortSource(in.value(), binding, in.span()); + Expression expanded = expandRecipe( + binding, + predicate.predicate(), + Optional.of(binding.outputQualifier()), + in.span(), + expansionBudget); + yield in.negated() ? new UnaryExpression(HogQlQuery.UnaryOperator.NOT, expanded, in.span()) : expanded; + } + case RelationMembershipRepresentation membership -> { + requireMembershipSource(in.value(), binding, membership.relation(), in.span()); + yield membershipExpression(binding, membership.relation(), in.negated(), in.span()); + } + }; + } + + private void requireCohortSource(Expression value, TableBinding binding, HogQlQuery.SourceSpan span) + { + if (!(value instanceof ColumnReference reference)) { + throw unsupportedExpansion(span, "HogQL cohort membership source must be a declared field reference"); + } + List parts = reference.parts(); + boolean qualified = parts.size() == 2 && binding.qualifier().equals(canonical(parts.getFirst().value())); + boolean matches = (parts.size() == 1 || qualified) && binding.fields().containsKey(canonical(parts.getLast().value())); + if (!matches) { + throw unsupportedExpansion(span, "HogQL cohort membership source does not match the catalog table"); + } + } + + private InSubqueryExpression membershipExpression( + TableBinding sourceBinding, + RelationMembershipRecipe membership, + boolean negated, + HogQlQuery.SourceSpan span) + { + expansionBudget.enter(span); + try { + BoundField sourceField = Optional.ofNullable(sourceBinding.fields().get(canonical(membership.sourceField()))) + .orElseThrow(() -> expansionError(span, "HogQL semantic entity references an unavailable source field")); + Expression source = resolveBoundField( + sourceBinding, + sourceField, + sourceBinding.starQualifier(bindings.size()), + span, + expansionBudget); + ResolvedRelation target = resolveRelationReference(membership.relation(), span, new RelationExpansionBudget()); + if (target.bindings().size() != 1) { + throw unsupportedExpansion(span, "HogQL semantic entity membership requires a single target relation"); + } + TableBinding targetBinding = target.bindings().getFirst(); + BoundField targetField = Optional.ofNullable(targetBinding.fields().get(canonical(membership.targetField()))) + .orElseThrow(() -> expansionError(span, "HogQL semantic entity references an unavailable target field")); + Expression targetValue = resolveBoundField( + targetBinding, + targetField, + targetBinding.starQualifier(1), + span, + expansionBudget); + HogQlQuery membershipQuery = new HogQlQuery( + List.of(), + true, + List.of(new ExpressionProjection(targetValue, Optional.empty())), + Optional.of(target.relation()), + Optional.empty(), + List.of(), + Optional.empty(), + List.of(), + Optional.empty(), + Optional.empty(), + span); + return new InSubqueryExpression(source, membershipQuery, negated, span, span); + } + finally { + expansionBudget.exit(); + } + } + + private void requireMembershipSource(Expression value, TableBinding binding, RelationMembershipRecipe membership, HogQlQuery.SourceSpan span) + { + if (!(value instanceof ColumnReference reference)) { + throw unsupportedExpansion(span, "HogQL cohort membership source must be a declared field reference"); + } + List parts = reference.parts(); + boolean matches = switch (parts.size()) { + case 1 -> canonical(parts.getFirst().value()).equals(canonical(membership.sourceField())); + case 2 -> binding.qualifier().equals(canonical(parts.getFirst().value())) && + canonical(parts.getLast().value()).equals(canonical(membership.sourceField())); + default -> false; + }; + if (!matches) { + throw unsupportedExpansion(span, "HogQL cohort membership source does not match the catalog"); + } + } + + private TableBinding requireEntityBinding(String tableName, String kind, HogQlQuery.SourceSpan span) + { + List matches = bindings.stream() + .filter(binding -> canonical(binding.relationName()).equals(canonical(tableName))) + .toList(); + if (matches.size() != 1) { + throw semanticEntityError( + HOGQL_RESOLUTION_ERROR, + span, + "HogQL " + kind + " requires exactly one " + tableName + " relation in scope"); + } + return matches.getFirst(); + } + + private EntityLookup entityLookup(Expression expression, String kind) + { + if (!(expression instanceof Literal literal) || (literal.kind() != HogQlQuery.LiteralKind.STRING && literal.kind() != HogQlQuery.LiteralKind.INTEGER)) { + throw semanticEntityError( + HOGQL_UNSUPPORTED_FEATURE, + expression.span(), + "HogQL " + kind + " reference must be a string or integer literal"); + } + return new EntityLookup(literal.value(), literal.kind() == HogQlQuery.LiteralKind.INTEGER); + } + + private static T requireEntity(List matches, String kind, HogQlQuery.SourceSpan span) + { + if (matches.isEmpty()) { + throw semanticEntityError(HOGQL_RESOLUTION_ERROR, span, "Unknown HogQL " + kind); + } + if (matches.size() > 1) { + throw semanticEntityError(HOGQL_RESOLUTION_ERROR, span, "Ambiguous HogQL " + kind); + } + return matches.getFirst(); + } + + private WindowDefinition resolveWindowDefinition(WindowDefinition definition) + { + return new WindowDefinition(definition.name(), (WindowSpecification) resolveWindow(definition.specification()), definition.span()); + } + + private Window resolveWindow(Window window) + { + return switch (window) { + case WindowReference reference -> reference; + case WindowSpecification specification -> new WindowSpecification( + specification.partitionBy().stream().map(this::resolveExpression).toList(), + resolveSortItems(specification.orderBy()), + specification.frame().map(this::resolveWindowFrame), + specification.span()); + }; + } + + private WindowFrame resolveWindowFrame(WindowFrame frame) + { + return new WindowFrame( + frame.type(), + new HogQlQuery.FrameBound( + frame.start().type(), + frame.start().value().map(this::resolveExpression), + frame.start().span()), + frame.end().map(bound -> new HogQlQuery.FrameBound( + bound.type(), + bound.value().map(this::resolveExpression), + bound.span())), + frame.span()); + } + + private Expression resolveColumn(ColumnReference reference) + { + List parts = reference.parts(); + if (lambdaArguments.contains(canonical(parts.getFirst().value()))) { + return reference; + } + Optional semanticPath = resolveSemanticPath(reference); + if (semanticPath.isPresent()) { + return semanticPath.orElseThrow(); + } + Optional propertyAccess = resolveDottedPropertyAccess(reference); + if (propertyAccess.isPresent()) { + return propertyAccess.orElseThrow(); + } + if (parts.size() == 2) { + String qualifier = canonical(parts.getFirst().value()); + for (BindingScope scope : visibleBindingScopes()) { + List qualifiedBindings = scope.bindings().stream() + .filter(candidate -> candidate.qualifier().equals(qualifier)) + .toList(); + if (!qualifiedBindings.isEmpty()) { + TableBinding binding = qualifiedBindings.getFirst(); + BoundField field = binding.fields().get(canonical(parts.getLast().value())); + if (field == null) { + throw resolutionError(reference, parts.getLast().value()); + } + return resolveBoundField(binding, field, Optional.of(binding.outputQualifier()), reference.span(), expansionBudget); + } + if (scope.relationQualifiers().contains(qualifier)) { + return reference; + } + } + if (innermostScopeIsFullyLogical()) { + throw resolutionError(reference, parts.getLast().value()); + } + return reference; + } + if (parts.size() > 2) { + String qualifier = canonical(parts.getFirst().value()); + if (visibleBindingScopes().stream().anyMatch(scope -> scope.bindings().stream().anyMatch(binding -> binding.qualifier().equals(qualifier))) || + innermostScopeIsFullyLogical()) { + throw resolutionError(reference, parts.getLast().value()); + } + return reference; + } + String logicalName = parts.getLast().value(); + Optional currentScope = currentBindingScope(); + List scopes = visibleBindingScopes(); + for (int scopeIndex = 0; scopeIndex < scopes.size(); scopeIndex++) { + BindingScope scope = scopes.get(scopeIndex); + List matches = scope.bindings().stream() + .map(binding -> new FieldMatch(binding, binding.fields().get(canonical(logicalName)))) + .filter(match -> match.field() != null) + .toList(); + if (matches.size() > 1) { + throw ambiguousResolutionError(reference, logicalName); + } + if (matches.size() == 1) { + if (!scope.allRelationsLogical() && scope.bindings().size() > 1) { + return reference; + } + FieldMatch match = matches.getFirst(); + boolean outer = currentScope.isEmpty() || scopeIndex > 0; + Optional qualifier = outer || scope.bindings().size() > 1 + ? Optional.of(match.binding().outputQualifier()) + : Optional.empty(); + return resolveBoundField(match.binding(), match.field(), qualifier, reference.span(), expansionBudget); + } + if (!scope.allRelationsLogical() && !scope.relationQualifiers().isEmpty()) { + return reference; + } + } + if (scopes.stream().anyMatch(BindingScope::allRelationsLogical)) { + throw resolutionError(reference, logicalName); + } + return reference; + } + + private boolean innermostScopeIsFullyLogical() + { + return visibleBindingScopes().stream() + .findFirst() + .map(BindingScope::allRelationsLogical) + .orElse(false); + } + + private Optional resolveSemanticPath(ColumnReference reference) + { + List parts = reference.parts(); + if (parts.size() < 2) { + return Optional.empty(); + } + + List candidates = new ArrayList<>(); + List qualifiedBindings = bindings.stream() + .filter(binding -> binding.qualifier().equals(canonical(parts.getFirst().value()))) + .toList(); + if (!qualifiedBindings.isEmpty()) { + if (qualifiedBindings.size() == 1 && isSemanticPathMember(qualifiedBindings.getFirst(), parts.get(1).value())) { + candidates.add(new PathCandidate(qualifiedBindings.getFirst(), parts.subList(1, parts.size()))); + } + } + else { + bindings.stream() + .filter(binding -> isSemanticPathMember(binding, parts.getFirst().value())) + .map(binding -> new PathCandidate(binding, parts)) + .forEach(candidates::add); + } + if (candidates.isEmpty()) { + return Optional.empty(); + } + if (candidates.size() > 1) { + throw ambiguousResolutionError(reference, parts.getFirst().value()); + } + if (expandedRelation.isEmpty()) { + throw unsupportedExpansion(reference.span(), "HogQL relationship paths are not supported inside explicit join criteria"); + } + return Optional.of(resolveSemanticPath(reference, candidates.getFirst())); + } + + private boolean isSemanticPathMember(TableBinding binding, String name) + { + return relationship(binding, name).isPresent() || lazyTable(binding, name).isPresent(); + } + + private Expression resolveSemanticPath(ColumnReference reference, PathCandidate candidate) + { + TableBinding owner = candidate.owner(); + List path = candidate.path(); + Optional lazyTable = lazyTable(owner, path.getFirst().value()); + if (lazyTable.isPresent()) { + if (path.size() != 2) { + throw resolutionError(reference, path.getLast().value()); + } + LazyTableDefinition definition = lazyTable.orElseThrow(); + LazyProjectionDefinition projection = definition.projections().stream() + .filter(candidateProjection -> canonical(candidateProjection.name()).equals(canonical(path.getLast().value()))) + .findFirst() + .orElseThrow(() -> resolutionError(reference, path.getLast().value())); + TableBinding terminal = ensureRelationshipPath(owner, definition.relationshipPath(), reference.span()); + return expandRecipe( + terminal, + projection.recipe(), + Optional.of(terminal.outputQualifier()), + reference.span(), + expansionBudget); + } + + LogicalTableDefinition currentTable = snapshot.logicalTable(owner.relationName()).orElseThrow(); + List relationshipPath = new ArrayList<>(); + int index = 0; + while (index < path.size() - 1) { + Optional relationship = relationship(currentTable, path.get(index).value()); + if (relationship.isEmpty()) { + break; + } + RelationshipDefinition definition = relationship.orElseThrow(); + relationshipPath.add(definition.name()); + currentTable = snapshot.logicalTable(definition.targetTable()) + .orElseThrow(() -> expansionError(reference.span(), "HogQL relationship references an unavailable target table")); + index++; + } + if (relationshipPath.isEmpty()) { + return reference; + } + + TableBinding terminal = ensureRelationshipPath(owner, relationshipPath, reference.span()); + List remaining = path.subList(index, path.size()); + if (remaining.size() == 1) { + BoundField field = terminal.fields().get(canonical(remaining.getFirst().value())); + if (field == null) { + throw resolutionError(reference, remaining.getFirst().value()); + } + return resolveBoundField(terminal, field, Optional.of(terminal.outputQualifier()), reference.span(), expansionBudget); + } + if (remaining.size() == 2) { + Optional property = properties(terminal).stream() + .filter(candidateProperty -> canonical(candidateProperty.name()).equals(canonical(remaining.getFirst().value()))) + .findFirst(); + if (property.isPresent()) { + return expandProperty( + terminal, + property.orElseThrow(), + new Literal(HogQlQuery.LiteralKind.STRING, remaining.getLast().value(), remaining.getLast().span()), + Optional.of(terminal.outputQualifier()), + reference.span(), + expansionBudget); + } + } + throw resolutionError(reference, remaining.getLast().value()); + } + + private TableBinding ensureRelationshipPath(TableBinding owner, List path, HogQlQuery.SourceSpan span) + { + TableBinding source = owner; + List prefix = new ArrayList<>(); + for (String relationshipName : path) { + prefix.add(canonical(relationshipName)); + RelationshipPathKey key = new RelationshipPathKey(owner.qualifier(), prefix); + TableBinding cached = relationshipPaths.get(key); + if (cached != null) { + source = cached; + continue; + } + RelationshipDefinition relationship = relationship(source, relationshipName) + .orElseThrow(() -> expansionError(span, "HogQL lazy table references an unavailable relationship")); + LogicalTableDefinition target = snapshot.logicalTable(relationship.targetTable()) + .orElseThrow(() -> expansionError(span, "HogQL relationship references an unavailable target table")); + source = addRelationshipJoin(source, target, relationship, span); + relationshipPaths.put(key, source); + } + return source; + } + + private TableBinding addRelationshipJoin( + TableBinding source, + LogicalTableDefinition target, + RelationshipDefinition relationship, + HogQlQuery.SourceSpan span) + { + expansionBudget.add(span); + ResolvedRelation targetRelation = resolveLogicalTable(target, span); + Identifier alias = nextGeneratedRelationAlias(span); + TableBinding targetBinding = targetRelation.bindings().getFirst().withAlias(alias); + Relation aliasedTarget = new AliasedRelation(targetRelation.relation(), alias, span); + + List predicates = relationship.joinKeys().stream() + .map(joinKey -> joinKeyPredicate(source, targetBinding, joinKey, span)) + .collect(java.util.stream.Collectors.toCollection(ArrayList::new)); + relationship.joinPredicate().ifPresent(predicate -> predicates.add(expandRecipe( + source, + predicate, + Optional.of(source.outputQualifier()), + span, + expansionBudget, + Map.of(), + Map.of(RelationshipJoinSide.SOURCE, source, RelationshipJoinSide.TARGET, targetBinding)))); + Expression criteria = predicates.stream() + .reduce((left, right) -> new BinaryExpression(HogQlQuery.BinaryOperator.AND, left, right, span)) + .orElseThrow(); + expandedRelation = Optional.of(new JoinRelation( + HogQlQuery.JoinType.LEFT, + expandedRelation.orElseThrow(), + aliasedTarget, + Optional.of(new JoinOn(criteria, span)), + span)); + return targetBinding; + } + + private Expression joinKeyPredicate(TableBinding source, TableBinding target, JoinKey joinKey, HogQlQuery.SourceSpan span) + { + BoundField sourceField = Optional.ofNullable(source.fields().get(canonical(joinKey.sourceField()))) + .orElseThrow(() -> expansionError(span, "HogQL relationship references an unavailable source field")); + BoundField targetField = Optional.ofNullable(target.fields().get(canonical(joinKey.targetField()))) + .orElseThrow(() -> expansionError(span, "HogQL relationship references an unavailable target field")); + return new BinaryExpression( + HogQlQuery.BinaryOperator.EQUAL, + physicalColumn(sourceField, Optional.of(source.outputQualifier()), span), + physicalColumn(targetField, Optional.of(target.outputQualifier()), span), + span); + } + + private Identifier nextGeneratedRelationAlias(HogQlQuery.SourceSpan span) + { + while (true) { + Identifier candidate = new Identifier("__hogql_lazy_" + ++generatedRelationId, true, span); + boolean used = bindings.stream().anyMatch(binding -> binding.qualifier().equals(canonical(candidate.value()))) || + relationshipPaths.values().stream().anyMatch(binding -> binding.qualifier().equals(canonical(candidate.value()))); + if (!used) { + return candidate; + } + } + } + + private Optional relationship(TableBinding binding, String name) + { + if (isInferredRelation(binding)) { + return Optional.empty(); + } + return snapshot.logicalTable(binding.relationName()).flatMap(table -> relationship(table, name)); + } + + private static Optional relationship(LogicalTableDefinition table, String name) + { + return table.relationships().stream() + .filter(candidate -> canonical(candidate.name()).equals(canonical(name))) + .findFirst(); + } + + private Optional lazyTable(TableBinding binding, String name) + { + if (isInferredRelation(binding)) { + return Optional.empty(); + } + return snapshot.snapshot().lazyTables().stream() + .filter(candidate -> canonical(candidate.table()).equals(canonical(binding.relationName()))) + .filter(candidate -> canonical(candidate.name()).equals(canonical(name))) + .findFirst(); + } + + private Expression resolveMemberAccess(MemberAccessExpression memberAccess) + { + if (memberAccess.base() instanceof ColumnReference reference) { + Optional propertyAccess = resolvePropertyAccess( + reference, + new Literal(HogQlQuery.LiteralKind.STRING, memberAccess.member().value(), memberAccess.member().span()), + memberAccess.span()); + if (propertyAccess.isPresent()) { + return propertyAccess.orElseThrow(); + } + } + return new MemberAccessExpression( + resolveExpression(memberAccess.base()), + memberAccess.member(), + memberAccess.span()); + } + + private Expression resolveSubscript(SubscriptExpression subscript) + { + if (subscript.base() instanceof ColumnReference reference) { + Optional propertyAccess = resolvePropertyAccess( + reference, + subscript.index(), + subscript.span()); + if (propertyAccess.isPresent()) { + return propertyAccess.orElseThrow(); + } + } + return new SubscriptExpression( + resolveExpression(subscript.base()), + resolveExpression(subscript.index()), + subscript.span()); + } + + private Optional resolveDottedPropertyAccess(ColumnReference reference) + { + List parts = reference.parts(); + if (parts.size() == 2 && bindings.stream().noneMatch(binding -> binding.qualifier().equals(canonical(parts.getFirst().value())))) { + return resolvePropertyAccess( + new ColumnReference(List.of(parts.getFirst()), parts.getFirst().span()), + new Literal(HogQlQuery.LiteralKind.STRING, parts.getLast().value(), parts.getLast().span()), + reference.span()); + } + if (parts.size() == 3) { + return resolvePropertyAccess( + new ColumnReference(parts.subList(0, 2), reference.span()), + new Literal(HogQlQuery.LiteralKind.STRING, parts.getLast().value(), parts.getLast().span()), + reference.span()); + } + return Optional.empty(); + } + + private Optional resolvePropertyAccess( + ColumnReference propertyReference, + Expression key, + HogQlQuery.SourceSpan span) + { + List parts = propertyReference.parts(); + List candidateBindings; + String propertyName; + boolean qualified; + if (parts.size() == 1) { + candidateBindings = bindings; + propertyName = parts.getFirst().value(); + qualified = false; + } + else if (parts.size() == 2) { + candidateBindings = bindings.stream() + .filter(binding -> binding.qualifier().equals(canonical(parts.getFirst().value()))) + .toList(); + propertyName = parts.getLast().value(); + qualified = true; + } + else { + return Optional.empty(); + } + + List propertyMatches = candidateBindings.stream() + .flatMap(binding -> properties(binding).stream() + .filter(property -> canonical(property.name()).equals(canonical(propertyName))) + .map(property -> new PropertyMatch(binding, property))) + .toList(); + if (propertyMatches.isEmpty()) { + return Optional.empty(); + } + List matchedBindings = propertyMatches.stream() + .map(PropertyMatch::binding) + .distinct() + .toList(); + if (matchedBindings.size() > 1) { + throw ambiguousPropertyResolutionError(span, propertyName); + } + + PropertyDefinition property = propertyMatches.getFirst().property(); + TableBinding binding = matchedBindings.getFirst(); + Optional qualifier = qualified || bindings.size() > 1 + ? Optional.of(binding.outputQualifier()) + : Optional.empty(); + return Optional.of(expandProperty(binding, property, resolveExpression(key), qualifier, span, expansionBudget)); + } + + private List properties(TableBinding binding) + { + if (isInferredRelation(binding)) { + return List.of(); + } + return snapshot.logicalTable(binding.relationName()) + .map(LogicalTableDefinition::properties) + .orElse(List.of()); + } + + private static boolean isInferredRelation(TableBinding binding) + { + return binding.relationName().startsWith(CTE_RELATION_PREFIX) || binding.relationName().startsWith(DERIVED_RELATION_PREFIX); + } + + private Expression resolveBoundField( + TableBinding binding, + BoundField field, + Optional qualifier, + HogQlQuery.SourceSpan span, + ExpansionBudget budget) + { + budget.add(span); + return field.expression() + .map(expression -> expandRecipe(binding, expression.recipe(), qualifier, span, budget)) + .orElseGet(() -> physicalColumn(field, qualifier, span)); + } + + private static ColumnReference physicalColumn(BoundField field, Optional qualifier, HogQlQuery.SourceSpan span) + { + List parts = new ArrayList<>(); + qualifier.ifPresent(identifier -> parts.add(new Identifier(identifier.value(), identifier.delimited(), span))); + parts.add(new Identifier(field.physicalColumn().value(), field.physicalColumn().delimited(), span)); + return new ColumnReference(parts, span); + } + + private Expression expandRecipe( + TableBinding binding, + ExpressionRecipe recipe, + Optional qualifier, + HogQlQuery.SourceSpan span, + ExpansionBudget budget) + { + return expandRecipe(binding, recipe, qualifier, span, budget, Map.of(), Map.of()); + } + + private Expression expandRecipe( + TableBinding binding, + ExpressionRecipe recipe, + Optional qualifier, + HogQlQuery.SourceSpan span, + ExpansionBudget budget, + Map arguments) + { + return expandRecipe(binding, recipe, qualifier, span, budget, arguments, Map.of()); + } + + private Expression expandRecipe( + TableBinding binding, + ExpressionRecipe recipe, + Optional qualifier, + HogQlQuery.SourceSpan span, + ExpansionBudget budget, + Map arguments, + Map scopedBindings) + { + budget.enter(span); + try { + return switch (recipe) { + case FieldReferenceRecipe reference -> { + BoundField field = binding.fields().get(canonical(reference.field())); + if (field == null) { + throw expansionError(span, "HogQL expression recipe references an unavailable field"); + } + yield resolveBoundField(binding, field, qualifier, span, budget); + } + case LiteralRecipe literal -> typedLiteral(literal.literal(), span); + case FunctionCallRecipe function -> expandFunction(binding, function, qualifier, span, budget, arguments, scopedBindings); + case OperatorRecipe operator -> expandOperator(binding, operator, qualifier, span, budget, arguments, scopedBindings); + case CastRecipe cast -> new CastExpression( + expandRecipe(binding, cast.expression(), qualifier, span, budget, arguments, scopedBindings), + new Identifier(cast.targetTypeSignature(), false, span), + false, + span); + case ArgumentReferenceRecipe reference -> { + Expression argument = arguments.get(reference.argument()); + if (argument == null) { + throw unsupportedExpansion(span, "HogQL recipe argument is unavailable in this expansion context"); + } + yield argument; + } + case ScopedFieldReferenceRecipe reference -> { + TableBinding scopedBinding = scopedBindings.get(reference.side()); + if (scopedBinding == null) { + throw unsupportedExpansion(span, "HogQL scoped field recipe is only valid inside a relationship predicate"); + } + BoundField field = scopedBinding.fields().get(canonical(reference.field())); + if (field == null) { + throw expansionError(span, "HogQL relationship predicate references an unavailable field"); + } + yield resolveBoundField( + scopedBinding, + field, + Optional.of(scopedBinding.outputQualifier()), + span, + budget); + } + case PropertyLookupRecipe lookup -> expandPropertyLookup(binding, lookup, qualifier, span, budget, arguments, scopedBindings); + }; + } + finally { + budget.exit(); + } + } + + private Expression expandFunction( + TableBinding binding, + FunctionCallRecipe function, + Optional qualifier, + HogQlQuery.SourceSpan span, + ExpansionBudget budget, + Map arguments, + Map scopedBindings) + { + FunctionCapabilityDefinition capability = snapshot.snapshot().functions().stream() + .filter(candidate -> canonical(candidate.name()).equals(canonical(function.name()))) + .findFirst() + .orElseThrow(() -> expansionError(span, "HogQL expression recipe references an unavailable function")); + if (capability.implementation() == FunctionImplementation.REWRITE || capability.trinoName().isEmpty()) { + throw unsupportedExpansion(span, "HogQL function recipe requires an unavailable compiler rewrite"); + } + boolean supportedArity = capability.signatures().stream() + .anyMatch(signature -> signature.variadic() + ? function.arguments().size() >= Math.max(0, signature.argumentTypes().size() - 1) + : function.arguments().size() == signature.argumentTypes().size()); + if (!supportedArity) { + throw expansionError(span, "HogQL function recipe does not match a declared signature"); + } + return new FunctionCall( + capability.trinoName().stream() + .map(name -> new Identifier(name.value(), name.delimited(), span)) + .toList(), + function.arguments().stream() + .map(argument -> expandRecipe(binding, argument, qualifier, span, budget, arguments, scopedBindings)) + .toList(), + false, + List.of(), + Optional.empty(), + span); + } + + private Expression expandOperator( + TableBinding binding, + OperatorRecipe operator, + Optional qualifier, + HogQlQuery.SourceSpan span, + ExpansionBudget budget, + Map recipeArguments, + Map scopedBindings) + { + List arguments = operator.arguments().stream() + .map(argument -> expandRecipe(binding, argument, qualifier, span, budget, recipeArguments, scopedBindings)) + .toList(); + return switch (operator.operator()) { + case NOT -> new UnaryExpression(HogQlQuery.UnaryOperator.NOT, arguments.getFirst(), span); + case NEGATE -> new UnaryExpression(HogQlQuery.UnaryOperator.NEGATE, arguments.getFirst(), span); + case IS_NULL -> new IsNullExpression(arguments.getFirst(), false, span, span); + case IS_NOT_NULL -> new IsNullExpression(arguments.getFirst(), true, span, span); + case SUBSCRIPT -> new SubscriptExpression(arguments.getFirst(), arguments.getLast(), span); + case JSON_OBJECT_LOOKUP -> new SubscriptExpression( + new CastExpression( + new FunctionCall( + new Identifier("json_parse", false, span), + List.of(arguments.getFirst()), + false, + List.of(), + Optional.empty(), + span), + new Identifier("map(varchar, json)", false, span), + false, + span), + arguments.getLast(), + span); + default -> new BinaryExpression(binaryOperator(operator.operator()), arguments.getFirst(), arguments.getLast(), span); + }; + } + + private Expression expandPropertyLookup( + TableBinding binding, + PropertyLookupRecipe lookup, + Optional qualifier, + HogQlQuery.SourceSpan span, + ExpansionBudget budget, + Map arguments, + Map scopedBindings) + { + TableBinding lookupBinding = canonical(binding.relationName()).equals(canonical(lookup.table())) + ? binding + : scopedBindings.values().stream() + .filter(candidate -> canonical(candidate.relationName()).equals(canonical(lookup.table()))) + .findFirst() + .orElseThrow(() -> expansionError(span, "HogQL property lookup recipe references an unavailable table")); + Optional lookupQualifier = lookupBinding == binding + ? qualifier + : Optional.of(lookupBinding.outputQualifier()); + PropertyDefinition property = properties(lookupBinding).stream() + .filter(candidate -> canonical(candidate.name()).equals(canonical(lookup.property()))) + .findFirst() + .orElseThrow(() -> expansionError(span, "HogQL property lookup recipe references an unavailable property")); + Expression key = expandRecipe(binding, lookup.key(), qualifier, span, budget, arguments, scopedBindings); + return expandProperty(lookupBinding, property, key, lookupQualifier, span, budget); + } + + private Expression expandProperty( + TableBinding binding, + PropertyDefinition property, + Expression key, + Optional qualifier, + HogQlQuery.SourceSpan span, + ExpansionBudget budget) + { + budget.enter(span); + try { + ExpressionRecipe recipe = property.lookupRecipe() + .orElseThrow(() -> unsupportedExpansion(span, "HogQL property lookup has no declared compiler recipe")); + String keyType = property.keyTypeSignature() + .orElseThrow(() -> unsupportedExpansion(span, "HogQL property lookup has no declared key type")); + String valueType = property.valueTypeSignature() + .orElseThrow(() -> unsupportedExpansion(span, "HogQL property lookup has no declared value type")); + BoundField source = binding.fields().get(canonical(property.sourceField())); + if (source == null) { + throw expansionError(span, "HogQL property lookup references an unavailable source field"); + } + Expression sourceExpression = resolveBoundField(binding, source, qualifier, span, budget); + Expression typedKey = new CastExpression(key, new Identifier(keyType, false, key.span()), false, key.span()); + Expression value = expandRecipe( + binding, + recipe, + qualifier, + span, + budget, + Map.of( + ExpressionArgument.PROPERTY_SOURCE, sourceExpression, + ExpressionArgument.PROPERTY_KEY, typedKey)); + return new CastExpression(value, new Identifier(valueType, false, span), false, span); + } + finally { + budget.exit(); + } + } + + private static HogQlQuery.BinaryOperator binaryOperator(SemanticOperator operator) + { + return switch (operator) { + case ADD -> HogQlQuery.BinaryOperator.ADD; + case SUBTRACT -> HogQlQuery.BinaryOperator.SUBTRACT; + case MULTIPLY -> HogQlQuery.BinaryOperator.MULTIPLY; + case DIVIDE -> HogQlQuery.BinaryOperator.DIVIDE; + case MODULUS -> HogQlQuery.BinaryOperator.MODULO; + case EQUAL -> HogQlQuery.BinaryOperator.EQUAL; + case NOT_EQUAL -> HogQlQuery.BinaryOperator.NOT_EQUAL; + case LESS_THAN -> HogQlQuery.BinaryOperator.LESS_THAN; + case LESS_THAN_OR_EQUAL -> HogQlQuery.BinaryOperator.LESS_THAN_OR_EQUAL; + case GREATER_THAN -> HogQlQuery.BinaryOperator.GREATER_THAN; + case GREATER_THAN_OR_EQUAL -> HogQlQuery.BinaryOperator.GREATER_THAN_OR_EQUAL; + case AND -> HogQlQuery.BinaryOperator.AND; + case OR -> HogQlQuery.BinaryOperator.OR; + case NOT, NEGATE, IS_NULL, IS_NOT_NULL, SUBSCRIPT, JSON_OBJECT_LOOKUP -> throw new IllegalArgumentException("operator cannot be lowered as binary"); + }; + } + + private static Expression typedLiteral(TypedLiteral literal, HogQlQuery.SourceSpan span) + { + Expression value = switch (literal.encoding()) { + case NULL -> new Literal(HogQlQuery.LiteralKind.NULL, "", span); + case BOOLEAN -> new Literal(HogQlQuery.LiteralKind.BOOLEAN, literal.value(), span); + case INTEGER -> new Literal(HogQlQuery.LiteralKind.INTEGER, literal.value(), span); + case STRING, DECIMAL, FLOAT, JSON -> new Literal(HogQlQuery.LiteralKind.STRING, literal.value(), span); + case BASE64 -> new FunctionCall( + new Identifier("from_base64", false, span), + List.of(new Literal(HogQlQuery.LiteralKind.STRING, literal.value(), span)), + false, + List.of(), + Optional.empty(), + span); + }; + return new CastExpression(value, new Identifier(literal.typeSignature(), false, span), false, span); + } + + private static TrinoException expansionError(HogQlQuery.SourceSpan span, String message) + { + return new TrinoException( + HOGQL_RESOLUTION_ERROR, + Optional.of(new Location(span.startLine(), span.startColumn())), + message, + null); + } + + private static TrinoException unsupportedExpansion(HogQlQuery.SourceSpan span, String message) + { + return new TrinoException( + HOGQL_UNSUPPORTED_FEATURE, + Optional.of(new Location(span.startLine(), span.startColumn())), + message, + null); + } + + private static TrinoException semanticEntityError(HogQlErrorCode errorCode, HogQlQuery.SourceSpan span, String message) + { + return new TrinoException( + errorCode, + Optional.of(new Location(span.startLine(), span.startColumn())), + message, + null); + } + + private static TrinoException cteInferenceError(HogQlQuery.SourceSpan span, String message) + { + return semanticEntityError(HOGQL_RESOLUTION_ERROR, span, message); + } + + private static TrinoException unsupportedColumns(HogQlQuery.SourceSpan span) + { + return semanticEntityError( + HOGQL_UNSUPPORTED_FEATURE, + span, + "HogQL COLUMNS requires a logical relation from the semantic catalog"); + } + + private static TrinoException limitError(HogQlQuery.SourceSpan span, String message) + { + return new TrinoException( + HOGQL_COMPILER_LIMIT_EXCEEDED, + Optional.of(new Location(span.startLine(), span.startColumn())), + message, + null); + } + + private static Identifier identifier(PhysicalIdentifier identifier, TableReference source) + { + return new Identifier(identifier.value(), identifier.delimited(), source.span()); + } + + private static Identifier identifier(PhysicalIdentifier identifier, HogQlQuery.SourceSpan span) + { + return new Identifier(identifier.value(), identifier.delimited(), span); + } + + private static TableBinding tableBinding( + LogicalTableDefinition definition, + String qualifier, + PhysicalIdentifier outputQualifier, + List physicalQualifier, + boolean aliased, + List fields) + { + return new TableBinding( + definition.name(), + qualifier, + outputQualifier, + physicalQualifier, + aliased, + fields, + TableBinding.fieldMap(fields)); + } + + private static String canonical(String value) + { + return value.toLowerCase(Locale.ENGLISH); + } + + private static TrinoException resolutionError(ColumnReference reference, String name) + { + return new TrinoException( + HOGQL_RESOLUTION_ERROR, + Optional.of(new Location(reference.span().startLine(), reference.span().startColumn())), + "Unknown HogQL field: " + name, + null); + } + + private static TrinoException ambiguousResolutionError(ColumnReference reference, String name) + { + return new TrinoException( + HOGQL_RESOLUTION_ERROR, + Optional.of(new Location(reference.span().startLine(), reference.span().startColumn())), + "Ambiguous HogQL field: " + name, + null); + } + + private static TrinoException ambiguousPropertyResolutionError(HogQlQuery.SourceSpan span, String name) + { + return new TrinoException( + HOGQL_RESOLUTION_ERROR, + Optional.of(new Location(span.startLine(), span.startColumn())), + "Ambiguous HogQL property source: " + name, + null); + } + + private static TrinoException starResolutionError(Identifier identifier, String message) + { + return starResolutionError(identifier.span(), message); + } + + private static TrinoException starResolutionError(HogQlQuery.SourceSpan span, String message) + { + return new TrinoException( + HOGQL_RESOLUTION_ERROR, + Optional.of(new Location(span.startLine(), span.startColumn())), + message, + null); + } + + private static TrinoException incompatibleUsingResolutionError(Identifier identifier) + { + return new TrinoException( + HOGQL_RESOLUTION_ERROR, + Optional.of(new Location(identifier.span().startLine(), identifier.span().startColumn())), + "HogQL USING field maps to different physical columns: " + identifier.value(), + null); + } + + private record ResolvedRelation(Relation relation, List bindings, boolean allLogical) + { + private ResolvedRelation + { + relation = requireNonNull(relation, "relation is null"); + bindings = List.copyOf(requireNonNull(bindings, "bindings is null")); + } + } + + private record BindingScope(List bindings, Set relationQualifiers, boolean allRelationsLogical) + { + private BindingScope + { + bindings = List.copyOf(requireNonNull(bindings, "bindings is null")); + relationQualifiers = Set.copyOf(requireNonNull(relationQualifiers, "relationQualifiers is null")); + } + } + + private record CteAnalysis(CommonTableExpression commonTable, TableBinding binding, List outputs) + { + private CteAnalysis + { + commonTable = requireNonNull(commonTable, "commonTable is null"); + binding = requireNonNull(binding, "binding is null"); + outputs = List.copyOf(requireNonNull(outputs, "outputs is null")); + } + } + + private record CteOutput(String name, Optional sourceName, HogQlProjectionDemand sourceDemand) + { + private CteOutput + { + name = requireNonNull(name, "name is null"); + sourceName = requireNonNull(sourceName, "sourceName is null"); + sourceDemand = requireNonNull(sourceDemand, "sourceDemand is null"); + } + } + + private record AliasedOutputDemand(RequiredOutputs sourceOutputs, List outputAliases) + { + private AliasedOutputDemand + { + sourceOutputs = requireNonNull(sourceOutputs, "sourceOutputs is null"); + outputAliases = List.copyOf(requireNonNull(outputAliases, "outputAliases is null")); + } + } + + private record SetOperationOutputs(List outputs, List left, List right) + { + private SetOperationOutputs + { + outputs = List.copyOf(requireNonNull(outputs, "outputs is null")); + left = List.copyOf(requireNonNull(left, "left is null")); + right = List.copyOf(requireNonNull(right, "right is null")); + } + } + + private record SetBranchDemand(RequiredOutputs left, RequiredOutputs right) + { + private SetBranchDemand + { + left = requireNonNull(left, "left is null"); + right = requireNonNull(right, "right is null"); + } + } + + private record TableBinding( + String relationName, + String qualifier, + PhysicalIdentifier outputQualifier, + List physicalQualifier, + boolean aliased, + List orderedFields, + Map fields) + { + private TableBinding(LogicalTableDefinition logicalTable, String qualifier, PhysicalIdentifier outputQualifier, List physicalQualifier, boolean aliased) + { + this(logicalTable.name(), qualifier, outputQualifier, physicalQualifier, aliased, boundFields(logicalTable), fieldMap(boundFields(logicalTable))); + } + + private TableBinding + { + relationName = requireNonNull(relationName, "relationName is null"); + qualifier = requireNonNull(qualifier, "qualifier is null"); + outputQualifier = requireNonNull(outputQualifier, "outputQualifier is null"); + physicalQualifier = List.copyOf(requireNonNull(physicalQualifier, "physicalQualifier is null")); + orderedFields = List.copyOf(requireNonNull(orderedFields, "orderedFields is null")); + fields = Map.copyOf(requireNonNull(fields, "fields is null")); + } + + private TableBinding withAlias(Identifier alias) + { + return new TableBinding( + relationName, + canonical(alias.value()), + new PhysicalIdentifier(alias.value(), alias.delimited()), + physicalQualifier, + true, + orderedFields, + fields); + } + + private Optional starQualifier(int relationCount) + { + return aliased || relationCount > 1 ? Optional.of(outputQualifier) : Optional.empty(); + } + + private static List boundFields(LogicalTableDefinition table) + { + return table.fields().stream() + .map(field -> new BoundField(field.name(), field.physicalColumn(), field.starVisible(), Optional.empty())) + .toList(); + } + + private static Map fieldMap(List orderedFields) + { + Map fields = new HashMap<>(); + orderedFields.forEach(field -> fields.put(canonical(field.name()), field)); + return fields; + } + } + + private record BoundField(String name, PhysicalIdentifier physicalColumn, boolean starVisible, Optional expression) + { + private BoundField + { + name = requireNonNull(name, "name is null"); + physicalColumn = requireNonNull(physicalColumn, "physicalColumn is null"); + expression = requireNonNull(expression, "expression is null"); + } + } + + private record FieldMatch(TableBinding binding, BoundField field) {} + + private record StarField(TableBinding binding, BoundField field) {} + + private record LazyStar(TableBinding binding, LazyTableDefinition definition) {} + + private record PropertyMatch(TableBinding binding, PropertyDefinition property) {} + + private record PathCandidate(TableBinding owner, List path) + { + private PathCandidate + { + owner = requireNonNull(owner, "owner is null"); + path = List.copyOf(requireNonNull(path, "path is null")); + } + } + + private record RelationshipPathKey(String ownerQualifier, List path) + { + private RelationshipPathKey + { + ownerQualifier = requireNonNull(ownerQualifier, "ownerQualifier is null"); + path = List.copyOf(requireNonNull(path, "path is null")); + } + } + + private record ProjectedField(String name, String sourceField, boolean starVisible) + { + private ProjectedField + { + name = requireNonNull(name, "name is null"); + sourceField = requireNonNull(sourceField, "sourceField is null"); + } + } + + private record EntityLookup(String value, boolean id) + { + private EntityLookup + { + value = requireNonNull(value, "value is null"); + } + + private boolean matches(String name, String entityId) + { + return id ? value.equals(entityId) : canonical(value).equals(canonical(name)); + } + } + + private static final class ExpansionBudget + { + private static final int MAXIMUM_NODES = 10_000; + private static final int MAXIMUM_DEPTH = 64; + private int nodes; + private int depth; + + private void add(HogQlQuery.SourceSpan span) + { + if (++nodes > MAXIMUM_NODES) { + throw limitError(span, "HogQL semantic expansion exceeded node limit"); + } + } + + private void enter(HogQlQuery.SourceSpan span) + { + add(span); + if (++depth > MAXIMUM_DEPTH) { + throw limitError(span, "HogQL semantic expansion exceeded depth limit"); + } + } + + private void exit() + { + depth--; + } + } + + private static final class RelationExpansionBudget + { + private static final int MAXIMUM_DEPTH = 64; + private int depth; + + private void enter(HogQlQuery.SourceSpan span) + { + if (++depth > MAXIMUM_DEPTH) { + throw limitError(span, "HogQL semantic relation expansion exceeded depth limit"); + } + } + + private void exit() + { + depth--; + } + } + + record ResolvedQuery(HogQlQuery query) + { + ResolvedQuery + { + query = requireNonNull(query, "query is null"); + } + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlTypedValue.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlTypedValue.java new file mode 100644 index 000000000000..f69cace05536 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlTypedValue.java @@ -0,0 +1,147 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import java.math.BigDecimal; +import java.util.List; +import java.util.Map; + +import static java.util.Objects.requireNonNull; + +public record HogQlTypedValue(String type, Value value) +{ + public HogQlTypedValue + { + type = requireNonNull(type, "type is null"); + value = requireNonNull(value, "value is null"); + if (type.isBlank()) { + throw new IllegalArgumentException("type is empty"); + } + } + + @Override + public String toString() + { + return "HogQlTypedValue[type=%s, value=]".formatted(type); + } + + public sealed interface Value + permits ArrayValue, + BooleanValue, + NullValue, + NumberValue, + ObjectValue, + StringValue {} + + public enum NullValue + implements Value + { + NULL; + + @Override + public String toString() + { + return ""; + } + } + + public record BooleanValue(boolean value) + implements Value + { + @Override + public String toString() + { + return ""; + } + } + + public record NumberValue(String value) + implements Value + { + public NumberValue + { + value = requireNonNull(value, "number value is null"); + try { + new BigDecimal(value); + } + catch (NumberFormatException _) { + throw new IllegalArgumentException("number value is invalid"); + } + } + + @Override + public String toString() + { + return ""; + } + } + + public record StringValue(String value) + implements Value + { + public StringValue + { + value = requireNonNull(value, "string value is null"); + } + + @Override + public String toString() + { + return ""; + } + } + + public record ArrayValue(List value) + implements Value + { + public ArrayValue + { + requireNonNull(value, "array value is null"); + if (value.stream().anyMatch(element -> element == null)) { + throw new IllegalArgumentException("array value contains a missing element"); + } + value = List.copyOf(value); + } + + @Override + public String toString() + { + return ""; + } + } + + public record ObjectValue(Map value) + implements Value + { + public ObjectValue + { + requireNonNull(value, "object value is null"); + for (Map.Entry entry : value.entrySet()) { + if (entry.getKey() == null || entry.getKey().isBlank()) { + throw new IllegalArgumentException("object value contains an empty field name"); + } + if (entry.getValue() == null) { + throw new IllegalArgumentException("object value contains a missing field value"); + } + } + value = Map.copyOf(value); + } + + @Override + public String toString() + { + return ""; + } + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java new file mode 100644 index 000000000000..79afc295d0df --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0FunctionRegistry.java @@ -0,0 +1,324 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCapabilityDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionImplementation; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionRewrite; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionSignature; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +final class HogQlV0FunctionRegistry +{ + private static final Map FUNCTIONS = functions( + scalar("coalesce", "coalesce", variadicSignature(1)), + scalar("if", "if", signature(3)), + scalar("abs", "abs", signature(1)), + scalar("lower", "lower", signature(1)), + scalar("upper", "upper", signature(1)), + scalar("length", "length", signature(1)), + scalar("concat", "concat", variadicSignature(2)), + scalar("replace", "replace", signature(3)), + scalar("nullIf", "nullif", signature(2)), + scalar("ifNull", "coalesce", signature(2)), + scalar("trim", "trim", signature(1)), + scalar("round", "round", signature(1), signature(2)), + nondeterministicScalar("now", "now", signature(0)), + nondeterministicScalar("current_timestamp", "now", signature(0)), + rewrite("isNotNull", FunctionRewrite.IS_NOT_NULL, "boolean", signature(1)), + rewrite("isNull", FunctionRewrite.IS_NULL, "boolean", signature(1)), + rewrite("toInt", FunctionRewrite.CAST_BIGINT, "bigint", signature(1)), + rewrite("toFloat", FunctionRewrite.CAST_DOUBLE, "double", signature(1)), + rewrite("toFloatOrZero", FunctionRewrite.FLOAT_OR_ZERO, "double", signature(1)), + rewrite("toFloatOrDefault", FunctionRewrite.FLOAT_OR_DEFAULT, "double", signature(2)), + rewrite("toDecimal", FunctionRewrite.DECIMAL_CAST, "decimal", signature(2)), + rewrite("intDiv", FunctionRewrite.INT_DIV, "bigint", signature(2)), + rewrite("arrayElement", FunctionRewrite.ARRAY_ELEMENT, "any", signature(2)), + rewrite("arrayFilter", FunctionRewrite.ARRAY_FILTER, "array(any)", signature(2)), + rewrite("arrayFirst", FunctionRewrite.ARRAY_FIRST, "any", signature(2)), + rewrite("arrayMap", FunctionRewrite.ARRAY_MAP, "array(any)", signature(2)), + rewrite("arraySum", FunctionRewrite.ARRAY_SUM, "any", signature(1)), + rewrite("arraySlice", FunctionRewrite.ARRAY_SLICE, "array(any)", signature(3)), + rewrite("arrayEnumerate", FunctionRewrite.ARRAY_ENUMERATE, "array(bigint)", signature(1)), + rewrite("range", FunctionRewrite.RANGE, "array(bigint)", signature(1), signature(2)), + rewrite("tupleElement", FunctionRewrite.TUPLE_ELEMENT, "any", signature(2)), + rewrite("splitByChar", FunctionRewrite.SPLIT_CHAR, "array(varchar)", signature(2)), + rewrite("has", FunctionRewrite.HAS, "boolean", signature(2)), + rewrite("assumeNotNull", FunctionRewrite.ASSUME_NOT_NULL, "any", signature(1)), + rewrite("empty", FunctionRewrite.EMPTY, "boolean", signature(1)), + rewrite("notEmpty", FunctionRewrite.NOT_EMPTY, "boolean", signature(1)), + rewrite("equals", FunctionRewrite.EQUALS, "boolean", signature(2)), + rewrite("plus", FunctionRewrite.PLUS, "any", signature(2)), + rewrite("multiply", FunctionRewrite.MULTIPLY, "any", signature(2)), + rewrite("multiplyDecimal", FunctionRewrite.MULTIPLY_DECIMAL, "decimal", signature(2)), + rewrite("divideDecimal", FunctionRewrite.DIVIDE_DECIMAL, "decimal", signature(2)), + rewrite("divide", FunctionRewrite.DIVIDE_DECIMAL, "any", signature(2)), + rewrite("in", FunctionRewrite.IN_ARRAY, "boolean", signature(2)), + rewrite("tuple", FunctionRewrite.TUPLE, "row", variadicSignature(1)), + rewrite("subtractMonths", FunctionRewrite.SUBTRACT_MONTHS, "any", signature(2)), + rewrite("subtractDays", FunctionRewrite.SUBTRACT_DAYS, "any", signature(2)), + rewrite("toIntervalMonth", FunctionRewrite.INTERVAL_MONTH, "interval year to month", signature(1)), + rewrite("toStartOfWeek", FunctionRewrite.START_WEEK, "timestamp", signature(1), signature(2)), + rewrite("subtractYears", FunctionRewrite.SUBTRACT_YEARS, "any", signature(2)), + rewrite("toIntOrZero", FunctionRewrite.INT_OR_ZERO, "bigint", signature(1)), + rewrite("_toInt16", FunctionRewrite.CAST_SMALLINT, "smallint", signature(1)), + rewrite("toUUID", FunctionRewrite.CAST_UUID, "uuid", signature(1)), + rewrite("toJSONString", FunctionRewrite.TO_JSON_STRING, "varchar", signature(1)), + rewrite("JSONHas", FunctionRewrite.JSON_HAS, "boolean", signature(2)), + rewrite("JSONExtractKeys", FunctionRewrite.JSON_EXTRACT_KEYS, "array(varchar)", signature(1), variadicSignature(2)), + rewrite("JSON_VALUE", FunctionRewrite.JSON_VALUE, "varchar", signature(2)), + rewrite("getSurveyResponse", FunctionRewrite.SURVEY_RESPONSE, "varchar", signature(2)), + rewrite("md5", FunctionRewrite.MD5, "varbinary", signature(1)), + rewrite("date_part", FunctionRewrite.DATE_PART, "bigint", signature(2)), + rewrite("minus", FunctionRewrite.MINUS, "any", signature(2)), + rewrite("notEquals", FunctionRewrite.NOT_EQUALS, "boolean", signature(2)), + rewrite("splitByString", FunctionRewrite.SPLIT_STRING, "array(varchar)", signature(2)), + rewrite("toString", FunctionRewrite.CAST_VARCHAR, "varchar", signature(1)), + rewrite("toDate", FunctionRewrite.CAST_DATE, "date", signature(1)), + rewrite("_toDate", FunctionRewrite.CAST_DATE, "date", signature(1)), + rewrite("toDateTime", FunctionRewrite.CAST_TIMESTAMP, "timestamp(0)", signature(1), signature(2)), + rewrite("toStartOfMonth", FunctionRewrite.DATE_TRUNC_MONTH, "timestamp", signature(1)), + rewrite("toStartOfDay", FunctionRewrite.DATE_TRUNC_DAY, "timestamp", signature(1)), + rewrite("toStartOfHour", FunctionRewrite.DATE_TRUNC_HOUR, "timestamp", signature(1)), + rewrite("toMonday", FunctionRewrite.DATE_TRUNC_WEEK, "timestamp", signature(1)), + rewrite("multiIf", FunctionRewrite.MULTI_IF, "any", variadicSignature(3)), + rewrite("JSONExtractString", FunctionRewrite.JSON_EXTRACT_STRING, "varchar", signature(1), variadicSignature(2)), + rewrite("JSONExtractInt", FunctionRewrite.JSON_EXTRACT_INT, "bigint", variadicSignature(2)), + rewrite("JSONExtractFloat", FunctionRewrite.JSON_EXTRACT_FLOAT, "double", variadicSignature(2)), + rewrite("JSONExtractBool", FunctionRewrite.JSON_EXTRACT_BOOL, "boolean", variadicSignature(2)), + rewrite("JSONExtractUInt", FunctionRewrite.JSON_EXTRACT_UINT, "bigint", variadicSignature(2)), + rewrite("JSONExtractArrayRaw", FunctionRewrite.JSON_EXTRACT_ARRAY_RAW, "array(varchar)", signature(1), variadicSignature(2)), + rewrite("JSONExtractRaw", FunctionRewrite.JSON_EXTRACT_RAW, "varchar", variadicSignature(2)), + rewrite("JSONLength", FunctionRewrite.JSON_LENGTH, "bigint", signature(1), variadicSignature(2)), + rewrite("JSONExtract", FunctionRewrite.JSON_EXTRACT_TYPED, "any", signature(2)), + rewrite("JSONExtractKeysAndValues", FunctionRewrite.JSON_KEYS_AND_VALUES, "array(row(varchar,any))", signature(2)), + rewrite("JSONExtractKeysAndValuesRaw", FunctionRewrite.JSON_KEYS_AND_VALUES_RAW, "array(row(varchar,varchar))", signature(1), variadicSignature(2)), + rewrite("today", FunctionRewrite.TODAY, "date", signature(0)), + rewrite("toIntervalDay", FunctionRewrite.INTERVAL_DAY, "interval day to second", signature(1)), + rewrite("addDays", FunctionRewrite.ADD_DAYS, "any", signature(2)), + rewrite("addMonths", FunctionRewrite.ADD_MONTHS, "any", signature(2)), + rewrite("dateAdd", FunctionRewrite.DATE_ADD, "any", signature(2), signature(3)), + rewrite("convertCurrency", FunctionRewrite.CONVERT_CURRENCY, "decimal", signature(3), signature(4)), + rewrite("toUnixTimestamp", FunctionRewrite.TO_UNIX_TIMESTAMP, "bigint", signature(1)), + rewrite("parseDateTimeBestEffort", FunctionRewrite.PARSE_TIMESTAMP, "timestamp(3)", signature(1)), + rewrite("not", FunctionRewrite.NOT, "boolean", signature(1)), + rewrite("and", FunctionRewrite.AND, "boolean", signature(2), variadicSignature(2)), + rewrite("or", FunctionRewrite.OR, "boolean", signature(2), variadicSignature(2)), + rewrite("greater", FunctionRewrite.GREATER, "boolean", signature(2)), + rewrite("greaterOrEquals", FunctionRewrite.GREATER_OR_EQUAL, "boolean", signature(2)), + rewrite("lessOrEquals", FunctionRewrite.LESS_OR_EQUAL, "boolean", signature(2)), + rewrite("like", FunctionRewrite.LIKE, "boolean", signature(2)), + rewrite("extract", FunctionRewrite.REGEX_EXTRACT, "varchar", signature(2)), + rewrite("replaceRegexpAll", FunctionRewrite.REGEX_REPLACE_ALL, "varchar", signature(3)), + rewrite("replaceRegexpOne", FunctionRewrite.REGEX_REPLACE_ONE, "varchar", signature(3)), + rewrite("extractAll", FunctionRewrite.REGEX_EXTRACT_ALL, "array(varchar)", signature(2)), + scalar("fromUnixTimestamp", "from_unixtime", signature(1)), + scalar("formatDateTime", "date_format", signature(2)), + scalar("toTimeZone", "at_timezone", signature(2)), + scalar("least", "least", variadicSignature(1)), + scalar("greatest", "greatest", variadicSignature(1)), + scalar("position", "strpos", signature(2)), + scalar("startsWith", "starts_with", signature(2)), + scalar("substring", "substring", signature(2), signature(3)), + scalar("log10", "log10", signature(1)), + scalar("exp", "exp", signature(1)), + scalar("match", "regexp_like", signature(2)), + scalar("floor", "floor", signature(1)), + scalar("toDayOfMonth", "day", signature(1)), + scalar("toDayOfWeek", "day_of_week", signature(1)), + scalar("mapFromArrays", "map", signature(2)), + scalar("mapUpdate", "map_concat", signature(2)), + scalar("toMonth", "month", signature(1)), + scalar("toYear", "year", signature(1)), + scalar("ceil", "ceiling", signature(1)), + scalar("pow", "power", signature(2)), + scalar("substringUTF8", "substring", signature(2), signature(3)), + scalar("arrayConcat", "concat", variadicSignature(2)), + scalar("roundBankers", "round", signature(1), signature(2)), + scalar("cityHash64", "cityhash64", signature(1)), + scalar("formatReadableTimeDelta", "format_readable_time_delta", signature(1), signature(2), signature(3)), + scalar("hasAny", "arrays_overlap", signature(2)), + scalar("parseDateTime", "date_parse", signature(2)), + scalar("toLastDayOfMonth", "last_day_of_month", signature(1)), + rewrite("map", FunctionRewrite.MAP_CONSTRUCTOR, "map(any,any)", signature(0), variadicSignature(2)), + scalar("dateDiff", "date_diff", signature(3)), + scalar("date_diff", "date_diff", signature(3)), + scalar("dateTrunc", "date_trunc", signature(2)), + rewrite("arraySort", FunctionRewrite.ARRAY_SORT, "array(any)", signature(1), signature(2)), + scalar("arrayMin", "array_min", signature(1)), + scalar("arrayDistinct", "array_distinct", signature(1)), + scalar("arrayFlatten", "flatten", signature(1)), + scalar("arrayStringConcat", "array_join", signature(1), signature(2)), + scalar("replaceAll", "replace", signature(3)), + aggregate("count", "count", true, signature(0), signature(1)), + aggregate("sum", "sum", true, signature(1)), + aggregate("min", "min", true, signature(1)), + aggregate("max", "max", true, signature(1)), + aggregate("avg", "avg", true, signature(1)), + aggregateWithOrderBy("array_agg", "array_agg", true, signature(1)), + aggregateWithOrderBy("groupArray", "array_agg", false, signature(1)), + aggregateRewrite("countIf", FunctionRewrite.COUNT_IF, "bigint", signature(1)), + aggregateRewrite("sumIf", FunctionRewrite.SUM_IF, "any", signature(2)), + aggregateRewrite("maxIf", FunctionRewrite.MAX_IF, "any", signature(2)), + aggregateRewrite("uniqIf", FunctionRewrite.UNIQ_IF, "bigint", signature(2)), + aggregateRewrite("uniqExact", FunctionRewrite.UNIQ_EXACT, "bigint", signature(1)), + aggregateRewrite("groupUniqArray", FunctionRewrite.GROUP_UNIQ_ARRAY, "array(any)", signature(1)), + aggregateRewrite("argMaxIf", FunctionRewrite.ARG_MAX_IF, "any", signature(3)), + aggregateRewrite("argMinIf", FunctionRewrite.ARG_MIN_IF, "any", signature(3)), + aggregateRewrite("anyIf", FunctionRewrite.ANY_IF, "any", signature(2)), + aggregateRewrite("minIf", FunctionRewrite.MIN_IF, "any", signature(2)), + aggregateRewrite("avgIf", FunctionRewrite.AVG_IF, "double", signature(2)), + aggregateRewrite("groupArrayIf", FunctionRewrite.GROUP_ARRAY_IF, "array(any)", signature(2), signature(3)), + aggregateRewrite("quantile", FunctionRewrite.QUANTILE, "any", signature(2)), + aggregateRewrite("quantileExact", FunctionRewrite.QUANTILE_EXACT, "any", signature(2)), + aggregateRewrite("quantileIf", FunctionRewrite.QUANTILE_IF, "any", signature(3)), + aggregateRewrite("uniqExactIf", FunctionRewrite.UNIQ_EXACT_IF, "bigint", signature(2)), + aggregateRewrite("groupUniqArrayIf", FunctionRewrite.GROUP_UNIQ_ARRAY_IF, "array(any)", signature(2)), + aggregateRewrite("countDistinct", FunctionRewrite.COUNT_DISTINCT, "bigint", signature(1)), + aggregateRewrite("medianIf", FunctionRewrite.MEDIAN_IF, "double", signature(2)), + nondeterministicAggregate("uniq", "approx_distinct", false, signature(1)), + nondeterministicAggregate("any", "arbitrary", false, signature(1)), + nondeterministicAggregate("argMin", "min_by", false, signature(2)), + nondeterministicAggregate("argMax", "max_by", false, signature(2)), + nondeterministicAggregate("anyLast", "arbitrary", false, signature(1)), + window("rank", "rank", signature(0)), + window("first_value", "first_value", signature(1)), + window("lag", "lag", signature(1), signature(2), signature(3)), + window("lagInFrame", "lag", signature(1), signature(2), signature(3)), + window("row_number", "row_number", signature(0))); + + private HogQlV0FunctionRegistry() {} + + public static Map functions() + { + return FUNCTIONS; + } + + private static Map functions(FunctionCapabilityDefinition... functions) + { + Map result = new LinkedHashMap<>(); + for (FunctionCapabilityDefinition function : functions) { + FunctionCapabilityDefinition previous = result.put(function.name().toLowerCase(java.util.Locale.ENGLISH), function); + if (previous != null) { + throw new IllegalArgumentException("duplicate HogQL v0 function"); + } + } + return Map.copyOf(result); + } + + private static FunctionCapabilityDefinition scalar(String hogQlName, String trinoName, FunctionSignature... signatures) + { + return function(hogQlName, trinoName, FunctionKind.SCALAR, false, false, false, false, true, signatures); + } + + private static FunctionCapabilityDefinition nondeterministicScalar(String hogQlName, String trinoName, FunctionSignature... signatures) + { + return function(hogQlName, trinoName, FunctionKind.SCALAR, false, false, false, false, false, signatures); + } + + private static FunctionCapabilityDefinition rewrite(String hogQlName, FunctionRewrite rewrite, String returnType, FunctionSignature... signatures) + { + return new FunctionCapabilityDefinition( + hogQlName, + FunctionKind.SCALAR, + FunctionImplementation.REWRITE, + List.of(), + java.util.Optional.of(rewrite), + java.util.Arrays.stream(signatures) + .map(signature -> new FunctionSignature(signature.argumentTypes(), returnType, signature.variadic())) + .toList(), + true, + false, + false, + false, + false); + } + + private static FunctionCapabilityDefinition aggregate(String hogQlName, String trinoName, boolean supportsDistinct, FunctionSignature... signatures) + { + return function(hogQlName, trinoName, FunctionKind.AGGREGATE, supportsDistinct, false, true, true, true, signatures); + } + + private static FunctionCapabilityDefinition aggregateRewrite(String hogQlName, FunctionRewrite rewrite, String returnType, FunctionSignature... signatures) + { + return new FunctionCapabilityDefinition( + hogQlName, + FunctionKind.AGGREGATE, + FunctionImplementation.REWRITE, + List.of(), + java.util.Optional.of(rewrite), + java.util.Arrays.stream(signatures) + .map(signature -> new FunctionSignature(signature.argumentTypes(), returnType, signature.variadic())) + .toList(), + true, + false, + false, + false, + true); + } + + private static FunctionCapabilityDefinition aggregateWithOrderBy(String hogQlName, String trinoName, boolean supportsDistinct, FunctionSignature... signatures) + { + return function(hogQlName, trinoName, FunctionKind.AGGREGATE, supportsDistinct, true, true, true, true, signatures); + } + + private static FunctionCapabilityDefinition nondeterministicAggregate(String hogQlName, String trinoName, boolean supportsDistinct, FunctionSignature... signatures) + { + return function(hogQlName, trinoName, FunctionKind.AGGREGATE, supportsDistinct, false, true, true, false, signatures); + } + + private static FunctionCapabilityDefinition window(String hogQlName, String trinoName, FunctionSignature... signatures) + { + return function(hogQlName, trinoName, FunctionKind.WINDOW, false, false, false, true, true, signatures); + } + + private static FunctionCapabilityDefinition function( + String hogQlName, + String trinoName, + FunctionKind kind, + boolean supportsDistinct, + boolean supportsOrderBy, + boolean supportsFilter, + boolean supportsWindow, + boolean deterministic, + FunctionSignature... signatures) + { + return new FunctionCapabilityDefinition( + hogQlName, + kind, + FunctionImplementation.STOCK, + List.of(new PhysicalIdentifier(trinoName, false)), + List.of(signatures), + deterministic, + supportsDistinct, + supportsOrderBy, + supportsFilter, + supportsWindow); + } + + private static FunctionSignature signature(int arity) + { + return new FunctionSignature(java.util.stream.Stream.generate(() -> "any").limit(arity).toList(), "any", false); + } + + private static FunctionSignature variadicSignature(int minimumArity) + { + return new FunctionSignature(java.util.stream.Stream.generate(() -> "any").limit(minimumArity + 1L).toList(), "any", true); + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0ProfileValidator.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0ProfileValidator.java new file mode 100644 index 000000000000..a90107fcb563 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/HogQlV0ProfileValidator.java @@ -0,0 +1,230 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot; +import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.AliasedRelation; +import io.trino.hogql.parser.tree.HogQlQuery.ArrayExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BetweenExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CaseExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CastExpression; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsList; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsRegex; +import io.trino.hogql.parser.tree.HogQlQuery.CommonTableReference; +import io.trino.hogql.parser.tree.HogQlQuery.Expression; +import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; +import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; +import io.trino.hogql.parser.tree.HogQlQuery.InCohortExpression; +import io.trino.hogql.parser.tree.HogQlQuery.InExpression; +import io.trino.hogql.parser.tree.HogQlQuery.InSubqueryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IntervalExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; +import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; +import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; +import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.LambdaExpression; +import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; +import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; +import io.trino.hogql.parser.tree.HogQlQuery.Projection; +import io.trino.hogql.parser.tree.HogQlQuery.Relation; +import io.trino.hogql.parser.tree.HogQlQuery.ScalarSubqueryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.SelectQueryBody; +import io.trino.hogql.parser.tree.HogQlQuery.SetOperation; +import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; +import io.trino.hogql.parser.tree.HogQlQuery.Star; +import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; +import io.trino.hogql.parser.tree.HogQlQuery.SubscriptExpression; +import io.trino.hogql.parser.tree.HogQlQuery.TablePlaceholder; +import io.trino.hogql.parser.tree.HogQlQuery.TableReference; +import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; +import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.UnnestRelation; +import io.trino.hogql.parser.tree.HogQlQuery.ValuesRelation; +import io.trino.hogql.parser.tree.HogQlQuery.Window; +import io.trino.hogql.parser.tree.HogQlQuery.WindowReference; +import io.trino.hogql.parser.tree.HogQlQuery.WindowSpecification; +import io.trino.spi.Location; +import io.trino.spi.TrinoException; + +import java.util.Locale; +import java.util.Optional; + +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE; +import static java.util.Objects.requireNonNull; + +final class HogQlV0ProfileValidator +{ + private HogQlV0ProfileValidator() {} + + public static void validate(HogQlQuery query, Optional snapshot) + { + requireNonNull(query, "query is null"); + requireNonNull(snapshot, "snapshot is null"); + query.with().forEach(commonTable -> validate(commonTable.query(), snapshot)); + switch (query.body()) { + case SelectQueryBody select -> { + select.projections().forEach(projection -> validate(projection, snapshot)); + select.from().ifPresent(relation -> validate(relation, snapshot)); + select.where().ifPresent(expression -> validate(expression, snapshot)); + select.groupBy().forEach(expression -> validate(expression, snapshot)); + select.having().ifPresent(expression -> validate(expression, snapshot)); + select.windows().forEach(window -> validate(window.specification(), snapshot)); + select.limitBy().ifPresent(limitBy -> { + validate(limitBy.limit(), snapshot); + limitBy.offset().ifPresent(expression -> validate(expression, snapshot)); + limitBy.partitionBy().forEach(expression -> validate(expression, snapshot)); + }); + } + case SetOperation setOperation -> { + validate(setOperation.left(), snapshot); + validate(setOperation.right(), snapshot); + } + } + query.orderBy().forEach(sortItem -> validate(sortItem.expression(), snapshot)); + query.limit().ifPresent(expression -> validate(expression, snapshot)); + query.offset().ifPresent(expression -> validate(expression, snapshot)); + } + + private static void validate(Projection projection, Optional snapshot) + { + switch (projection) { + case ColumnsList columns -> columns.expressions().forEach(expression -> validate(expression, snapshot)); + case ColumnsRegex _ -> {} + case ExpressionProjection expression -> validate(expression.expression(), snapshot); + case Star star -> star.replacements().forEach(replacement -> validate(replacement.expression(), snapshot)); + } + } + + private static void validate(Relation relation, Optional snapshot) + { + switch (relation) { + case AliasedRelation alias -> validate(alias.relation(), snapshot); + case CommonTableReference _, TablePlaceholder _ -> {} + case JoinRelation join -> { + validate(join.left(), snapshot); + validate(join.right(), snapshot); + join.criteria().filter(JoinOn.class::isInstance) + .map(JoinOn.class::cast) + .ifPresent(on -> validate(on.expression(), snapshot)); + } + case HogQlQuery.PivotRelation pivot -> throw unsupported(pivot.span(), "PIVOT is outside the HogQL v0 profile"); + case SubqueryRelation subquery -> validate(subquery.query(), snapshot); + case TableReference table -> validateTable(table, snapshot); + case UnnestRelation unnest -> unnest.expressions().forEach(expression -> validate(expression, snapshot)); + case ValuesRelation values -> values.rows().forEach(row -> row.forEach(expression -> validate(expression, snapshot))); + } + } + + private static void validateTable(TableReference table, Optional snapshot) + { + if (table.parts().size() != 1 || snapshot.isEmpty()) { + return; + } + String name = canonical(table.parts().getFirst().value()); + HogQlSemanticCatalogSnapshot catalog = snapshot.orElseThrow(); + boolean deferredRelation = catalog.virtualTables().stream().anyMatch(tableDefinition -> canonical(tableDefinition.name()).equals(name)) || + catalog.savedQueries().stream().anyMatch(query -> canonical(query.name()).equals(name)) || + catalog.materializedViews().stream().anyMatch(view -> canonical(view.name()).equals(name)); + if (deferredRelation) { + throw unsupported(table.span(), "Semantic relation " + table.parts().getFirst().value() + " is outside the HogQL v0 profile"); + } + } + + private static void validate(Expression expression, Optional snapshot) + { + switch (expression) { + case ArrayExpression array -> array.values().forEach(value -> validate(value, snapshot)); + case BetweenExpression between -> { + validate(between.value(), snapshot); + validate(between.min(), snapshot); + validate(between.max(), snapshot); + } + case BinaryExpression binary -> { + validate(binary.left(), snapshot); + validate(binary.right(), snapshot); + } + case CaseExpression caseExpression -> { + caseExpression.operand().ifPresent(value -> validate(value, snapshot)); + caseExpression.whenClauses().forEach(when -> { + validate(when.operand(), snapshot); + validate(when.result(), snapshot); + }); + caseExpression.defaultValue().ifPresent(value -> validate(value, snapshot)); + } + case CastExpression cast -> validate(cast.value(), snapshot); + case ColumnReference _, Literal _, Placeholder _ -> {} + case FunctionCall function -> { + if (function.nameParts().size() != 1) { + throw unsupported(function.span(), "Qualified functions are outside the HogQL v0 profile"); + } + function.arguments().forEach(argument -> validate(argument, snapshot)); + function.orderBy().forEach(sortItem -> validate(sortItem.expression(), snapshot)); + function.filter().ifPresent(filter -> validate(filter, snapshot)); + function.window().ifPresent(window -> validate(window, snapshot)); + } + case InCohortExpression in -> throw unsupported(in.span(), "Cohorts are outside the HogQL v0 profile"); + case InExpression in -> { + validate(in.value(), snapshot); + in.values().forEach(value -> validate(value, snapshot)); + } + case InSubqueryExpression in -> { + validate(in.value(), snapshot); + validate(in.query(), snapshot); + } + case IntervalExpression interval -> validate(interval.value(), snapshot); + case IsNullExpression isNull -> validate(isNull.value(), snapshot); + case LambdaExpression lambda -> validate(lambda.body(), snapshot); + case MemberAccessExpression memberAccess -> validate(memberAccess.base(), snapshot); + case ScalarSubqueryExpression subquery -> validate(subquery.query(), snapshot); + case SubscriptExpression subscript -> { + validate(subscript.base(), snapshot); + validate(subscript.index(), snapshot); + } + case TupleExpression tuple -> tuple.values().forEach(value -> validate(value, snapshot)); + case UnaryExpression unary -> validate(unary.operand(), snapshot); + } + } + + private static void validate(Window window, Optional snapshot) + { + switch (window) { + case WindowReference _ -> {} + case WindowSpecification specification -> { + specification.partitionBy().forEach(expression -> validate(expression, snapshot)); + specification.orderBy().forEach(sortItem -> validate(sortItem.expression(), snapshot)); + specification.frame().ifPresent(frame -> { + frame.start().value().ifPresent(value -> validate(value, snapshot)); + frame.end().flatMap(HogQlQuery.FrameBound::value).ifPresent(value -> validate(value, snapshot)); + }); + } + } + } + + private static String canonical(String value) + { + return value.toLowerCase(Locale.ENGLISH); + } + + private static TrinoException unsupported(SourceSpan span, String message) + { + return new TrinoException( + HOGQL_UNSUPPORTED_FEATURE, + Optional.of(new Location(span.startLine(), span.startColumn())), + message, + null); + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java new file mode 100644 index 000000000000..b62f0ef6d64e --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/TrinoAstFactory.java @@ -0,0 +1,1025 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.AliasedRelation; +import io.trino.hogql.parser.tree.HogQlQuery.ArrayExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BetweenExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CaseExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CastExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CastTypeDialect; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsList; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsRegex; +import io.trino.hogql.parser.tree.HogQlQuery.CommonTableExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CommonTableReference; +import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; +import io.trino.hogql.parser.tree.HogQlQuery.FrameBound; +import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; +import io.trino.hogql.parser.tree.HogQlQuery.Identifier; +import io.trino.hogql.parser.tree.HogQlQuery.InCohortExpression; +import io.trino.hogql.parser.tree.HogQlQuery.InExpression; +import io.trino.hogql.parser.tree.HogQlQuery.InSubqueryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IntervalExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; +import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; +import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; +import io.trino.hogql.parser.tree.HogQlQuery.JoinUsing; +import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; +import io.trino.hogql.parser.tree.HogQlQuery.PivotRelation; +import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; +import io.trino.hogql.parser.tree.HogQlQuery.Projection; +import io.trino.hogql.parser.tree.HogQlQuery.Relation; +import io.trino.hogql.parser.tree.HogQlQuery.ScalarSubqueryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.SelectQueryBody; +import io.trino.hogql.parser.tree.HogQlQuery.SetOperation; +import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; +import io.trino.hogql.parser.tree.HogQlQuery.Star; +import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; +import io.trino.hogql.parser.tree.HogQlQuery.SubscriptExpression; +import io.trino.hogql.parser.tree.HogQlQuery.TablePlaceholder; +import io.trino.hogql.parser.tree.HogQlQuery.TableReference; +import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; +import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.UnnestRelation; +import io.trino.hogql.parser.tree.HogQlQuery.ValuesRelation; +import io.trino.hogql.parser.tree.HogQlQuery.Window; +import io.trino.hogql.parser.tree.HogQlQuery.WindowDefinition; +import io.trino.hogql.parser.tree.HogQlQuery.WindowFrame; +import io.trino.hogql.parser.tree.HogQlQuery.WindowReference; +import io.trino.hogql.parser.tree.HogQlQuery.WindowSpecification; +import io.trino.spi.Location; +import io.trino.spi.TrinoException; +import io.trino.sql.parser.SqlParser; +import io.trino.sql.tree.AllColumns; +import io.trino.sql.tree.ArithmeticBinaryExpression; +import io.trino.sql.tree.ArithmeticUnaryExpression; +import io.trino.sql.tree.Array; +import io.trino.sql.tree.BetweenPredicate; +import io.trino.sql.tree.BooleanLiteral; +import io.trino.sql.tree.CallArgument; +import io.trino.sql.tree.Cast; +import io.trino.sql.tree.CoalesceExpression; +import io.trino.sql.tree.ComparisonPredicate; +import io.trino.sql.tree.DereferenceExpression; +import io.trino.sql.tree.DoubleLiteral; +import io.trino.sql.tree.Except; +import io.trino.sql.tree.Expression; +import io.trino.sql.tree.GroupBy; +import io.trino.sql.tree.IfExpression; +import io.trino.sql.tree.InListExpression; +import io.trino.sql.tree.InPredicate; +import io.trino.sql.tree.Intersect; +import io.trino.sql.tree.IntervalField; +import io.trino.sql.tree.IntervalLiteral; +import io.trino.sql.tree.IsNullPredicate; +import io.trino.sql.tree.Lateral; +import io.trino.sql.tree.Limit; +import io.trino.sql.tree.LambdaArgumentDeclaration; +import io.trino.sql.tree.LambdaExpression; +import io.trino.sql.tree.LikePredicate; +import io.trino.sql.tree.LogicalExpression; +import io.trino.sql.tree.LongLiteral; +import io.trino.sql.tree.NodeLocation; +import io.trino.sql.tree.NotExpression; +import io.trino.sql.tree.NullLiteral; +import io.trino.sql.tree.NullIfExpression; +import io.trino.sql.tree.Offset; +import io.trino.sql.tree.OrderBy; +import io.trino.sql.tree.Parameter; +import io.trino.sql.tree.Predicated; +import io.trino.sql.tree.QualifiedName; +import io.trino.sql.tree.Query; +import io.trino.sql.tree.QueryBody; +import io.trino.sql.tree.QuerySpecification; +import io.trino.sql.tree.Row; +import io.trino.sql.tree.SearchedCaseExpression; +import io.trino.sql.tree.Select; +import io.trino.sql.tree.SelectItem; +import io.trino.sql.tree.SimpleCaseExpression; +import io.trino.sql.tree.SimpleGroupBy; +import io.trino.sql.tree.SimpleIntervalQualifier; +import io.trino.sql.tree.SingleColumn; +import io.trino.sql.tree.SortItem.NullOrdering; +import io.trino.sql.tree.SortItem.Ordering; +import io.trino.sql.tree.Statement; +import io.trino.sql.tree.StringLiteral; +import io.trino.sql.tree.Table; +import io.trino.sql.tree.TableSubquery; +import io.trino.sql.tree.Union; +import io.trino.sql.tree.Unnest; +import io.trino.sql.tree.Values; +import io.trino.sql.tree.WhenClause; +import io.trino.sql.tree.With; +import io.trino.sql.tree.WithQuery; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalInt; + +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE; + +final class TrinoAstFactory +{ + private static final SqlParser SQL_PARSER = new SqlParser(); + + private TrinoAstFactory() {} + + public static Statement createStatement(HogQlQuery query, Map parameterIds) + { + return createQuery(query, parameterIds); + } + + private static Query createQuery(HogQlQuery query, Map parameterIds) + { + query = HogQlLimitByRewriter.rewrite(query); + NodeLocation location = location(query.span()); + QueryBody queryBody = createQueryBody(query, parameterIds); + boolean setOperation = query.body() instanceof SetOperation; + return new Query( + location, + List.of(), + List.of(), + createWith(query, parameterIds), + queryBody, + setOperation ? createOrderBy(query, parameterIds) : Optional.empty(), + setOperation ? query.offset().map(offset -> new Offset(location(offset.span()), createExpression(offset, parameterIds))) : Optional.empty(), + setOperation ? query.limit().map(limit -> new Limit(location(limit.span()), createExpression(limit, parameterIds))) : Optional.empty()); + } + + private static QueryBody createQueryBody(HogQlQuery query, Map parameterIds) + { + return switch (query.body()) { + case SelectQueryBody select -> createQuerySpecification(query, select, parameterIds); + case SetOperation setOperation -> createSetOperation(setOperation, parameterIds); + }; + } + + private static QuerySpecification createQuerySpecification( + HogQlQuery query, + SelectQueryBody select, + Map parameterIds) + { + NodeLocation location = location(select.span()); + QuerySpecification querySpecification = new QuerySpecification( + location, + new Select(location, select.distinct(), select.projections().stream() + .flatMap(projection -> createSelectItems(projection, parameterIds).stream()) + .toList()), + select.from().map(relation -> createRelation(relation, parameterIds)), + select.where().map(expression -> createExpression(expression, parameterIds)), + createGroupBy(select.groupBy(), parameterIds), + select.having().map(expression -> createExpression(expression, parameterIds)), + select.windows().stream() + .map(window -> createWindowDefinition(window, parameterIds)) + .toList(), + createOrderBy(query, parameterIds), + query.offset().map(offset -> new Offset(location(offset.span()), createExpression(offset, parameterIds))), + query.limit().map(limit -> new Limit(location(limit.span()), createExpression(limit, parameterIds)))); + return querySpecification; + } + + private static QueryBody createSetOperation(SetOperation setOperation, Map parameterIds) + { + Deque pending = new ArrayDeque<>(); + Map lowered = new IdentityHashMap<>(); + pending.push(new SetOperationFrame(setOperation, false)); + + while (!pending.isEmpty()) { + SetOperationFrame frame = pending.pop(); + if (!frame.operandsLowered()) { + pending.push(new SetOperationFrame(frame.operation(), true)); + addInlineSetOperand(pending, frame.operation().right(), frame.operation().rightParenthesized()); + addInlineSetOperand(pending, frame.operation().left(), frame.operation().leftParenthesized()); + continue; + } + + SetOperation operation = frame.operation(); + QueryBody left = createSetOperand(operation.left(), operation.leftParenthesized(), parameterIds, lowered); + QueryBody right = createSetOperand(operation.right(), operation.rightParenthesized(), parameterIds, lowered); + lowered.put(operation, createSetOperation(operation, left, right)); + } + return lowered.get(setOperation); + } + + private static void addInlineSetOperand(Deque pending, HogQlQuery query, boolean parenthesized) + { + if (!requiresQueryWrapper(query, parenthesized) && query.body() instanceof SetOperation operation) { + pending.push(new SetOperationFrame(operation, false)); + } + } + + private static QueryBody createSetOperation(SetOperation setOperation, QueryBody left, QueryBody right) + { + NodeLocation location = location(setOperation.operatorSpan()); + return switch (setOperation.type()) { + case EXCEPT -> new Except(location, left, right, setOperation.distinct(), Optional.empty()); + case INTERSECT -> new Intersect(location, List.of(left, right), setOperation.distinct(), Optional.empty()); + case UNION -> new Union(location, List.of(left, right), setOperation.distinct(), Optional.empty()); + }; + } + + private static QueryBody createSetOperand( + HogQlQuery query, + boolean parenthesized, + Map parameterIds, + Map lowered) + { + if (requiresQueryWrapper(query, parenthesized)) { + return new TableSubquery(location(query.span()), createQuery(query, parameterIds)); + } + if (query.body() instanceof SetOperation setOperation) { + QueryBody result = lowered.get(setOperation); + if (result == null) { + throw new IllegalStateException("set operation operand has not been lowered"); + } + return result; + } + return createQueryBody(query, parameterIds); + } + + private static boolean requiresQueryWrapper(HogQlQuery query, boolean parenthesized) + { + return parenthesized || !query.with().isEmpty() || !query.orderBy().isEmpty() || query.limit().isPresent() || query.offset().isPresent(); + } + + private record SetOperationFrame(SetOperation operation, boolean operandsLowered) {} + + private static Optional createWith(HogQlQuery query, Map parameterIds) + { + if (query.with().isEmpty()) { + return Optional.empty(); + } + return Optional.of(new With( + location(query.span()), + false, + query.with().stream() + .map(commonTable -> createWithQuery(commonTable, parameterIds)) + .toList())); + } + + private static WithQuery createWithQuery(CommonTableExpression commonTable, Map parameterIds) + { + return new WithQuery( + location(commonTable.span()), + createIdentifier(commonTable.name()), + createQuery(commonTable.query(), parameterIds), + commonTable.columnAliases().isEmpty() + ? Optional.empty() + : Optional.of(commonTable.columnAliases().stream() + .map(TrinoAstFactory::createIdentifier) + .toList())); + } + + private static Optional createOrderBy(HogQlQuery query, Map parameterIds) + { + return createOrderBy(query.orderBy(), parameterIds); + } + + private static Optional createOrderBy(List sortItems, Map parameterIds) + { + if (sortItems.isEmpty()) { + return Optional.empty(); + } + return Optional.of(new OrderBy( + location(sortItems.getFirst().span()), + sortItems.stream() + .map(sortItem -> new io.trino.sql.tree.SortItem( + location(sortItem.span()), + createExpression(sortItem.expression(), parameterIds), + switch (sortItem.direction()) { + case ASCENDING -> Ordering.ASCENDING; + case DESCENDING -> Ordering.DESCENDING; + }, + switch (sortItem.nullPlacement()) { + case FIRST -> NullOrdering.FIRST; + case LAST -> NullOrdering.LAST; + case UNDEFINED -> NullOrdering.UNDEFINED; + })) + .toList())); + } + + private static Optional createGroupBy(List expressions, Map parameterIds) + { + if (expressions.isEmpty()) { + return Optional.empty(); + } + NodeLocation location = location(expressions.getFirst().span()); + return Optional.of(new GroupBy( + location, + false, + List.of(new SimpleGroupBy( + location, + expressions.stream() + .map(expression -> createExpression(expression, parameterIds)) + .toList())))); + } + + private static List createSelectItems(Projection projection, Map parameterIds) + { + return switch (projection) { + case ColumnsList columns -> columns.expressions().stream() + .map(expression -> new SingleColumn( + location(expression.span()), + createExpression(expression, parameterIds), + Optional.empty())) + .map(SelectItem.class::cast) + .toList(); + case ColumnsRegex columns -> throw unsupportedColumns(columns.span()); + case Star star -> List.of(createAllColumns(star)); + case ExpressionProjection expression -> List.of(new SingleColumn( + location(expression.span()), + createExpression(expression.expression(), parameterIds), + expression.alias().map(TrinoAstFactory::createIdentifier))); + }; + } + + private static AllColumns createAllColumns(Star star) + { + if (!star.replacements().isEmpty()) { + Identifier target = star.replacements().getFirst().target(); + throw unsupportedSemanticExpression( + target.span(), + "HogQL star replacement requires a logical relation from the semantic catalog: " + target.value()); + } + if (!star.exclusions().isEmpty()) { + ColumnReference exclusion = star.exclusions().getFirst(); + throw unsupportedSemanticExpression( + exclusion.span(), + "HogQL star exclusions require a logical relation from the semantic catalog: " + + String.join(".", exclusion.parts().stream().map(Identifier::value).toList())); + } + Optional target = star.qualifier().isEmpty() + ? Optional.empty() + : Optional.of(createColumnReference(new ColumnReference(star.qualifier(), star.span()))); + return new AllColumns(location(star.span()), target, List.of()); + } + + private static TrinoException unsupportedColumns(SourceSpan span) + { + return unsupportedSemanticExpression(span, "HogQL COLUMNS requires a logical relation from the semantic catalog"); + } + + private static Expression createExpression(HogQlQuery.Expression expression, Map parameterIds) + { + return switch (expression) { + case ArrayExpression array -> new Array( + location(array.span()), + array.values().stream() + .map(value -> createExpression(value, parameterIds)) + .toList()); + case BetweenExpression between -> createBetweenExpression(between, parameterIds); + case BinaryExpression binary -> createBinaryExpression(binary, parameterIds); + case CaseExpression caseExpression -> createCaseExpression(caseExpression, parameterIds); + case CastExpression cast -> new Cast( + location(cast.span()), + createExpression(cast.value(), parameterIds), + createCastType(cast.type(), cast.typeDialect()), + cast.safe()); + case ColumnReference reference -> createColumnReference(reference); + case FunctionCall function -> createFunctionCall(function, parameterIds); + case InCohortExpression in -> throw unsupportedSemanticExpression(in.span(), "HogQL IN COHORT requires a semantic catalog snapshot"); + case InExpression in -> createInExpression(in, parameterIds); + case InSubqueryExpression in -> createInSubqueryExpression(in, parameterIds); + case IntervalExpression interval -> createIntervalExpression(interval, parameterIds); + case IsNullExpression isNull -> new Predicated( + location(isNull.predicateSpan()), + createExpression(isNull.value(), parameterIds), + new IsNullPredicate(location(isNull.predicateSpan()), isNull.negated())); + case HogQlQuery.LambdaExpression lambda -> new LambdaExpression( + location(lambda.span()), + lambda.arguments().stream() + .map(argument -> new LambdaArgumentDeclaration(location(argument.span()), createIdentifier(argument))) + .toList(), + createExpression(lambda.body(), parameterIds)); + case Literal literal -> createLiteral(literal); + case MemberAccessExpression memberAccess -> new DereferenceExpression( + location(memberAccess.span()), + createExpression(memberAccess.base(), parameterIds), + createIdentifier(memberAccess.member())); + case Placeholder placeholder -> new Parameter(location(placeholder.span()), parameterIds.get(placeholder.span())); + case ScalarSubqueryExpression subquery -> new io.trino.sql.tree.SubqueryExpression( + location(subquery.span()), + createQuery(subquery.query(), parameterIds)); + case SubscriptExpression subscript -> new io.trino.sql.tree.SubscriptExpression( + location(subscript.span()), + createExpression(subscript.base(), parameterIds), + createExpression(subscript.index(), parameterIds)); + case TupleExpression tuple -> new Row( + location(tuple.span()), + tuple.values().stream() + .map(value -> new Row.Field(location(value.span()), Optional.empty(), createExpression(value, parameterIds))) + .toList()); + case UnaryExpression unary -> switch (unary.operator()) { + case NEGATE -> ArithmeticUnaryExpression.negative(location(unary.span()), createExpression(unary.operand(), parameterIds)); + case NOT -> new NotExpression(location(unary.span()), createExpression(unary.operand(), parameterIds)); + case POSITIVE -> ArithmeticUnaryExpression.positive(location(unary.span()), createExpression(unary.operand(), parameterIds)); + }; + }; + } + + private static Expression createIntervalExpression(IntervalExpression interval, Map parameterIds) + { + NodeLocation location = location(interval.span()); + int multiplier = switch (interval.unit()) { + case WEEK -> 7; + case QUARTER -> 3; + default -> 1; + }; + IntervalField field = switch (interval.unit()) { + case SECOND -> new IntervalField.Second(OptionalInt.empty()); + case MINUTE -> new IntervalField.Minute(); + case HOUR -> new IntervalField.Hour(); + case DAY, WEEK -> new IntervalField.Day(); + case MONTH, QUARTER -> new IntervalField.Month(); + case YEAR -> new IntervalField.Year(); + }; + IntervalLiteral unitInterval = new IntervalLiteral( + location, + Integer.toString(multiplier), + IntervalLiteral.Sign.POSITIVE, + new SimpleIntervalQualifier(location, OptionalInt.empty(), field)); + return new ArithmeticBinaryExpression( + location, + ArithmeticBinaryExpression.Operator.MULTIPLY, + createExpression(interval.value(), parameterIds), + unitInterval); + } + + private static io.trino.sql.tree.DataType createCastType(Identifier type, CastTypeDialect typeDialect) + { + if (typeDialect == CastTypeDialect.TRINO) { + return SQL_PARSER.createType(type.value()); + } + try { + return SQL_PARSER.createType(HogQlCastTypeTranslator.translate(type.value())); + } + catch (IllegalArgumentException exception) { + throw unsupportedSemanticExpression( + type.span(), + "HogQL cast type cannot be represented exactly in Trino: " + type.value() + " (" + exception.getMessage() + ")"); + } + } + + private static Expression createFunctionCall(FunctionCall function, Map parameterIds) + { + if (function.nameParts().size() == 1 && function.name().value().equalsIgnoreCase("matchesAction")) { + throw unsupportedSemanticExpression(function.span(), "HogQL matchesAction requires a semantic catalog snapshot and events relation"); + } + List arguments = function.arguments().stream() + .map(argument -> createExpression(argument, parameterIds)) + .toList(); + if (function.nameParts().size() == 1 && function.name().value().equalsIgnoreCase("coalesce")) { + if (arguments.size() == 1) { + return arguments.getFirst(); + } + return new CoalesceExpression(location(function.span()), arguments); + } + if (function.nameParts().size() == 1 && function.name().value().equalsIgnoreCase("if")) { + return new IfExpression( + location(function.span()), + arguments.get(0), + arguments.get(1), + arguments.size() == 3 ? arguments.get(2) : null); + } + if (function.nameParts().size() == 1 && function.name().value().equalsIgnoreCase("nullif")) { + return new NullIfExpression(location(function.span()), arguments.get(0), arguments.get(1)); + } + List callArguments = new ArrayList<>(); + for (int index = 0; index < arguments.size(); index++) { + callArguments.add(new CallArgument(location(function.arguments().get(index).span()), Optional.empty(), arguments.get(index))); + } + if (function.nameParts().size() == 1 && function.name().value().equalsIgnoreCase("array_join") && callArguments.size() == 1) { + callArguments.add(new CallArgument(location(function.span()), Optional.empty(), new StringLiteral(location(function.span()), ""))); + } + return new io.trino.sql.tree.FunctionCall( + location(function.span()), + QualifiedName.of(function.nameParts().stream() + .map(TrinoAstFactory::createIdentifier) + .toList()), + function.window().map(window -> createWindow(window, parameterIds)), + function.filter().map(filter -> createExpression(filter, parameterIds)), + createOrderBy(function.orderBy(), parameterIds), + function.distinct(), + function.nullTreatment().map(_ -> io.trino.sql.tree.FunctionCall.NullTreatment.IGNORE), + Optional.empty(), + callArguments); + } + + private static io.trino.sql.tree.WindowDefinition createWindowDefinition( + WindowDefinition definition, + Map parameterIds) + { + return new io.trino.sql.tree.WindowDefinition( + location(definition.span()), + createIdentifier(definition.name()), + (io.trino.sql.tree.WindowSpecification) createWindow(definition.specification(), parameterIds)); + } + + private static io.trino.sql.tree.Window createWindow(Window window, Map parameterIds) + { + return switch (window) { + case WindowReference reference -> new io.trino.sql.tree.WindowReference( + location(reference.span()), + createIdentifier(reference.name())); + case WindowSpecification specification -> new io.trino.sql.tree.WindowSpecification( + location(specification.span()), + Optional.empty(), + specification.partitionBy().stream() + .map(expression -> createExpression(expression, parameterIds)) + .toList(), + createOrderBy(specification.orderBy(), parameterIds), + specification.frame().map(frame -> createWindowFrame(frame, parameterIds))); + }; + } + + private static io.trino.sql.tree.WindowFrame createWindowFrame( + WindowFrame frame, + Map parameterIds) + { + return new io.trino.sql.tree.WindowFrame( + location(frame.span()), + switch (frame.type()) { + case RANGE -> io.trino.sql.tree.WindowFrame.Type.RANGE; + case ROWS -> io.trino.sql.tree.WindowFrame.Type.ROWS; + }, + createFrameBound(frame.start(), parameterIds), + frame.end().map(bound -> createFrameBound(bound, parameterIds)), + List.of(), + Optional.empty(), + Optional.empty(), + Optional.empty(), + List.of(), + List.of()); + } + + private static io.trino.sql.tree.FrameBound createFrameBound( + FrameBound bound, + Map parameterIds) + { + io.trino.sql.tree.FrameBound.Type type = switch (bound.type()) { + case CURRENT_ROW -> io.trino.sql.tree.FrameBound.Type.CURRENT_ROW; + case FOLLOWING -> io.trino.sql.tree.FrameBound.Type.FOLLOWING; + case PRECEDING -> io.trino.sql.tree.FrameBound.Type.PRECEDING; + case UNBOUNDED_FOLLOWING -> io.trino.sql.tree.FrameBound.Type.UNBOUNDED_FOLLOWING; + case UNBOUNDED_PRECEDING -> io.trino.sql.tree.FrameBound.Type.UNBOUNDED_PRECEDING; + }; + return bound.value() + .map(value -> new io.trino.sql.tree.FrameBound( + location(bound.span()), + type, + createExpression(value, parameterIds))) + .orElseGet(() -> new io.trino.sql.tree.FrameBound(location(bound.span()), type)); + } + + private static Expression createCaseExpression(CaseExpression caseExpression, Map parameterIds) + { + NodeLocation location = location(caseExpression.span()); + List whenClauses = caseExpression.whenClauses().stream() + .map(when -> new WhenClause( + location(when.span()), + createExpression(when.operand(), parameterIds), + createExpression(when.result(), parameterIds))) + .toList(); + Optional defaultValue = caseExpression.defaultValue().map(value -> createExpression(value, parameterIds)); + return caseExpression.operand() + .map(operand -> new SimpleCaseExpression(location, createExpression(operand, parameterIds), whenClauses, defaultValue)) + .orElseGet(() -> new SearchedCaseExpression(location, whenClauses, defaultValue)); + } + + private static Expression createBetweenExpression(BetweenExpression between, Map parameterIds) + { + NodeLocation location = location(between.predicateSpan()); + return new Predicated( + location, + createExpression(between.value(), parameterIds), + new BetweenPredicate( + location, + between.negated(), + Optional.empty(), + createExpression(between.min(), parameterIds), + createExpression(between.max(), parameterIds))); + } + + private static Expression createInExpression(InExpression in, Map parameterIds) + { + NodeLocation location = location(in.predicateSpan()); + Expression testedValue = createExpression(in.value(), parameterIds); + if (in.values().isEmpty()) { + return new IfExpression( + location, + new Predicated(location, testedValue, new IsNullPredicate(location, false)), + new NullLiteral(location), + new BooleanLiteral(location, Boolean.toString(in.negated()))); + } + return new Predicated( + location, + testedValue, + new InPredicate( + location, + in.negated(), + new InListExpression( + location, + in.values().stream() + .map(value -> createExpression(value, parameterIds)) + .toList()))); + } + + private static Expression createInSubqueryExpression(InSubqueryExpression in, Map parameterIds) + { + NodeLocation location = location(in.predicateSpan()); + return new Predicated( + location, + createExpression(in.value(), parameterIds), + new InPredicate( + location, + in.negated(), + new io.trino.sql.tree.SubqueryExpression(location(in.query().span()), createQuery(in.query(), parameterIds)))); + } + + private static Expression createBinaryExpression(BinaryExpression binary, Map parameterIds) + { + NodeLocation location = location(binary.span()); + Expression left = createExpression(binary.left(), parameterIds); + Expression right = createExpression(binary.right(), parameterIds); + return switch (binary.operator()) { + case ADD -> new ArithmeticBinaryExpression(location, ArithmeticBinaryExpression.Operator.ADD, left, right); + case SUBTRACT -> new ArithmeticBinaryExpression(location, ArithmeticBinaryExpression.Operator.SUBTRACT, left, right); + case MULTIPLY -> new ArithmeticBinaryExpression(location, ArithmeticBinaryExpression.Operator.MULTIPLY, left, right); + case DIVIDE -> new ArithmeticBinaryExpression(location, ArithmeticBinaryExpression.Operator.DIVIDE, left, right); + case MODULO -> new ArithmeticBinaryExpression(location, ArithmeticBinaryExpression.Operator.MODULO, left, right); + case AND -> new LogicalExpression(location, LogicalExpression.Operator.AND, List.of(left, right)); + case CONCAT -> new io.trino.sql.tree.FunctionCall( + location, + QualifiedName.of("concat"), + List.of(left, right)); + case OR -> new LogicalExpression(location, LogicalExpression.Operator.OR, List.of(left, right)); + case EQUAL -> comparison(location, ComparisonPredicate.Operator.EQUAL, left, right); + case NOT_EQUAL -> comparison(location, ComparisonPredicate.Operator.NOT_EQUAL, left, right); + case LESS_THAN -> comparison(location, ComparisonPredicate.Operator.LESS_THAN, left, right); + case LESS_THAN_OR_EQUAL -> comparison(location, ComparisonPredicate.Operator.LESS_THAN_OR_EQUAL, left, right); + case GREATER_THAN -> comparison(location, ComparisonPredicate.Operator.GREATER_THAN, left, right); + case GREATER_THAN_OR_EQUAL -> comparison(location, ComparisonPredicate.Operator.GREATER_THAN_OR_EQUAL, left, right); + case LIKE -> like(location, left, right, false, false); + case NOT_LIKE -> like(location, left, right, true, false); + case ILIKE -> like(location, left, right, false, true); + case NOT_ILIKE -> like(location, left, right, true, true); + }; + } + + private static Expression like(NodeLocation location, Expression value, Expression pattern, boolean negated, boolean caseInsensitive) + { + if (caseInsensitive) { + value = lower(location, value); + pattern = lower(location, pattern); + } + return new Predicated(location, value, new LikePredicate(location, negated, pattern, Optional.empty())); + } + + private static Expression lower(NodeLocation location, Expression value) + { + return new io.trino.sql.tree.FunctionCall( + location, + QualifiedName.of("lower"), + Optional.empty(), + Optional.empty(), + Optional.empty(), + false, + Optional.empty(), + Optional.empty(), + List.of(new CallArgument(location, Optional.empty(), value))); + } + + private static Expression comparison(NodeLocation location, ComparisonPredicate.Operator operator, Expression left, Expression right) + { + return new Predicated(location, left, new ComparisonPredicate(location, operator, right)); + } + + private static Expression createLiteral(Literal literal) + { + NodeLocation location = location(literal.span()); + return switch (literal.kind()) { + case BOOLEAN -> new BooleanLiteral(location, literal.value()); + case FLOAT -> new DoubleLiteral(location, literal.value()); + case INTEGER -> new LongLiteral(location, literal.value()); + case NULL -> new NullLiteral(location); + case STRING -> new StringLiteral(location, literal.value()); + }; + } + + private static Expression createColumnReference(ColumnReference reference) + { + List parts = reference.parts(); + Expression expression = createIdentifier(parts.getFirst()); + for (Identifier part : parts.subList(1, parts.size())) { + expression = new DereferenceExpression(location(reference.span()), expression, createIdentifier(part)); + } + return expression; + } + + private static io.trino.sql.tree.Relation createRelation(Relation relation, Map parameterIds) + { + return switch (relation) { + case AliasedRelation alias -> new io.trino.sql.tree.AliasedRelation( + location(alias.span()), + createRelation(alias.relation(), parameterIds), + createIdentifier(alias.alias()), + alias.columnAliases().isEmpty() + ? null + : alias.columnAliases().stream() + .map(TrinoAstFactory::createIdentifier) + .toList()); + case CommonTableReference commonTable -> new Table( + location(commonTable.span()), + QualifiedName.of(List.of(createIdentifier(commonTable.name())))); + case JoinRelation join -> createJoin(join, parameterIds); + case PivotRelation pivot -> new io.trino.sql.tree.Pivot( + location(pivot.span()), + createRelation(pivot.input(), parameterIds), + pivot.aggregations().stream() + .map(aggregation -> new io.trino.sql.tree.PivotAggregation( + location(aggregation.span()), + createExpression(aggregation.expression(), parameterIds), + aggregation.alias().map(TrinoAstFactory::createIdentifier))) + .toList(), + pivot.pivotColumns().stream() + .map(column -> createExpression(column, parameterIds)) + .toList(), + pivot.valueGroups().stream() + .map(group -> new io.trino.sql.tree.PivotValueGroup( + location(group.span()), + group.values().stream() + .map(value -> createExpression(value, parameterIds)) + .toList(), + group.alias().map(TrinoAstFactory::createIdentifier))) + .toList(), + createGroupBy(pivot.groupBy(), parameterIds)); + case SubqueryRelation subquery -> new TableSubquery(location(subquery.span()), createQuery(subquery.query(), parameterIds)); + case TablePlaceholder _ -> throw new IllegalArgumentException("table placeholder was not validated"); + case TableReference table -> createTable(table); + case UnnestRelation unnest -> new io.trino.sql.tree.AliasedRelation( + location(unnest.span()), + new Unnest( + location(unnest.span()), + unnest.expressions().stream().map(expression -> createExpression(expression, parameterIds)).toList(), + false), + createIdentifier(unnest.alias()), + unnest.columnAliases().stream().map(TrinoAstFactory::createIdentifier).toList()); + case ValuesRelation values -> createValuesRelation(values, parameterIds); + }; + } + + private static io.trino.sql.tree.Relation createJoin(JoinRelation join, Map parameterIds) + { + if (join.type() == HogQlQuery.JoinType.LEFT_ANY || join.type() == HogQlQuery.JoinType.INNER_ANY) { + return createAnyJoin(join, parameterIds); + } + return new io.trino.sql.tree.Join( + location(join.span()), + switch (join.type()) { + case CROSS -> io.trino.sql.tree.Join.Type.CROSS; + case INNER -> io.trino.sql.tree.Join.Type.INNER; + case INNER_ANY -> throw new IllegalStateException("INNER ANY JOIN must be lowered as a lateral join"); + case LEFT -> io.trino.sql.tree.Join.Type.LEFT; + case LEFT_ANY -> throw new IllegalStateException("LEFT ANY JOIN must be lowered as a lateral join"); + case RIGHT -> io.trino.sql.tree.Join.Type.RIGHT; + case FULL -> io.trino.sql.tree.Join.Type.FULL; + }, + createRelation(join.left(), parameterIds), + createRelation(join.right(), parameterIds), + join.criteria().map(criteria -> switch (criteria) { + case JoinOn on -> new io.trino.sql.tree.JoinOn(createExpression(on.expression(), parameterIds)); + case JoinUsing using -> new io.trino.sql.tree.JoinUsing(using.columns().stream() + .map(TrinoAstFactory::createIdentifier) + .toList()); + })); + } + + private static io.trino.sql.tree.Relation createAnyJoin(JoinRelation join, Map parameterIds) + { + HogQlQuery.JoinCriteria criteria = join.criteria().orElseThrow(); + Identifier rightAlias = anyJoinAlias(join.right()) + .orElseThrow(() -> unsupportedSemanticExpression( + join.right().span(), + "HogQL ANY JOIN requires a right relation alias for Trino correlation")); + HogQlQuery.Expression predicate = switch (criteria) { + case JoinOn on -> on.expression(); + case JoinUsing using -> anyJoinUsingPredicate(join.left(), rightAlias, using); + }; + HogQlQuery.Expression orderingKey = anyJoinOrderingKey(predicate, rightAlias) + .orElseThrow(() -> unsupportedSemanticExpression( + criteria.span(), + "HogQL ANY JOIN requires an equality predicate on a qualified right column for Trino decorrelation")); + NodeLocation rightLocation = location(join.right().span()); + QuerySpecification matches = new QuerySpecification( + rightLocation, + new Select(rightLocation, false, List.of(new AllColumns(rightLocation))), + Optional.of(createRelation(join.right(), parameterIds)), + Optional.of(createExpression(predicate, parameterIds)), + Optional.empty(), + Optional.empty(), + List.of(), + Optional.of(new OrderBy( + location(orderingKey.span()), + List.of(new io.trino.sql.tree.SortItem( + location(orderingKey.span()), + createExpression(orderingKey, parameterIds), + Ordering.ASCENDING, + NullOrdering.UNDEFINED)))), + Optional.empty(), + Optional.of(new Limit(rightLocation, new LongLiteral(rightLocation, "1")))); + Query query = new Query( + rightLocation, + List.of(), + List.of(), + Optional.empty(), + matches, + Optional.empty(), + Optional.empty(), + Optional.empty()); + io.trino.sql.tree.Relation lateral = new Lateral(rightLocation, query); + lateral = new io.trino.sql.tree.AliasedRelation( + rightLocation, + lateral, + createIdentifier(rightAlias), + null); + return new io.trino.sql.tree.Join( + location(join.span()), + join.type() == HogQlQuery.JoinType.LEFT_ANY ? io.trino.sql.tree.Join.Type.LEFT : io.trino.sql.tree.Join.Type.INNER, + createRelation(join.left(), parameterIds), + lateral, + Optional.of(new io.trino.sql.tree.JoinOn(new BooleanLiteral(location(join.span()), "true")))); + } + + private static HogQlQuery.Expression anyJoinUsingPredicate(Relation left, Identifier rightAlias, JoinUsing using) + { + Identifier leftAlias = anyJoinAlias(left) + .orElseThrow(() -> unsupportedSemanticExpression( + left.span(), + "HogQL ANY JOIN with USING requires a left relation alias for Trino correlation")); + HogQlQuery.Expression predicate = null; + for (Identifier column : using.columns()) { + ColumnReference leftColumn = new ColumnReference(List.of(leftAlias, column), using.span()); + ColumnReference rightColumn = new ColumnReference(List.of(rightAlias, column), using.span()); + HogQlQuery.Expression equality = new BinaryExpression(HogQlQuery.BinaryOperator.EQUAL, leftColumn, rightColumn, using.span()); + predicate = predicate == null + ? equality + : new BinaryExpression(HogQlQuery.BinaryOperator.AND, predicate, equality, using.span()); + } + if (predicate == null) { + throw unsupportedSemanticExpression(using.span(), "HogQL ANY JOIN USING requires at least one column"); + } + return predicate; + } + + private static Optional anyJoinOrderingKey(HogQlQuery.Expression expression, Identifier rightAlias) + { + if (!(expression instanceof BinaryExpression binary)) { + return Optional.empty(); + } + if (binary.operator() == HogQlQuery.BinaryOperator.AND) { + return anyJoinOrderingKey(binary.left(), rightAlias) + .or(() -> anyJoinOrderingKey(binary.right(), rightAlias)); + } + if (binary.operator() != HogQlQuery.BinaryOperator.EQUAL) { + return Optional.empty(); + } + if (isQualifiedBy(binary.left(), rightAlias)) { + return Optional.of(binary.left()); + } + if (isQualifiedBy(binary.right(), rightAlias)) { + return Optional.of(binary.right()); + } + return Optional.empty(); + } + + private static boolean isQualifiedBy(HogQlQuery.Expression expression, Identifier qualifier) + { + return expression instanceof ColumnReference reference && + reference.parts().size() > 1 && + reference.parts().getFirst().value().equalsIgnoreCase(qualifier.value()); + } + + private static Optional anyJoinAlias(Relation relation) + { + return switch (relation) { + case AliasedRelation alias -> Optional.of(alias.alias()); + case CommonTableReference commonTable -> Optional.of(commonTable.name()); + case JoinRelation join -> throw unsupportedSemanticExpression( + join.span(), + "HogQL LEFT ANY JOIN with a compound right relation cannot preserve relation qualifiers in Trino"); + case TableReference table -> { + if (table.parts().size() > 1) { + throw unsupportedSemanticExpression( + table.span(), + "HogQL LEFT ANY JOIN with an unaliased qualified table cannot preserve its qualifier in Trino"); + } + yield Optional.of(table.parts().getLast()); + } + case UnnestRelation unnest -> Optional.of(unnest.alias()); + case PivotRelation _, SubqueryRelation _, TablePlaceholder _, ValuesRelation _ -> Optional.empty(); + }; + } + + private static TableSubquery createValuesRelation(ValuesRelation values, Map parameterIds) + { + NodeLocation valuesLocation = new NodeLocation(values.span().startLine(), values.span().startColumn() + 1); + Values body = new Values( + valuesLocation, + values.rows().stream() + .map(row -> createValuesRow(row, parameterIds)) + .toList()); + Query query = new Query( + valuesLocation, + List.of(), + List.of(), + Optional.empty(), + body, + Optional.empty(), + Optional.empty(), + Optional.empty()); + return new TableSubquery(location(values.span()), query); + } + + private static Expression createValuesRow(List row, Map parameterIds) + { + if (row.size() == 1) { + return createExpression(row.getFirst(), parameterIds); + } + return new Row( + location(row.getFirst().span()), + row.stream() + .map(value -> new Row.Field(location(value.span()), Optional.empty(), createExpression(value, parameterIds))) + .toList()); + } + + private static Table createTable(TableReference table) + { + List parts = table.parts().stream() + .map(TrinoAstFactory::createIdentifier) + .toList(); + return new Table(location(table.span()), QualifiedName.of(parts)); + } + + private static io.trino.sql.tree.Identifier createIdentifier(Identifier identifier) + { + return new io.trino.sql.tree.Identifier( + location(identifier.span()), + identifier.value(), + identifier.delimited() || !isValidTrinoIdentifier(identifier.value())); + } + + private static boolean isValidTrinoIdentifier(String value) + { + if (value.isEmpty() || !isValidTrinoIdentifierFirstCharacter(value.charAt(0))) { + return false; + } + for (int index = 1; index < value.length(); index++) { + char character = value.charAt(index); + if (!isValidTrinoIdentifierFirstCharacter(character) && (character < '0' || character > '9')) { + return false; + } + } + return true; + } + + private static boolean isValidTrinoIdentifierFirstCharacter(char character) + { + return (character >= 'a' && character <= 'z') || + (character >= 'A' && character <= 'Z') || + character == '_'; + } + + private static NodeLocation location(SourceSpan span) + { + return new NodeLocation(span.startLine(), span.startColumn()); + } + + private static TrinoException unsupportedSemanticExpression(SourceSpan span, String message) + { + return new TrinoException( + HOGQL_UNSUPPORTED_FEATURE, + Optional.of(new Location(span.startLine(), span.startColumn())), + message, + null); + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/BoundedAsyncHogQlExchangeRateSnapshotCache.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/BoundedAsyncHogQlExchangeRateSnapshotCache.java new file mode 100644 index 000000000000..cca111bb1d6a --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/BoundedAsyncHogQlExchangeRateSnapshotCache.java @@ -0,0 +1,396 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.compiler.catalog.HogQlExchangeRateException.Failure; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Executor; +import java.util.function.LongSupplier; + +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +public final class BoundedAsyncHogQlExchangeRateSnapshotCache + implements HogQlExchangeRateSnapshotCache +{ + private final Object lock = new Object(); + private final int maximumEntries; + private final long refreshAfterNanos; + private final long expireAfterNanos; + private final long failureBackoffNanos; + private final LongSupplier ticker; + private final Executor executor; + private final SnapshotLoader loader; + private final LinkedHashMap entries = new LinkedHashMap<>(16, 0.75f, true); + private final LinkedHashMap observedGenerations = new LinkedHashMap<>(16, 0.75f, true); + + public BoundedAsyncHogQlExchangeRateSnapshotCache( + int maximumEntries, + Duration refreshAfter, + Duration expireAfter, + Duration failureBackoff, + LongSupplier ticker, + Executor executor, + SnapshotLoader loader) + { + if (maximumEntries <= 0) { + throw new IllegalArgumentException("maximumEntries must be positive"); + } + this.maximumEntries = maximumEntries; + refreshAfterNanos = nonNegativeNanos(refreshAfter, "refreshAfter"); + expireAfterNanos = positiveNanos(expireAfter, "expireAfter"); + failureBackoffNanos = nonNegativeNanos(failureBackoff, "failureBackoff"); + if (refreshAfterNanos > expireAfterNanos) { + throw new IllegalArgumentException("refreshAfter must not exceed expireAfter"); + } + this.ticker = requireNonNull(ticker, "ticker is null"); + this.executor = requireNonNull(executor, "executor is null"); + this.loader = requireNonNull(loader, "loader is null"); + } + + @Override + public Optional currentSnapshot(OptionalLong expectedGeneration) + { + expectedGeneration = HogQlExchangeRateSnapshotCache.validateExpectedGeneration(expectedGeneration); + RefreshTask refreshTask; + Optional result; + List evictedRefreshes = new ArrayList<>(1); + long now = ticker.getAsLong(); + synchronized (lock) { + Entry entry = entries.get(expectedGeneration); + if (entry == null) { + entry = inheritedExactEntry(expectedGeneration, now).orElseGet(Entry::new); + entries.put(expectedGeneration, entry); + } + HogQlExchangeRateSnapshot snapshot = entry.snapshot; + long age = snapshot == null ? Long.MAX_VALUE : elapsedNanos(entry.loadedAtNanos, now); + result = snapshot != null && age < expireAfterNanos ? Optional.of(snapshot) : Optional.empty(); + boolean refreshRequired = snapshot == null || + (expectedGeneration.isEmpty() && age >= refreshAfterNanos) || + (expectedGeneration.isPresent() && age >= expireAfterNanos); + refreshTask = refreshRequired ? prepareRefresh(expectedGeneration, entry, now, false) : null; + evictEntries(evictedRefreshes); + if (!entries.containsKey(expectedGeneration)) { + refreshTask = null; + } + } + cancelRefreshes(evictedRefreshes, "HogQL exchange-rate cache entry was evicted"); + dispatch(refreshTask); + return result; + } + + public CompletionStage prewarm(OptionalLong expectedGeneration) + { + expectedGeneration = HogQlExchangeRateSnapshotCache.validateExpectedGeneration(expectedGeneration); + RefreshTask refreshTask; + CompletableFuture future; + List evictedRefreshes = new ArrayList<>(1); + long now = ticker.getAsLong(); + synchronized (lock) { + Entry entry = entries.get(expectedGeneration); + if (entry == null) { + entry = inheritedExactEntry(expectedGeneration, now).orElseGet(Entry::new); + entries.put(expectedGeneration, entry); + } + boolean unexpired = entry.snapshot != null && elapsedNanos(entry.loadedAtNanos, now) < expireAfterNanos; + if (unexpired && expectedGeneration.isPresent()) { + refreshTask = null; + future = CompletableFuture.completedFuture(entry.snapshot); + } + else { + refreshTask = prepareRefresh(expectedGeneration, entry, now, true); + future = refreshTask == null ? entry.refresh : refreshTask.result(); + } + evictEntries(evictedRefreshes); + if (!entries.containsKey(expectedGeneration)) { + refreshTask = null; + } + } + cancelRefreshes(evictedRefreshes, "HogQL exchange-rate cache entry was evicted"); + dispatch(refreshTask); + return future.minimalCompletionStage(); + } + + public CompletionStage prewarm() + { + return prewarm(OptionalLong.empty()); + } + + public void invalidate() + { + List refreshes = new ArrayList<>(); + synchronized (lock) { + for (Entry entry : entries.values()) { + if (entry.refresh != null) { + refreshes.add(new RefreshCancellation(entry.refresh, entry.upstream)); + entry.refresh = null; + entry.upstream = null; + } + } + entries.clear(); + } + cancelRefreshes(refreshes, "HogQL exchange-rate cache was invalidated"); + } + + private Optional inheritedExactEntry(OptionalLong expectedGeneration, long now) + { + if (expectedGeneration.isEmpty()) { + return Optional.empty(); + } + Entry latest = entries.get(OptionalLong.empty()); + if (latest == null || latest.snapshot == null || latest.snapshot.generation() != expectedGeneration.orElseThrow()) { + return Optional.empty(); + } + if (elapsedNanos(latest.loadedAtNanos, now) >= expireAfterNanos) { + return Optional.empty(); + } + Entry inherited = new Entry(); + inherited.snapshot = latest.snapshot; + inherited.loadedAtNanos = latest.loadedAtNanos; + return Optional.of(inherited); + } + + private RefreshTask prepareRefresh(OptionalLong expectedGeneration, Entry entry, long now, boolean force) + { + if (entry.refresh != null) { + return null; + } + if (!force && entry.refreshBackoffActive && elapsedNanos(entry.lastRefreshFailureAtNanos, now) < failureBackoffNanos) { + return null; + } + CompletableFuture result = new CompletableFuture<>(); + entry.refresh = result; + return new RefreshTask(expectedGeneration, entry, result); + } + + private void dispatch(RefreshTask refreshTask) + { + if (refreshTask == null) { + return; + } + try { + executor.execute(() -> load(refreshTask)); + } + catch (RuntimeException e) { + completeFailure(refreshTask, e); + } + } + + private void load(RefreshTask refreshTask) + { + CompletionStage loaded; + try { + loaded = requireNonNull(loader.load(refreshTask.expectedGeneration()), "snapshot loader returned null"); + } + catch (RuntimeException failure) { + completeFailure(refreshTask, failure); + return; + } + catch (Error failure) { + completeFailure(refreshTask, failure); + throw failure; + } + CompletableFuture upstream = loaded.toCompletableFuture(); + boolean owned; + synchronized (lock) { + Entry entry = entries.get(refreshTask.expectedGeneration()); + owned = entry == refreshTask.entry() && entry.refresh == refreshTask.result(); + if (owned) { + entry.upstream = upstream; + } + } + if (!owned) { + upstream.cancel(true); + return; + } + loaded.whenComplete((snapshot, failure) -> { + if (failure != null) { + completeFailure(refreshTask, failure); + } + else if (snapshot == null) { + completeFailure(refreshTask, new NullPointerException("snapshot loader completed with null")); + } + else { + completeSuccess(refreshTask, snapshot); + } + }); + } + + private void completeSuccess(RefreshTask refreshTask, HogQlExchangeRateSnapshot snapshot) + { + Throwable rejection = validateLoadedSnapshot(refreshTask, snapshot); + if (rejection != null) { + completeFailure(refreshTask, rejection); + return; + } + + boolean published = false; + synchronized (lock) { + Entry entry = entries.get(refreshTask.expectedGeneration()); + if (entry == refreshTask.entry() && entry.refresh == refreshTask.result()) { + HogQlExchangeRateSnapshot observed = observedGenerations.get(snapshot.generation()); + if (observed != null && !observed.equals(snapshot)) { + rejection = reject(entry, "HogQL exchange-rate generation content changed"); + } + else if (refreshTask.expectedGeneration().isEmpty() && entry.snapshot != null && snapshot.generation() < entry.snapshot.generation()) { + rejection = reject(entry, format("HogQL exchange-rate generation regressed from %s to %s", entry.snapshot.generation(), snapshot.generation())); + } + else { + observedGenerations.put(snapshot.generation(), snapshot); + while (observedGenerations.size() > maximumEntries) { + observedGenerations.remove(observedGenerations.entrySet().iterator().next().getKey()); + } + entry.snapshot = snapshot; + entry.loadedAtNanos = ticker.getAsLong(); + entry.refreshBackoffActive = false; + entry.refresh = null; + entry.upstream = null; + published = true; + } + } + } + if (published) { + refreshTask.result().complete(snapshot); + } + else if (rejection != null) { + refreshTask.result().completeExceptionally(rejection); + } + else { + refreshTask.result().completeExceptionally(new CancellationException("HogQL exchange-rate refresh no longer owns its cache entry")); + } + } + + private Throwable reject(Entry entry, String message) + { + entry.refresh = null; + entry.upstream = null; + entry.lastRefreshFailureAtNanos = ticker.getAsLong(); + entry.refreshBackoffActive = true; + return new HogQlExchangeRateException(Failure.GENERATION_MISMATCH, message); + } + + private static Throwable validateLoadedSnapshot(RefreshTask refreshTask, HogQlExchangeRateSnapshot snapshot) + { + if (refreshTask.expectedGeneration().isPresent() && snapshot.generation() != refreshTask.expectedGeneration().orElseThrow()) { + return new HogQlExchangeRateException(Failure.GENERATION_MISMATCH, "HogQL exchange-rate snapshot generation does not match the refreshed generation"); + } + return null; + } + + private void completeFailure(RefreshTask refreshTask, Throwable failure) + { + requireNonNull(failure, "failure is null"); + boolean owned; + synchronized (lock) { + Entry entry = entries.get(refreshTask.expectedGeneration()); + owned = entry == refreshTask.entry() && entry.refresh == refreshTask.result(); + if (owned) { + entry.refresh = null; + entry.upstream = null; + entry.lastRefreshFailureAtNanos = ticker.getAsLong(); + entry.refreshBackoffActive = true; + } + } + if (owned) { + refreshTask.result().completeExceptionally(failure); + } + else { + refreshTask.result().completeExceptionally(new CancellationException("HogQL exchange-rate refresh no longer owns its cache entry")); + } + } + + private void evictEntries(List evictedRefreshes) + { + while (entries.size() > maximumEntries) { + Map.Entry victim = entries.entrySet().stream() + .filter(entry -> entry.getKey().isPresent()) + .findFirst() + .orElseGet(() -> entries.entrySet().iterator().next()); + entries.remove(victim.getKey()); + if (victim.getValue().refresh != null) { + evictedRefreshes.add(new RefreshCancellation(victim.getValue().refresh, victim.getValue().upstream)); + victim.getValue().refresh = null; + victim.getValue().upstream = null; + } + } + } + + private static void cancelRefreshes(List refreshes, String message) + { + refreshes.forEach(refresh -> { + if (refresh.upstream() != null) { + refresh.upstream().cancel(true); + } + refresh.result().completeExceptionally(new CancellationException(message)); + }); + } + + private static long elapsedNanos(long start, long end) + { + return Math.max(0, end - start); + } + + private static long nonNegativeNanos(Duration duration, String name) + { + requireNonNull(duration, name + " is null"); + if (duration.isNegative()) { + throw new IllegalArgumentException(name + " must not be negative"); + } + return duration.toNanos(); + } + + private static long positiveNanos(Duration duration, String name) + { + long nanos = nonNegativeNanos(duration, name); + if (nanos == 0) { + throw new IllegalArgumentException(name + " must be positive"); + } + return nanos; + } + + @FunctionalInterface + public interface SnapshotLoader + { + CompletionStage load(OptionalLong expectedGeneration); + } + + private static final class Entry + { + private HogQlExchangeRateSnapshot snapshot; + private long loadedAtNanos; + private long lastRefreshFailureAtNanos; + private boolean refreshBackoffActive; + private CompletableFuture refresh; + private CompletableFuture upstream; + } + + private record RefreshTask( + OptionalLong expectedGeneration, + Entry entry, + CompletableFuture result) {} + + private record RefreshCancellation( + CompletableFuture result, + CompletableFuture upstream) {} +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/BoundedAsyncHogQlSemanticCatalogSnapshotCache.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/BoundedAsyncHogQlSemanticCatalogSnapshotCache.java new file mode 100644 index 000000000000..db1eb7dcd1b2 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/BoundedAsyncHogQlSemanticCatalogSnapshotCache.java @@ -0,0 +1,455 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogException.Failure; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.CancellationException; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Executor; +import java.util.function.LongSupplier; + +import static java.lang.String.format; +import static java.util.Objects.requireNonNull; + +public final class BoundedAsyncHogQlSemanticCatalogSnapshotCache + implements HogQlSemanticCatalogSnapshotCache +{ + private final Object lock = new Object(); + private final int maximumEntries; + private final long refreshAfterNanos; + private final long expireAfterNanos; + private final long failureBackoffNanos; + private final LongSupplier ticker; + private final Executor executor; + private final SnapshotLoader loader; + private final LinkedHashMap entries = new LinkedHashMap<>(16, 0.75f, true); + private final LinkedHashMap observedGenerations = new LinkedHashMap<>(16, 0.75f, true); + + public BoundedAsyncHogQlSemanticCatalogSnapshotCache( + int maximumEntries, + Duration refreshAfter, + Duration expireAfter, + Duration failureBackoff, + LongSupplier ticker, + Executor executor, + SnapshotLoader loader) + { + if (maximumEntries <= 0) { + throw new IllegalArgumentException("maximumEntries must be positive"); + } + this.maximumEntries = maximumEntries; + this.refreshAfterNanos = nonNegativeNanos(refreshAfter, "refreshAfter"); + this.expireAfterNanos = positiveNanos(expireAfter, "expireAfter"); + this.failureBackoffNanos = nonNegativeNanos(failureBackoff, "failureBackoff"); + if (refreshAfterNanos > expireAfterNanos) { + throw new IllegalArgumentException("refreshAfter must not exceed expireAfter"); + } + this.ticker = requireNonNull(ticker, "ticker is null"); + this.executor = requireNonNull(executor, "executor is null"); + this.loader = requireNonNull(loader, "loader is null"); + } + + @Override + public Optional currentSnapshot(PhysicalIdentifier catalog) + { + return currentSnapshot(catalog, OptionalLong.empty()); + } + + @Override + public Optional currentSnapshot(PhysicalIdentifier catalog, OptionalLong expectedGeneration) + { + requireNonNull(catalog, "catalog is null"); + requireNonNull(expectedGeneration, "expectedGeneration is null"); + CacheKey cacheKey = new CacheKey(catalog, expectedGeneration); + RefreshTask refreshTask; + Optional result; + List evictedRefreshes = new ArrayList<>(1); + long now = ticker.getAsLong(); + synchronized (lock) { + Entry entry = entries.get(cacheKey); + if (entry == null) { + entry = inheritedExactEntry(cacheKey, now).orElseGet(Entry::new); + entries.put(cacheKey, entry); + } + + HogQlSemanticCatalogSnapshot snapshot = entry.snapshot; + long age = snapshot == null ? Long.MAX_VALUE : elapsedNanos(entry.loadedAtNanos, now); + result = snapshot != null && age < expireAfterNanos ? Optional.of(snapshot) : Optional.empty(); + boolean refreshRequired = snapshot == null || + (expectedGeneration.isEmpty() && age >= refreshAfterNanos) || + (expectedGeneration.isPresent() && age >= expireAfterNanos); + refreshTask = refreshRequired ? prepareRefresh(cacheKey, entry, now, false) : null; + evictEntries(evictedRefreshes); + if (!entries.containsKey(cacheKey)) { + refreshTask = null; + } + } + cancelRefreshes(evictedRefreshes, "HogQL semantic catalog cache entry was evicted"); + dispatch(refreshTask); + return result; + } + + public CompletionStage prewarm(PhysicalIdentifier catalog) + { + return prewarm(catalog, OptionalLong.empty()); + } + + public CompletionStage prewarm(PhysicalIdentifier catalog, OptionalLong expectedGeneration) + { + requireNonNull(catalog, "catalog is null"); + requireNonNull(expectedGeneration, "expectedGeneration is null"); + CacheKey cacheKey = new CacheKey(catalog, expectedGeneration); + RefreshTask refreshTask; + CompletableFuture future; + List evictedRefreshes = new ArrayList<>(1); + long now = ticker.getAsLong(); + synchronized (lock) { + Entry entry = entries.get(cacheKey); + if (entry == null) { + entry = inheritedExactEntry(cacheKey, now).orElseGet(Entry::new); + entries.put(cacheKey, entry); + } + boolean unexpired = entry.snapshot != null && elapsedNanos(entry.loadedAtNanos, now) < expireAfterNanos; + if (unexpired && expectedGeneration.isPresent()) { + refreshTask = null; + future = CompletableFuture.completedFuture(entry.snapshot); + } + else { + refreshTask = prepareRefresh(cacheKey, entry, now, true); + future = refreshTask == null ? entry.refresh : refreshTask.result(); + } + evictEntries(evictedRefreshes); + if (!entries.containsKey(cacheKey)) { + refreshTask = null; + } + } + cancelRefreshes(evictedRefreshes, "HogQL semantic catalog cache entry was evicted"); + dispatch(refreshTask); + return future.minimalCompletionStage(); + } + + public void invalidate(PhysicalIdentifier catalog) + { + requireNonNull(catalog, "catalog is null"); + List refreshes = new ArrayList<>(); + synchronized (lock) { + var iterator = entries.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry cacheEntry = iterator.next(); + if (cacheEntry.getKey().catalog().equals(catalog)) { + iterator.remove(); + if (cacheEntry.getValue().refresh != null) { + refreshes.add(new RefreshCancellation(cacheEntry.getValue().refresh, cacheEntry.getValue().upstream)); + cacheEntry.getValue().refresh = null; + cacheEntry.getValue().upstream = null; + } + } + } + } + cancelRefreshes(refreshes, "HogQL semantic catalog cache entry was invalidated"); + } + + private Optional inheritedExactEntry(CacheKey cacheKey, long now) + { + if (cacheKey.expectedGeneration().isEmpty()) { + return Optional.empty(); + } + Entry latest = entries.get(new CacheKey(cacheKey.catalog(), OptionalLong.empty())); + if (latest == null || latest.snapshot == null || latest.snapshot.generation() != cacheKey.expectedGeneration().orElseThrow()) { + return Optional.empty(); + } + if (elapsedNanos(latest.loadedAtNanos, now) >= expireAfterNanos) { + return Optional.empty(); + } + Entry inherited = new Entry(); + inherited.snapshot = latest.snapshot; + inherited.loadedAtNanos = latest.loadedAtNanos; + return Optional.of(inherited); + } + + private RefreshTask prepareRefresh(CacheKey cacheKey, Entry entry, long now, boolean force) + { + if (entry.refresh != null) { + return null; + } + if (!force && entry.refreshBackoffActive && elapsedNanos(entry.lastRefreshFailureAtNanos, now) < failureBackoffNanos) { + return null; + } + CompletableFuture result = new CompletableFuture<>(); + entry.refresh = result; + return new RefreshTask(cacheKey, entry, result); + } + + private void dispatch(RefreshTask refreshTask) + { + if (refreshTask == null) { + return; + } + try { + executor.execute(() -> load(refreshTask)); + } + catch (RuntimeException e) { + completeFailure(refreshTask, e); + } + } + + private void load(RefreshTask refreshTask) + { + CompletionStage loaded; + try { + loaded = requireNonNull(loader.load( + refreshTask.cacheKey().catalog(), + refreshTask.cacheKey().expectedGeneration()), "snapshot loader returned null"); + } + catch (RuntimeException failure) { + completeFailure(refreshTask, failure); + return; + } + catch (Error failure) { + completeFailure(refreshTask, failure); + throw failure; + } + CompletableFuture upstream = loaded.toCompletableFuture(); + boolean owned; + synchronized (lock) { + Entry entry = entries.get(refreshTask.cacheKey()); + owned = entry == refreshTask.entry() && entry.refresh == refreshTask.result(); + if (owned) { + entry.upstream = upstream; + } + } + if (!owned) { + upstream.cancel(true); + return; + } + try { + loaded.whenComplete((snapshot, failure) -> { + if (failure != null) { + completeFailure(refreshTask, failure); + return; + } + if (snapshot == null) { + completeFailure(refreshTask, new NullPointerException("snapshot loader completed with null")); + return; + } + completeSuccess(refreshTask, snapshot); + }); + } + catch (RuntimeException failure) { + completeFailure(refreshTask, failure); + } + catch (Error failure) { + completeFailure(refreshTask, failure); + throw failure; + } + } + + private void completeSuccess(RefreshTask refreshTask, HogQlSemanticCatalogSnapshot snapshot) + { + Throwable rejection = validateLoadedSnapshot(refreshTask, snapshot); + if (rejection != null) { + completeFailure(refreshTask, rejection); + return; + } + + boolean published = false; + synchronized (lock) { + Entry entry = entries.get(refreshTask.cacheKey()); + if (entry == refreshTask.entry() && entry.refresh == refreshTask.result()) { + HogQlSemanticCatalogSnapshot observed = observedGenerations.get(new GenerationKey(snapshot.catalog(), snapshot.generation())); + if (observed != null && !observed.equals(snapshot)) { + entry.refresh = null; + entry.upstream = null; + entry.lastRefreshFailureAtNanos = ticker.getAsLong(); + entry.refreshBackoffActive = true; + rejection = new HogQlSemanticCatalogException( + Failure.GENERATION_MISMATCH, + "HogQL semantic catalog generation content changed"); + } + else if (refreshTask.cacheKey().expectedGeneration().isEmpty() && entry.snapshot != null && snapshot.generation() < entry.snapshot.generation()) { + entry.refresh = null; + entry.upstream = null; + entry.lastRefreshFailureAtNanos = ticker.getAsLong(); + entry.refreshBackoffActive = true; + rejection = new HogQlSemanticCatalogException( + Failure.GENERATION_MISMATCH, + format("HogQL semantic catalog generation regressed from %s to %s", entry.snapshot.generation(), snapshot.generation())); + } + else { + observedGenerations.put(new GenerationKey(snapshot.catalog(), snapshot.generation()), snapshot); + while (observedGenerations.size() > maximumEntries) { + observedGenerations.remove(observedGenerations.entrySet().iterator().next().getKey()); + } + entry.snapshot = snapshot; + entry.loadedAtNanos = ticker.getAsLong(); + entry.refreshBackoffActive = false; + entry.refresh = null; + entry.upstream = null; + published = true; + } + } + } + if (published) { + refreshTask.result().complete(snapshot); + } + else if (rejection != null) { + refreshTask.result().completeExceptionally(rejection); + } + else { + refreshTask.result().completeExceptionally(new CancellationException("HogQL semantic catalog refresh no longer owns its cache entry")); + } + } + + private Throwable validateLoadedSnapshot(RefreshTask refreshTask, HogQlSemanticCatalogSnapshot snapshot) + { + if (!snapshot.catalog().equals(refreshTask.cacheKey().catalog())) { + return new HogQlSemanticCatalogException( + Failure.CATALOG_MISMATCH, + "HogQL semantic catalog snapshot does not match the refreshed catalog"); + } + if (refreshTask.cacheKey().expectedGeneration().isPresent() && + snapshot.generation() != refreshTask.cacheKey().expectedGeneration().orElseThrow()) { + return new HogQlSemanticCatalogException( + Failure.GENERATION_MISMATCH, + "HogQL semantic catalog snapshot generation does not match the refreshed generation"); + } + return null; + } + + private void completeFailure(RefreshTask refreshTask, Throwable failure) + { + requireNonNull(failure, "failure is null"); + boolean owned; + synchronized (lock) { + Entry entry = entries.get(refreshTask.cacheKey()); + owned = entry == refreshTask.entry() && entry.refresh == refreshTask.result(); + if (owned) { + entry.refresh = null; + entry.upstream = null; + entry.lastRefreshFailureAtNanos = ticker.getAsLong(); + entry.refreshBackoffActive = true; + } + } + if (owned) { + refreshTask.result().completeExceptionally(failure); + } + else { + refreshTask.result().completeExceptionally(new CancellationException("HogQL semantic catalog refresh no longer owns its cache entry")); + } + } + + private void evictEntries(List evictedRefreshes) + { + while (entries.size() > maximumEntries) { + Map.Entry victim = entries.entrySet().stream() + .filter(entry -> entry.getKey().expectedGeneration().isPresent()) + .findFirst() + .orElseGet(() -> entries.entrySet().iterator().next()); + entries.remove(victim.getKey()); + if (victim.getValue().refresh != null) { + evictedRefreshes.add(new RefreshCancellation(victim.getValue().refresh, victim.getValue().upstream)); + victim.getValue().refresh = null; + victim.getValue().upstream = null; + } + } + } + + private static void cancelRefreshes(List refreshes, String message) + { + refreshes.forEach(refresh -> cancelRefresh(refresh, message)); + } + + private static void cancelRefresh(RefreshCancellation refresh, String message) + { + if (refresh.upstream() != null) { + refresh.upstream().cancel(true); + } + refresh.result().completeExceptionally(new CancellationException(message)); + } + + private static long elapsedNanos(long start, long end) + { + return Math.max(0, end - start); + } + + private static long nonNegativeNanos(Duration duration, String name) + { + requireNonNull(duration, name + " is null"); + if (duration.isNegative()) { + throw new IllegalArgumentException(name + " must not be negative"); + } + return duration.toNanos(); + } + + private static long positiveNanos(Duration duration, String name) + { + long nanos = nonNegativeNanos(duration, name); + if (nanos == 0) { + throw new IllegalArgumentException(name + " must be positive"); + } + return nanos; + } + + @FunctionalInterface + public interface SnapshotLoader + { + CompletionStage load(PhysicalIdentifier catalog, OptionalLong expectedGeneration); + } + + private static final class Entry + { + private HogQlSemanticCatalogSnapshot snapshot; + private long loadedAtNanos; + private long lastRefreshFailureAtNanos; + private boolean refreshBackoffActive; + private CompletableFuture refresh; + private CompletableFuture upstream; + } + + private record RefreshTask( + CacheKey cacheKey, + Entry entry, + CompletableFuture result) {} + + private record RefreshCancellation( + CompletableFuture result, + CompletableFuture upstream) {} + + private record CacheKey(PhysicalIdentifier catalog, OptionalLong expectedGeneration) + { + private CacheKey + { + catalog = requireNonNull(catalog, "catalog is null"); + expectedGeneration = requireNonNull(expectedGeneration, "expectedGeneration is null"); + } + } + + private record GenerationKey(PhysicalIdentifier catalog, long generation) + { + private GenerationKey + { + catalog = requireNonNull(catalog, "catalog is null"); + } + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateException.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateException.java new file mode 100644 index 000000000000..5ff60555464c --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateException.java @@ -0,0 +1,58 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.compiler.HogQlErrorCode; +import io.trino.spi.TrinoException; + +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_CATALOG_GENERATION_MISMATCH; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_CATALOG_NOT_READY; +import static java.util.Objects.requireNonNull; + +public final class HogQlExchangeRateException + extends TrinoException +{ + private final Failure failure; + + public HogQlExchangeRateException(Failure failure, String message) + { + super(errorCode(requireNonNull(failure, "failure is null")), message); + this.failure = failure; + } + + public HogQlExchangeRateException(Failure failure, String message, Throwable cause) + { + super(errorCode(requireNonNull(failure, "failure is null")), message, cause); + this.failure = failure; + } + + public Failure failure() + { + return failure; + } + + private static HogQlErrorCode errorCode(Failure failure) + { + return switch (failure) { + case UNAVAILABLE -> HOGQL_CATALOG_NOT_READY; + case GENERATION_MISMATCH -> HOGQL_CATALOG_GENERATION_MISMATCH; + }; + } + + public enum Failure + { + UNAVAILABLE, + GENERATION_MISMATCH, + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshot.java new file mode 100644 index 000000000000..0df0092cd9fe --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshot.java @@ -0,0 +1,119 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import java.time.LocalDate; +import java.time.format.DateTimeParseException; +import java.util.List; +import java.util.regex.Pattern; + +import static java.util.Objects.requireNonNull; + +public record HogQlExchangeRateSnapshot( + int protocolVersion, + int schemaVersion, + long generation, + String baseCurrency, + int decimalScale, + List rates) +{ + public static final int PROTOCOL_VERSION = 1; + public static final int SCHEMA_VERSION = 1; + public static final String BASE_CURRENCY = "USD"; + public static final int DECIMAL_SCALE = 10; + public static final int MAXIMUM_RATES = 1_000_000; + + private static final Pattern CURRENCY_PATTERN = Pattern.compile("[A-Z]{3}"); + private static final Pattern UNSCALED_RATE_PATTERN = Pattern.compile("0|[1-9][0-9]{0,17}"); + private static final String UNSCALED_ONE = "10000000000"; + + public HogQlExchangeRateSnapshot + { + if (protocolVersion != PROTOCOL_VERSION) { + throw new IllegalArgumentException("unsupported HogQL exchange-rate protocol"); + } + if (schemaVersion != SCHEMA_VERSION) { + throw new IllegalArgumentException("unsupported HogQL exchange-rate schema"); + } + if (generation <= 0) { + throw new IllegalArgumentException("HogQL exchange-rate generation must be positive"); + } + baseCurrency = requireNonNull(baseCurrency, "baseCurrency is null"); + if (!baseCurrency.equals(BASE_CURRENCY)) { + throw new IllegalArgumentException("unsupported HogQL exchange-rate base currency"); + } + if (decimalScale != DECIMAL_SCALE) { + throw new IllegalArgumentException("unsupported HogQL exchange-rate decimal scale"); + } + rates = List.copyOf(requireNonNull(rates, "rates is null")); + if (rates.isEmpty()) { + throw new IllegalArgumentException("HogQL exchange-rate snapshot has no rates"); + } + if (rates.size() > MAXIMUM_RATES) { + throw new IllegalArgumentException("HogQL exchange-rate snapshot exceeds rate limit"); + } + + boolean baseCurrencyPresent = false; + ExchangeRate previous = null; + for (ExchangeRate rate : rates) { + requireNonNull(rate, "rate is null"); + if (rate.currency().equals(BASE_CURRENCY)) { + baseCurrencyPresent = true; + if (!rate.unscaledRate().equals(UNSCALED_ONE)) { + throw new IllegalArgumentException("HogQL base-currency rate must equal one"); + } + } + if (previous != null && compare(previous, rate) >= 0) { + throw new IllegalArgumentException("HogQL exchange rates are not strictly sorted"); + } + previous = rate; + } + if (!baseCurrencyPresent) { + throw new IllegalArgumentException("HogQL exchange-rate snapshot has no base-currency rate"); + } + } + + private static int compare(ExchangeRate left, ExchangeRate right) + { + int currencyComparison = left.currency().compareTo(right.currency()); + if (currencyComparison != 0) { + return currencyComparison; + } + return left.effectiveDate().compareTo(right.effectiveDate()); + } + + public record ExchangeRate(String currency, String effectiveDate, String unscaledRate) + { + public ExchangeRate + { + currency = requireNonNull(currency, "currency is null"); + effectiveDate = requireNonNull(effectiveDate, "effectiveDate is null"); + unscaledRate = requireNonNull(unscaledRate, "unscaledRate is null"); + if (!CURRENCY_PATTERN.matcher(currency).matches()) { + throw new IllegalArgumentException("invalid HogQL exchange-rate currency"); + } + try { + if (!LocalDate.parse(effectiveDate).toString().equals(effectiveDate)) { + throw new IllegalArgumentException("noncanonical HogQL exchange-rate date"); + } + } + catch (DateTimeParseException e) { + throw new IllegalArgumentException("invalid HogQL exchange-rate date", e); + } + if (!UNSCALED_RATE_PATTERN.matcher(unscaledRate).matches()) { + throw new IllegalArgumentException("invalid HogQL exchange-rate value"); + } + } + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshotCache.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshotCache.java new file mode 100644 index 000000000000..56643f6997dc --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshotCache.java @@ -0,0 +1,44 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import java.util.Optional; +import java.util.OptionalLong; + +import static java.util.Objects.requireNonNull; + +@FunctionalInterface +public interface HogQlExchangeRateSnapshotCache +{ + Optional currentSnapshot(OptionalLong expectedGeneration); + + default Optional currentSnapshot() + { + return currentSnapshot(OptionalLong.empty()); + } + + default Optional currentSnapshot(long expectedGeneration) + { + return currentSnapshot(OptionalLong.of(expectedGeneration)); + } + + static OptionalLong validateExpectedGeneration(OptionalLong expectedGeneration) + { + requireNonNull(expectedGeneration, "expectedGeneration is null"); + if (expectedGeneration.isPresent() && expectedGeneration.orElseThrow() <= 0) { + throw new IllegalArgumentException("expected exchange-rate generation must be positive"); + } + return expectedGeneration; + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshotJsonDecoder.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshotJsonDecoder.java new file mode 100644 index 000000000000..dcfad08ca4ed --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshotJsonDecoder.java @@ -0,0 +1,199 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.StreamReadConstraints; +import com.fasterxml.jackson.core.StreamReadFeature; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshot.ExchangeRate; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotLoader.LoadRequest; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import static java.util.Objects.requireNonNull; + +public final class HogQlExchangeRateSnapshotJsonDecoder +{ + public static final int MAXIMUM_PAYLOAD_BYTES = 32 * 1024 * 1024; + + private static final Set SNAPSHOT_FIELDS = Set.of( + "protocolVersion", + "schemaVersion", + "generation", + "baseCurrency", + "decimalScale", + "rates"); + private static final Set RATE_FIELDS = Set.of("currency", "effectiveDate", "unscaledRate"); + + private final ObjectMapper objectMapper; + + public HogQlExchangeRateSnapshotJsonDecoder() + { + JsonFactory jsonFactory = JsonFactory.builder() + .streamReadConstraints(StreamReadConstraints.builder() + .maxNestingDepth(16) + .maxStringLength(MAXIMUM_PAYLOAD_BYTES) + .maxNumberLength(64) + .build()) + .enable(StreamReadFeature.STRICT_DUPLICATE_DETECTION) + .build(); + objectMapper = new ObjectMapper(jsonFactory); + } + + public HogQlExchangeRateSnapshot decode(byte[] payload, LoadRequest request) + { + requireNonNull(request, "request is null"); + if (payload == null) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + if (payload.length > MAXIMUM_PAYLOAD_BYTES) { + throw failure(DecodeFailure.LIMIT_EXCEEDED); + } + + try (JsonParser parser = objectMapper.createParser(payload)) { + JsonNode document = objectMapper.readTree(parser); + if (!(document instanceof ObjectNode root) || parser.nextToken() != null) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + requireFields(root, SNAPSHOT_FIELDS); + int protocolVersion = integer(root, "protocolVersion"); + if (protocolVersion != HogQlExchangeRateSnapshot.PROTOCOL_VERSION) { + throw failure(DecodeFailure.UNSUPPORTED_PROTOCOL); + } + int schemaVersion = integer(root, "schemaVersion"); + if (schemaVersion != HogQlExchangeRateSnapshot.SCHEMA_VERSION) { + throw failure(DecodeFailure.UNSUPPORTED_SCHEMA); + } + long generation = longInteger(root, "generation"); + if (request.expectedGeneration().isPresent() && generation != request.expectedGeneration().orElseThrow()) { + throw failure(DecodeFailure.GENERATION_MISMATCH); + } + String baseCurrency = text(root, "baseCurrency"); + int decimalScale = integer(root, "decimalScale"); + ArrayNode rateNodes = array(root, "rates"); + if (rateNodes.size() > HogQlExchangeRateSnapshot.MAXIMUM_RATES) { + throw failure(DecodeFailure.LIMIT_EXCEEDED); + } + List rates = new ArrayList<>(rateNodes.size()); + for (JsonNode node : rateNodes) { + if (!(node instanceof ObjectNode rate)) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + requireFields(rate, RATE_FIELDS); + rates.add(new ExchangeRate( + text(rate, "currency"), + text(rate, "effectiveDate"), + text(rate, "unscaledRate"))); + } + return new HogQlExchangeRateSnapshot(protocolVersion, schemaVersion, generation, baseCurrency, decimalScale, rates); + } + catch (DecodeException e) { + throw e; + } + catch (IOException | RuntimeException e) { + throw new DecodeException(DecodeFailure.INVALID_PAYLOAD, e); + } + } + + private static void requireFields(ObjectNode object, Set expected) + { + Set actual = new HashSet<>(); + object.fieldNames().forEachRemaining(actual::add); + if (!actual.equals(expected)) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + } + + private static int integer(ObjectNode object, String field) + { + JsonNode value = object.get(field); + if (value == null || !value.isIntegralNumber() || !value.canConvertToInt()) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + return value.intValue(); + } + + private static long longInteger(ObjectNode object, String field) + { + JsonNode value = object.get(field); + if (value == null || !value.isIntegralNumber() || !value.canConvertToLong()) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + return value.longValue(); + } + + private static String text(ObjectNode object, String field) + { + JsonNode value = object.get(field); + if (value == null || !value.isTextual()) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + return value.textValue(); + } + + private static ArrayNode array(ObjectNode object, String field) + { + JsonNode value = object.get(field); + if (!(value instanceof ArrayNode array)) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + return array; + } + + private static DecodeException failure(DecodeFailure failure) + { + return new DecodeException(failure); + } + + public enum DecodeFailure + { + INVALID_PAYLOAD, + LIMIT_EXCEEDED, + UNSUPPORTED_PROTOCOL, + UNSUPPORTED_SCHEMA, + GENERATION_MISMATCH, + } + + public static final class DecodeException + extends IllegalArgumentException + { + private final DecodeFailure failure; + + private DecodeException(DecodeFailure failure) + { + super("invalid HogQL exchange-rate snapshot"); + this.failure = requireNonNull(failure, "failure is null"); + } + + private DecodeException(DecodeFailure failure, Throwable cause) + { + super("invalid HogQL exchange-rate snapshot", cause); + this.failure = requireNonNull(failure, "failure is null"); + } + + public DecodeFailure failure() + { + return failure; + } + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshotLoader.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshotLoader.java new file mode 100644 index 000000000000..03a7b0eb1c78 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshotLoader.java @@ -0,0 +1,63 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import java.util.OptionalLong; +import java.util.concurrent.CompletionStage; + +import static java.util.Objects.requireNonNull; + +@FunctionalInterface +public interface HogQlExchangeRateSnapshotLoader +{ + CompletionStage load(LoadRequest request); + + static HogQlExchangeRateSnapshotLoader fromJsonTransport(JsonTransport transport, HogQlExchangeRateSnapshotJsonDecoder decoder) + { + requireNonNull(transport, "transport is null"); + requireNonNull(decoder, "decoder is null"); + return request -> { + requireNonNull(request, "request is null"); + CompletionStage response = requireNonNull(transport.load(request), "transport returned null"); + return response.thenApply(payload -> decoder.decode(payload, request)); + }; + } + + @FunctionalInterface + interface JsonTransport + { + CompletionStage load(LoadRequest request); + } + + record LoadRequest(OptionalLong expectedGeneration) + { + public LoadRequest + { + expectedGeneration = requireNonNull(expectedGeneration, "expectedGeneration is null"); + if (expectedGeneration.isPresent() && expectedGeneration.orElseThrow() <= 0) { + throw new IllegalArgumentException("expected exchange-rate generation must be positive"); + } + } + + public static LoadRequest latest() + { + return new LoadRequest(OptionalLong.empty()); + } + + public static LoadRequest pinned(long generation) + { + return new LoadRequest(OptionalLong.of(generation)); + } + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshotProvider.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshotProvider.java new file mode 100644 index 000000000000..7c089abf0f8d --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlExchangeRateSnapshotProvider.java @@ -0,0 +1,53 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.compiler.catalog.HogQlExchangeRateException.Failure; + +import java.util.OptionalLong; + +import static java.util.Objects.requireNonNull; + +@FunctionalInterface +public interface HogQlExchangeRateSnapshotProvider +{ + PinnedSnapshot pin(OptionalLong expectedGeneration); + + static HogQlExchangeRateSnapshotProvider fromCache(HogQlExchangeRateSnapshotCache cache) + { + requireNonNull(cache, "cache is null"); + return expectedGeneration -> { + HogQlExchangeRateSnapshotCache.validateExpectedGeneration(expectedGeneration); + HogQlExchangeRateSnapshot snapshot = cache.currentSnapshot(expectedGeneration) + .orElseThrow(() -> new HogQlExchangeRateException(Failure.UNAVAILABLE, "HogQL exchange-rate snapshot is unavailable")); + if (expectedGeneration.isPresent() && snapshot.generation() != expectedGeneration.orElseThrow()) { + throw new HogQlExchangeRateException(Failure.GENERATION_MISMATCH, "HogQL exchange-rate snapshot generation does not match the request"); + } + return new PinnedSnapshot(snapshot); + }; + } + + record PinnedSnapshot(HogQlExchangeRateSnapshot snapshot) + { + public PinnedSnapshot + { + snapshot = requireNonNull(snapshot, "snapshot is null"); + } + + public long generation() + { + return snapshot.generation(); + } + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogException.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogException.java new file mode 100644 index 000000000000..905fefe45ab9 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogException.java @@ -0,0 +1,54 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.compiler.HogQlErrorCode; +import io.trino.spi.TrinoException; + +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_CATALOG_GENERATION_MISMATCH; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_CATALOG_NOT_READY; +import static java.util.Objects.requireNonNull; + +public final class HogQlSemanticCatalogException + extends TrinoException +{ + private final Failure failure; + + public HogQlSemanticCatalogException(Failure failure, String message) + { + super(errorCode(requireNonNull(failure, "failure is null")), message); + this.failure = requireNonNull(failure, "failure is null"); + } + + public Failure failure() + { + return failure; + } + + private static HogQlErrorCode errorCode(Failure failure) + { + return switch (failure) { + case UNAVAILABLE -> HOGQL_CATALOG_NOT_READY; + case CATALOG_MISMATCH, LANGUAGE_VERSION_MISMATCH, GENERATION_MISMATCH -> HOGQL_CATALOG_GENERATION_MISMATCH; + }; + } + + public enum Failure + { + UNAVAILABLE, + CATALOG_MISMATCH, + LANGUAGE_VERSION_MISMATCH, + GENERATION_MISMATCH, + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java new file mode 100644 index 000000000000..e42574c2c6e3 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshot.java @@ -0,0 +1,1526 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.trino.hogql.parser.HogQlLanguageVersion; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Base64; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.regex.Pattern; + +import static java.util.Objects.requireNonNull; + +public record HogQlSemanticCatalogSnapshot( + int protocolVersion, + int schemaVersion, + HogQlLanguageVersion languageVersion, + PhysicalIdentifier catalog, + long generation, + List logicalTables, + List expressionFields, + List virtualTables, + List savedQueries, + List materializedViews, + List functions, + List modifierDefaults, + List lazyTables, + List actions, + List cohorts) +{ + public static final int PROTOCOL_VERSION = 1; + public static final int SCHEMA_VERSION = 2; + private static final int MAX_SEMANTIC_DEFINITIONS = 10_000; + private static final int MAX_RECIPE_DEPTH = 64; + private static final int MAX_RECIPE_NODES = 4_096; + private static final int MAX_RELATION_DEPTH = 64; + private static final ObjectMapper JSON_MAPPER = new ObjectMapper(); + + public HogQlSemanticCatalogSnapshot + { + if (protocolVersion != PROTOCOL_VERSION) { + throw new IllegalArgumentException("unsupported HogQL semantic catalog protocol"); + } + if (schemaVersion != SCHEMA_VERSION) { + throw new IllegalArgumentException("unsupported HogQL semantic catalog schema"); + } + if (generation <= 0) { + throw new IllegalArgumentException("HogQL semantic catalog generation must be positive"); + } + languageVersion = requireNonNull(languageVersion, "languageVersion is null"); + catalog = requireNonNull(catalog, "catalog is null"); + logicalTables = copy(logicalTables, "logicalTables"); + expressionFields = copy(expressionFields, "expressionFields"); + virtualTables = copy(virtualTables, "virtualTables"); + savedQueries = copy(savedQueries, "savedQueries"); + materializedViews = copy(materializedViews, "materializedViews"); + functions = copy(functions, "functions"); + modifierDefaults = copy(modifierDefaults, "modifierDefaults"); + lazyTables = copy(lazyTables, "lazyTables"); + actions = copy(actions, "actions"); + cohorts = copy(cohorts, "cohorts"); + int definitions = expressionFields.size() + virtualTables.size() + savedQueries.size() + materializedViews.size() + functions.size() + modifierDefaults.size() + lazyTables.size() + actions.size() + cohorts.size(); + if (definitions > MAX_SEMANTIC_DEFINITIONS) { + throw new IllegalArgumentException("semantic metadata exceeds definition limit"); + } + + Map tables = indexTables(catalog, logicalTables); + validateLogicalReferences(tables); + Map declaredFunctions = validateFunctions(functions); + Map expressions = indexExpressionFields(expressionFields, tables); + RecipeCounter counter = new RecipeCounter(); + validatePropertyRecipes(tables, expressions, declaredFunctions, counter); + validateExpressionRecipes(expressionFields, tables, expressions, declaredFunctions, counter); + validateModifiers(modifierDefaults); + validateRelationshipPredicates(tables, expressions, declaredFunctions, counter); + Map relations = validateRelations(catalog, tables, expressions, virtualTables, savedQueries, materializedViews); + validateLazyTables(lazyTables, tables, expressions, declaredFunctions, counter); + validateSemanticEntities(actions, cohorts, tables, expressions, declaredFunctions, relations, counter); + } + + public HogQlSemanticCatalogSnapshot( + int protocolVersion, + int schemaVersion, + HogQlLanguageVersion languageVersion, + PhysicalIdentifier catalog, + long generation, + List logicalTables, + List expressionFields, + List virtualTables, + List savedQueries, + List materializedViews, + List functions, + List modifierDefaults) + { + this(protocolVersion, schemaVersion, languageVersion, catalog, generation, logicalTables, expressionFields, virtualTables, savedQueries, materializedViews, functions, modifierDefaults, List.of(), List.of(), List.of()); + } + + public HogQlSemanticCatalogSnapshot( + int schemaVersion, + HogQlLanguageVersion languageVersion, + PhysicalIdentifier catalog, + long generation, + List logicalTables) + { + this(PROTOCOL_VERSION, schemaVersion, languageVersion, catalog, generation, logicalTables, List.of(), List.of(), List.of(), List.of(), List.of(), List.of(), List.of(), List.of(), List.of()); + } + + public Optional logicalTable(String name) + { + String canonical = canonical(name, "logical table"); + return logicalTables.stream().filter(table -> canonical(table.name(), "logical table").equals(canonical)).findFirst(); + } + + private static Map indexTables(PhysicalIdentifier catalog, List definitions) + { + Map tables = new HashMap<>(); + for (LogicalTableDefinition table : definitions) { + if (!table.physicalTable().catalog().equals(catalog)) { + throw new IllegalArgumentException("logical table physical reference uses another catalog"); + } + if (tables.put(canonical(table.name(), "logical table"), table) != null) { + throw new IllegalArgumentException("duplicate logical table"); + } + Set members = new HashSet<>(); + table.fields().forEach(field -> addUnique(members, field.name(), "duplicate logical member")); + Set properties = new HashSet<>(); + for (PropertyDefinition property : table.properties()) { + String name = canonical(property.name(), "property name"); + if (!properties.add(name)) { + throw new IllegalArgumentException("duplicate logical member"); + } + if (members.contains(name) && !name.equals(canonical(property.sourceField(), "property source field"))) { + throw new IllegalArgumentException("duplicate logical member"); + } + members.add(name); + } + table.relationships().forEach(relationship -> addUnique(members, relationship.name(), "duplicate logical member")); + } + return Map.copyOf(tables); + } + + private static void validateLogicalReferences(Map tables) + { + for (LogicalTableDefinition table : tables.values()) { + Set fields = fieldNames(table); + for (PropertyDefinition property : table.properties()) { + requireReference(fields, property.sourceField(), "property has unknown source field"); + } + for (RelationshipDefinition relationship : table.relationships()) { + LogicalTableDefinition target = tables.get(canonical(relationship.targetTable(), "relationship target table")); + if (target == null) { + throw new IllegalArgumentException("relationship has unknown target table"); + } + Set targetFields = fieldNames(target); + for (JoinKey joinKey : relationship.joinKeys()) { + requireReference(fields, joinKey.sourceField(), "relationship has unknown source field"); + requireReference(targetFields, joinKey.targetField(), "relationship has unknown target field"); + } + } + } + } + + private static void validatePropertyRecipes( + Map tables, + Map expressions, + Map functions, + RecipeCounter counter) + { + for (LogicalTableDefinition table : tables.values()) { + for (PropertyDefinition property : table.properties()) { + if (property.lookupRecipe().isEmpty()) { + continue; + } + Map arguments = new HashMap<>(); + validateRecipe( + property.lookupRecipe().orElseThrow(), + 1, + counter, + new RecipeValidationContext(table.name(), tables, expressions, functions, null, false, false, arguments, Map.of())); + if (!arguments.containsKey(ExpressionArgument.PROPERTY_SOURCE) || !arguments.containsKey(ExpressionArgument.PROPERTY_KEY)) { + throw new IllegalArgumentException("property lookup recipe must reference source and key arguments"); + } + } + } + } + + private static void validateRelationshipPredicates( + Map tables, + Map expressions, + Map functions, + RecipeCounter counter) + { + for (LogicalTableDefinition source : tables.values()) { + for (RelationshipDefinition relationship : source.relationships()) { + if (relationship.joinPredicate().isEmpty()) { + continue; + } + LogicalTableDefinition target = tables.get(canonical(relationship.targetTable(), "relationship target table")); + validateRecipe( + relationship.joinPredicate().orElseThrow(), + 1, + counter, + new RecipeValidationContext( + source.name(), + tables, + expressions, + functions, + null, + false, + true, + null, + Map.of(RelationshipJoinSide.SOURCE, source.name(), RelationshipJoinSide.TARGET, target.name()))); + } + } + } + + private static Map validateFunctions(List definitions) + { + Map functions = new HashMap<>(); + for (FunctionCapabilityDefinition function : definitions) { + if (functions.put(canonical(function.name(), "function name"), function) != null) { + throw new IllegalArgumentException("duplicate function"); + } + if (function.implementation() == FunctionImplementation.REWRITE && !function.trinoName().isEmpty()) { + throw new IllegalArgumentException("rewrite function cannot name a Trino function"); + } + if (function.implementation() == FunctionImplementation.REWRITE) { + if (function.rewrite().isEmpty()) { + throw new IllegalArgumentException("rewrite function must declare a rewrite"); + } + FunctionRewrite rewrite = function.rewrite().orElseThrow(); + if (rewriteFunctionKind(rewrite) != function.kind()) { + throw new IllegalArgumentException("rewrite function kind must be " + rewriteFunctionKind(rewrite).name().toLowerCase(Locale.ENGLISH)); + } + if (!function.deterministic()) { + throw new IllegalArgumentException("rewrite function must be deterministic"); + } + if (function.supportsDistinct()) { + throw new IllegalArgumentException("rewrite function cannot support DISTINCT"); + } + if (function.supportsOrderBy()) { + throw new IllegalArgumentException("rewrite function cannot support ORDER BY"); + } + if (function.supportsFilter()) { + throw new IllegalArgumentException("rewrite function cannot support FILTER"); + } + if (function.supportsWindow() && rewriteFunctionKind(rewrite) != FunctionKind.AGGREGATE) { + throw new IllegalArgumentException("scalar rewrite function cannot support window invocation"); + } + if (function.signatures().stream().anyMatch(signature -> !validRewriteSignature(rewrite, signature))) { + throw new IllegalArgumentException("rewrite function declares an invalid signature"); + } + if ((rewrite == FunctionRewrite.IS_NULL || rewrite == FunctionRewrite.IS_NOT_NULL) && + function.signatures().stream().anyMatch(signature -> !signature.returnType().equalsIgnoreCase("boolean"))) { + throw new IllegalArgumentException("null predicate rewrite function signatures must return boolean"); + } + } + else { + if (function.rewrite().isPresent()) { + throw new IllegalArgumentException("non-rewrite function cannot declare a rewrite"); + } + if (function.trinoName().isEmpty()) { + throw new IllegalArgumentException("function must name a Trino function"); + } + } + } + return Map.copyOf(functions); + } + + private static FunctionKind rewriteFunctionKind(FunctionRewrite rewrite) + { + return switch (rewrite) { + case ANY_IF, ARG_MAX_IF, ARG_MIN_IF, AVG_IF, COUNT_DISTINCT, COUNT_IF, GROUP_ARRAY_IF, + GROUP_UNIQ_ARRAY, GROUP_UNIQ_ARRAY_IF, MAX_IF, MEDIAN_IF, MIN_IF, QUANTILE, QUANTILE_EXACT, QUANTILE_IF, SUM_IF, + UNIQ_EXACT, UNIQ_EXACT_IF, UNIQ_IF -> FunctionKind.AGGREGATE; + default -> FunctionKind.SCALAR; + }; + } + + private static boolean validRewriteSignature(FunctionRewrite rewrite, FunctionSignature signature) + { + return switch (rewrite) { + case ARRAY_ENUMERATE, ARRAY_SUM, ASSUME_NOT_NULL, CAST_BIGINT, CAST_DATE, CAST_DOUBLE, CAST_SMALLINT, + CAST_UUID, CAST_VARCHAR, EMPTY, FLOAT_OR_ZERO, INTERVAL_MONTH, INT_OR_ZERO, + MD5, NOT_EMPTY, TO_JSON_STRING, + DATE_TRUNC_DAY, DATE_TRUNC_HOUR, DATE_TRUNC_MONTH, DATE_TRUNC_WEEK, + GROUP_UNIQ_ARRAY, INTERVAL_DAY, IS_NOT_NULL, IS_NULL, COUNT_IF, + NOT, PARSE_TIMESTAMP, TO_UNIX_TIMESTAMP, + COUNT_DISTINCT, UNIQ_EXACT -> + !signature.variadic() && signature.argumentTypes().size() == 1; + case START_WEEK -> + !signature.variadic() && (signature.argumentTypes().size() == 1 || signature.argumentTypes().size() == 2); + case ADD_DAYS, ADD_MONTHS, ANY_IF, ARRAY_ELEMENT, ARRAY_FILTER, ARRAY_FIRST, ARRAY_MAP, + AVG_IF, DATE_PART, DECIMAL_CAST, DIVIDE_DECIMAL, EQUALS, FLOAT_OR_DEFAULT, GREATER, GREATER_OR_EQUAL, HAS, + GROUP_UNIQ_ARRAY_IF, JSON_EXTRACT_TYPED, JSON_HAS, JSON_KEYS_AND_VALUES, JSON_VALUE, + IN_ARRAY, INT_DIV, LESS_OR_EQUAL, LIKE, MAX_IF, MIN_IF, MULTIPLY, MULTIPLY_DECIMAL, + MEDIAN_IF, MINUS, NOT_EQUALS, PLUS, QUANTILE, QUANTILE_EXACT, REGEX_EXTRACT, REGEX_EXTRACT_ALL, SPLIT_CHAR, SPLIT_STRING, SURVEY_RESPONSE, + SUBTRACT_DAYS, SUBTRACT_MONTHS, SUBTRACT_YEARS, SUM_IF, + TUPLE_ELEMENT, UNIQ_EXACT_IF, UNIQ_IF -> + !signature.variadic() && signature.argumentTypes().size() == 2; + case ARG_MAX_IF, ARG_MIN_IF, ARRAY_SLICE, QUANTILE_IF, REGEX_REPLACE_ALL, REGEX_REPLACE_ONE -> + !signature.variadic() && signature.argumentTypes().size() == 3; + case CONVERT_CURRENCY -> !signature.variadic() && + (signature.argumentTypes().size() == 3 || signature.argumentTypes().size() == 4); + case GROUP_ARRAY_IF -> !signature.variadic() && + (signature.argumentTypes().size() == 2 || signature.argumentTypes().size() == 3); + case CAST_TIMESTAMP -> !signature.variadic() && + (signature.argumentTypes().size() == 1 || signature.argumentTypes().size() == 2); + case DATE_ADD -> !signature.variadic() && + (signature.argumentTypes().size() == 2 || signature.argumentTypes().size() == 3); + case JSON_EXTRACT_BOOL, JSON_EXTRACT_FLOAT, JSON_EXTRACT_INT, JSON_EXTRACT_RAW, JSON_EXTRACT_UINT -> + signature.variadic() && signature.argumentTypes().size() == 3; + case JSON_EXTRACT_ARRAY_RAW, JSON_EXTRACT_KEYS, JSON_EXTRACT_STRING, JSON_KEYS_AND_VALUES_RAW, JSON_LENGTH -> + (!signature.variadic() && signature.argumentTypes().size() == 1) || + (signature.variadic() && signature.argumentTypes().size() == 3); + case AND, OR -> (!signature.variadic() && signature.argumentTypes().size() == 2) || + (signature.variadic() && signature.argumentTypes().size() == 3); + case ARRAY_SORT, RANGE -> !signature.variadic() && + (signature.argumentTypes().size() == 1 || signature.argumentTypes().size() == 2); + case MAP_CONSTRUCTOR -> (!signature.variadic() && signature.argumentTypes().isEmpty()) || + (signature.variadic() && signature.argumentTypes().size() == 3); + case MULTI_IF -> signature.variadic() && signature.argumentTypes().size() == 4; + case TUPLE -> signature.variadic() && signature.argumentTypes().size() == 2; + case TODAY -> !signature.variadic() && signature.argumentTypes().isEmpty(); + }; + } + + private static Map indexExpressionFields(List definitions, Map tables) + { + Map fields = new HashMap<>(); + for (ExpressionFieldDefinition field : definitions) { + LogicalTableDefinition table = tables.get(canonical(field.table(), "expression field table")); + if (table == null) { + throw new IllegalArgumentException("expression field references an unknown table"); + } + Set members = new HashSet<>(fieldNames(table)); + table.properties().forEach(property -> members.add(canonical(property.name(), "property name"))); + table.relationships().forEach(relationship -> members.add(canonical(relationship.name(), "relationship name"))); + if (members.contains(canonical(field.name(), "expression field name"))) { + throw new IllegalArgumentException("expression field conflicts with an existing logical member"); + } + if (fields.put(expressionKey(field.table(), field.name()), field) != null) { + throw new IllegalArgumentException("duplicate expression field"); + } + } + return Map.copyOf(fields); + } + + private static void validateExpressionRecipes( + List definitions, + Map tables, + Map expressions, + Map functions, + RecipeCounter counter) + { + Map> dependencies = new HashMap<>(); + for (ExpressionFieldDefinition definition : definitions) { + List fieldDependencies = new ArrayList<>(); + validateRecipe( + definition.recipe(), + 1, + counter, + new RecipeValidationContext(definition.table(), tables, expressions, functions, fieldDependencies, true, true, null, Map.of())); + dependencies.put(expressionKey(definition.table(), definition.name()), List.copyOf(fieldDependencies)); + } + Map states = new HashMap<>(); + dependencies.keySet().forEach(name -> visitDependency(name, 1, dependencies, states)); + } + + private static void validateRecipe( + ExpressionRecipe recipe, + int depth, + RecipeCounter counter, + RecipeValidationContext context) + { + requireNonNull(recipe, "expression recipe is null"); + if (depth > MAX_RECIPE_DEPTH) { + throw new IllegalArgumentException("expression recipe exceeds depth limit"); + } + counter.add(); + switch (recipe) { + case FieldReferenceRecipe reference -> { + if (!context.allowFieldReferences()) { + throw new IllegalArgumentException("field reference is not valid in this recipe"); + } + if (!canonical(reference.table(), "field reference table").equals(canonical(context.ownerTable(), "expression field table"))) { + throw new IllegalArgumentException("field reference crosses tables without a relationship path"); + } + LogicalTableDefinition table = context.tables().get(canonical(reference.table(), "field reference table")); + if (table == null) { + throw new IllegalArgumentException("field reference has unknown table"); + } + String key = expressionKey(reference.table(), reference.field()); + if (context.expressions().containsKey(key)) { + if (context.dependencies() != null) { + context.dependencies().add(key); + } + } + else { + requireReference(fieldNames(table), reference.field(), "field reference has unknown field"); + } + } + case LiteralRecipe _ -> {} + case FunctionCallRecipe call -> { + FunctionCapabilityDefinition function = context.functions().get(canonical(call.name(), "function name")); + if (function == null) { + throw new IllegalArgumentException("function call references an undeclared function"); + } + if (!acceptsArity(function, call.arguments().size())) { + throw new IllegalArgumentException("function call has an unsupported argument count"); + } + call.arguments().forEach(argument -> validateRecipe(argument, depth + 1, counter, context)); + } + case OperatorRecipe operator -> operator.arguments().forEach(argument -> validateRecipe(argument, depth + 1, counter, context)); + case CastRecipe cast -> validateRecipe(cast.expression(), depth + 1, counter, context); + case ArgumentReferenceRecipe reference -> { + if (context.arguments() == null) { + throw new IllegalArgumentException("argument reference is not valid in this recipe"); + } + context.arguments().merge(reference.argument(), 1, Integer::sum); + } + case ScopedFieldReferenceRecipe reference -> { + String tableName = context.scopedTables().get(reference.side()); + if (tableName == null) { + throw new IllegalArgumentException("scoped field reference is not valid in this recipe"); + } + LogicalTableDefinition table = context.tables().get(canonical(tableName, "scoped field table")); + requireReference(semanticFieldNames(table, context.expressions()), reference.field(), "scoped field reference has unknown field"); + } + case PropertyLookupRecipe lookup -> { + if (!context.allowPropertyLookups() || !contextAllowsTable(context, lookup.table())) { + throw new IllegalArgumentException("property lookup is not valid in this recipe"); + } + LogicalTableDefinition table = context.tables().get(canonical(lookup.table(), "property lookup table")); + if (table == null || table.properties().stream().noneMatch(property -> canonical(property.name(), "property name").equals(canonical(lookup.property(), "property lookup name")))) { + throw new IllegalArgumentException("property lookup references an unknown property"); + } + validateRecipe(lookup.key(), depth + 1, counter, context); + } + } + } + + private static boolean contextAllowsTable(RecipeValidationContext context, String table) + { + String canonicalTable = canonical(table, "recipe table"); + if (canonicalTable.equals(canonical(context.ownerTable(), "recipe owner table"))) { + return true; + } + return context.scopedTables().values().stream() + .map(value -> canonical(value, "scoped recipe table")) + .anyMatch(canonicalTable::equals); + } + + private static boolean acceptsArity(FunctionCapabilityDefinition function, int arity) + { + return function.signatures().stream().anyMatch(signature -> signature.variadic() + ? arity >= signature.argumentTypes().size() - 1 + : arity == signature.argumentTypes().size()); + } + + private static void visitDependency(String name, int depth, Map> dependencies, Map states) + { + if (depth > MAX_RECIPE_DEPTH) { + throw new IllegalArgumentException("expression field dependency exceeds depth limit"); + } + if (states.get(name) == VisitState.VISITING) { + throw new IllegalArgumentException("expression field dependency cycle"); + } + if (states.putIfAbsent(name, VisitState.VISITING) == VisitState.VISITED) { + return; + } + dependencies.getOrDefault(name, List.of()).forEach(dependency -> visitDependency(dependency, depth + 1, dependencies, states)); + states.put(name, VisitState.VISITED); + } + + private static void validateModifiers(List modifiers) + { + Set names = new HashSet<>(); + for (SemanticModifierDefault modifier : modifiers) { + addUnique(names, modifier.name(), "duplicate modifier"); + if (modifier.behavior() == ModifierBehavior.TRINO_SESSION_PROPERTY && modifier.sessionProperty().isEmpty()) { + throw new IllegalArgumentException("modifier must name a session property"); + } + if (modifier.behavior() == ModifierBehavior.TRINO_SESSION_PROPERTY && modifier.sessionProperty().size() > 2) { + throw new IllegalArgumentException("modifier has an invalid session property name"); + } + if (modifier.behavior() != ModifierBehavior.TRINO_SESSION_PROPERTY && !modifier.sessionProperty().isEmpty()) { + throw new IllegalArgumentException("modifier cannot name a session property"); + } + } + } + + private static Map validateRelations( + PhysicalIdentifier catalog, + Map tables, + Map expressions, + List virtualTables, + List savedQueries, + List materializedViews) + { + Map relations = new HashMap<>(); + tables.forEach((name, table) -> { + Set fields = new HashSet<>(fieldNames(table)); + expressions.values().stream() + .filter(expression -> canonical(expression.table(), "expression field table").equals(name)) + .forEach(expression -> fields.add(canonical(expression.name(), "expression field name"))); + relations.put(name, new SemanticRelation(RelationKind.LOGICAL_TABLE, fields, null, null)); + }); + savedQueries.forEach(saved -> addRelation(relations, saved.name(), new SemanticRelation(RelationKind.SAVED_QUERY, referencedFields(saved.fields()), null, saved))); + materializedViews.forEach(view -> { + if (!view.physicalView().catalog().equals(catalog)) { + throw new IllegalArgumentException("materialized view physical reference uses another catalog"); + } + addRelation(relations, view.name(), new SemanticRelation(RelationKind.MATERIALIZED_VIEW, referencedFields(view.fields()), null, null)); + }); + virtualTables.forEach(virtual -> addRelation(relations, virtual.name(), new SemanticRelation(RelationKind.VIRTUAL_TABLE, null, virtual, null))); + Map states = new HashMap<>(); + virtualTables.forEach(virtual -> resolveRelation(canonical(virtual.name(), "relation name"), relations, states, 1)); + savedQueries.forEach(saved -> resolveRelation(canonical(saved.name(), "relation name"), relations, states, 1)); + return Map.copyOf(relations); + } + + private static void validateLazyTables( + List definitions, + Map tables, + Map expressions, + Map functions, + RecipeCounter counter) + { + Map> members = new HashMap<>(); + tables.forEach((name, table) -> members.put(name, semanticMemberNames(table, expressions))); + for (LazyTableDefinition definition : definitions) { + LogicalTableDefinition owner = tables.get(canonical(definition.table(), "lazy table owner")); + if (owner == null) { + throw new IllegalArgumentException("lazy table references an unknown owner table"); + } + if (!members.get(canonical(owner.name(), "lazy table owner")).add(canonical(definition.name(), "lazy table name"))) { + throw new IllegalArgumentException("lazy table conflicts with an existing logical member"); + } + if (definition.relationshipPath().isEmpty()) { + throw new IllegalArgumentException("lazy table must include a relationship path"); + } + if (definition.relationshipPath().size() > MAX_RELATION_DEPTH) { + throw new IllegalArgumentException("lazy table relationship path exceeds depth limit"); + } + LogicalTableDefinition terminal = owner; + for (String relationshipName : definition.relationshipPath()) { + LogicalTableDefinition pathSource = terminal; + RelationshipDefinition relationship = pathSource.relationships().stream() + .filter(candidate -> canonical(candidate.name(), "relationship name").equals(canonical(relationshipName, "relationship path"))) + .findFirst() + .orElseThrow(() -> new IllegalArgumentException("lazy table references an unknown relationship")); + terminal = tables.get(canonical(relationship.targetTable(), "relationship target table")); + } + if (definition.projections().isEmpty()) { + throw new IllegalArgumentException("lazy table must include projections"); + } + Set projectionNames = new HashSet<>(); + for (LazyProjectionDefinition projection : definition.projections()) { + addUnique(projectionNames, projection.name(), "duplicate projection on lazy table"); + validateRecipe( + projection.recipe(), + 1, + counter, + new RecipeValidationContext(terminal.name(), tables, expressions, functions, null, true, true, null, Map.of())); + } + } + } + + private static void validateSemanticEntities( + List actions, + List cohorts, + Map tables, + Map expressions, + Map functions, + Map relations, + RecipeCounter counter) + { + Set actionNames = new HashSet<>(); + for (ActionReference action : actions) { + addUnique(actionNames, action.name(), "duplicate action"); + validateSemanticEntity(action.name(), action.table(), action.representation(), "action", tables, expressions, functions, relations, counter); + } + Set cohortNames = new HashSet<>(); + for (CohortReference cohort : cohorts) { + addUnique(cohortNames, cohort.name(), "duplicate cohort"); + validateSemanticEntity(cohort.name(), cohort.table(), cohort.representation(), "cohort", tables, expressions, functions, relations, counter); + } + } + + private static void validateSemanticEntity( + String name, + String tableName, + SemanticEntityRepresentation representation, + String kind, + Map tables, + Map expressions, + Map functions, + Map relations, + RecipeCounter counter) + { + LogicalTableDefinition owner = tables.get(canonical(tableName, kind + " table")); + if (owner == null) { + throw new IllegalArgumentException(kind + " references an unknown table"); + } + switch (representation) { + case PredicateRepresentation predicate -> validateRecipe( + predicate.predicate(), + 1, + counter, + new RecipeValidationContext(owner.name(), tables, expressions, functions, null, true, true, null, Map.of())); + case RelationMembershipRepresentation membershipRepresentation -> { + RelationMembershipRecipe membership = membershipRepresentation.relation(); + SemanticRelation relation = relations.get(canonical(membership.relation().name(), kind + " relation")); + if (relation == null || relation.kind != membership.relation().kind()) { + throw new IllegalArgumentException(kind + " references an unknown or mismatched relation"); + } + requireReference(semanticFieldNames(owner, expressions), membership.sourceField(), kind + " references an unknown source field"); + requireReference(requireNonNull(relation.fields, "resolved relation fields are null"), membership.targetField(), kind + " references an unknown target field"); + } + } + } + + private static void addRelation(Map relations, String name, SemanticRelation relation) + { + if (relations.put(canonical(name, "relation name"), relation) != null) { + throw new IllegalArgumentException("duplicate relation"); + } + } + + private static Set resolveRelation(String name, Map relations, Map states, int depth) + { + if (depth > MAX_RELATION_DEPTH) { + throw new IllegalArgumentException("virtual table reference exceeds depth limit"); + } + SemanticRelation relation = relations.get(name); + if (relation == null) { + throw new IllegalArgumentException("invalid semantic relation reference"); + } + if (relation.kind != RelationKind.VIRTUAL_TABLE && relation.kind != RelationKind.SAVED_QUERY) { + return relation.fields; + } + if (states.get(name) == VisitState.VISITING) { + throw new IllegalArgumentException("semantic relation reference cycle"); + } + if (states.get(name) == VisitState.VISITED) { + return requireNonNull(relation.fields, "resolved relation fields are null"); + } + states.put(name, VisitState.VISITING); + RelationReference reference = relation.kind == RelationKind.VIRTUAL_TABLE ? relation.virtualTable.source() : relation.savedQuery.target(); + if (relation.kind == RelationKind.SAVED_QUERY && reference.kind() == RelationKind.SAVED_QUERY) { + throw new IllegalArgumentException("saved query target must be logical, virtual, or materialized"); + } + SemanticRelation source = relations.get(canonical(reference.name(), "semantic relation source")); + if (source == null || source.kind != reference.kind()) { + throw new IllegalArgumentException("semantic relation references an unknown or mismatched source"); + } + Set sourceFields = source.kind == RelationKind.VIRTUAL_TABLE || source.kind == RelationKind.SAVED_QUERY + ? resolveRelation(canonical(reference.name(), "semantic relation source"), relations, states, depth + 1) + : requireNonNull(source.fields, "source relation fields are null"); + if (relation.kind == RelationKind.SAVED_QUERY) { + if (!sourceFields.containsAll(relation.fields)) { + throw new IllegalArgumentException("saved query declares a field missing from its target"); + } + } + else { + Set fields = new HashSet<>(); + for (VirtualProjection projection : relation.virtualTable.projections()) { + requireReference(sourceFields, projection.sourceField(), "virtual table projection references an unknown source field"); + addUnique(fields, projection.name(), "duplicate projection on virtual table"); + } + relation.fields = Set.copyOf(fields); + } + states.put(name, VisitState.VISITED); + return requireNonNull(relation.fields, "resolved relation fields are null"); + } + + private static Set referencedFields(List fields) + { + Set names = new HashSet<>(); + fields.forEach(field -> addUnique(names, field.name(), "duplicate referenced field")); + return names; + } + + private static Set fieldNames(LogicalTableDefinition table) + { + Set fields = new HashSet<>(); + table.fields().forEach(field -> fields.add(canonical(field.name(), "logical field"))); + return fields; + } + + private static Set semanticFieldNames(LogicalTableDefinition table, Map expressions) + { + Set fields = fieldNames(requireNonNull(table, "logical table is null")); + expressions.values().stream() + .filter(expression -> canonical(expression.table(), "expression table").equals(canonical(table.name(), "logical table"))) + .forEach(expression -> fields.add(canonical(expression.name(), "expression field"))); + return fields; + } + + private static Set semanticMemberNames(LogicalTableDefinition table, Map expressions) + { + Set members = semanticFieldNames(table, expressions); + table.properties().forEach(property -> members.add(canonical(property.name(), "property name"))); + table.relationships().forEach(relationship -> members.add(canonical(relationship.name(), "relationship name"))); + return members; + } + + private static void requireReference(Set names, String name, String message) + { + if (!names.contains(canonical(name, "reference"))) { + throw new IllegalArgumentException(message); + } + } + + private static void addUnique(Set names, String name, String message) + { + if (!names.add(canonical(name, "definition name"))) { + throw new IllegalArgumentException(message); + } + } + + private static String expressionKey(String table, String field) + { + return canonical(table, "expression field table") + "." + canonical(field, "expression field name"); + } + + private static String canonical(String value, String kind) + { + return definition(value, kind).toLowerCase(Locale.ENGLISH); + } + + private static String definition(String value, String kind) + { + requireNonNull(value, kind + " is null"); + if (value.isBlank() || value.indexOf(';') >= 0 || value.indexOf('\0') >= 0 || value.indexOf('\n') >= 0 || value.indexOf('\r') >= 0 || value.contains("--") || value.contains("/*") || value.contains("*/")) { + throw new IllegalArgumentException("invalid " + kind); + } + return value; + } + + private static List copy(List values, String name) + { + return List.copyOf(requireNonNull(values, name + " is null")); + } + + public record LogicalTableDefinition(String name, PhysicalQualifiedName physicalTable, List fields, List properties, List relationships) + { + public LogicalTableDefinition + { + name = definition(name, "logical table name"); + physicalTable = requireNonNull(physicalTable, "physicalTable is null"); + fields = copy(fields, "fields"); + properties = copy(properties, "properties"); + relationships = copy(relationships, "relationships"); + } + } + + public record LogicalFieldDefinition(String name, PhysicalIdentifier physicalColumn, String trinoTypeSignature, LogicalType logicalType, boolean nullable, boolean starVisible) + { + public LogicalFieldDefinition + { + name = definition(name, "logical field name"); + physicalColumn = requireNonNull(physicalColumn, "physicalColumn is null"); + trinoTypeSignature = definition(trinoTypeSignature, "Trino type signature"); + logicalType = requireNonNull(logicalType, "logicalType is null"); + } + } + + public record PropertyDefinition( + String name, + String sourceField, + PropertyStorage storage, + LogicalType logicalType, + boolean nullable, + Optional keyTypeSignature, + Optional valueTypeSignature, + Optional lookupRecipe) + { + public PropertyDefinition + { + name = definition(name, "property name"); + sourceField = definition(sourceField, "property source field"); + storage = requireNonNull(storage, "storage is null"); + logicalType = requireNonNull(logicalType, "logicalType is null"); + keyTypeSignature = requireNonNull(keyTypeSignature, "keyTypeSignature is null").map(value -> definition(value, "property key type signature")); + valueTypeSignature = requireNonNull(valueTypeSignature, "valueTypeSignature is null").map(value -> definition(value, "property value type signature")); + lookupRecipe = requireNonNull(lookupRecipe, "lookupRecipe is null"); + if (lookupRecipe.isPresent() != keyTypeSignature.isPresent() || lookupRecipe.isPresent() != valueTypeSignature.isPresent()) { + throw new IllegalArgumentException("property lookup recipe and type signatures must be declared together"); + } + } + + public PropertyDefinition(String name, String sourceField, PropertyStorage storage, LogicalType logicalType, boolean nullable) + { + this(name, sourceField, storage, logicalType, nullable, Optional.empty(), Optional.empty(), Optional.empty()); + } + } + + public record RelationshipDefinition(String name, String targetTable, RelationshipCardinality cardinality, List joinKeys, Optional joinPredicate) + { + public RelationshipDefinition + { + name = definition(name, "relationship name"); + targetTable = definition(targetTable, "relationship target table"); + cardinality = requireNonNull(cardinality, "cardinality is null"); + joinKeys = copy(joinKeys, "joinKeys"); + if (joinKeys.isEmpty()) { + throw new IllegalArgumentException("relationship must have at least one join key"); + } + joinPredicate = requireNonNull(joinPredicate, "joinPredicate is null"); + } + + public RelationshipDefinition(String name, String targetTable, RelationshipCardinality cardinality, List joinKeys) + { + this(name, targetTable, cardinality, joinKeys, Optional.empty()); + } + } + + public record JoinKey(String sourceField, String targetField) + { + public JoinKey + { + sourceField = definition(sourceField, "relationship source field"); + targetField = definition(targetField, "relationship target field"); + } + } + + public record ExpressionFieldDefinition(String table, String name, String trinoTypeSignature, LogicalType logicalType, boolean nullable, boolean starVisible, ExpressionRecipe recipe) + { + public ExpressionFieldDefinition + { + table = definition(table, "expression field table"); + name = definition(name, "expression field name"); + trinoTypeSignature = definition(trinoTypeSignature, "expression field type signature"); + logicalType = requireNonNull(logicalType, "logicalType is null"); + recipe = requireNonNull(recipe, "recipe is null"); + } + } + + public sealed interface ExpressionRecipe + permits ArgumentReferenceRecipe, + CastRecipe, + FieldReferenceRecipe, + FunctionCallRecipe, + LiteralRecipe, + OperatorRecipe, + PropertyLookupRecipe, + ScopedFieldReferenceRecipe + { + ExpressionRecipeKind kind(); + } + + public record FieldReferenceRecipe(String table, String field) + implements ExpressionRecipe + { + public FieldReferenceRecipe + { + table = definition(table, "field reference table"); + field = definition(field, "field reference field"); + } + + @Override + public ExpressionRecipeKind kind() + { + return ExpressionRecipeKind.FIELD_REFERENCE; + } + } + + public record LiteralRecipe(TypedLiteral literal) + implements ExpressionRecipe + { + public LiteralRecipe + { + literal = requireNonNull(literal, "literal is null"); + } + + @Override + public ExpressionRecipeKind kind() + { + return ExpressionRecipeKind.LITERAL; + } + } + + public record FunctionCallRecipe(String name, List arguments) + implements ExpressionRecipe + { + public FunctionCallRecipe + { + name = definition(name, "function name"); + arguments = copy(arguments, "arguments"); + } + + @Override + public ExpressionRecipeKind kind() + { + return ExpressionRecipeKind.FUNCTION_CALL; + } + } + + public record OperatorRecipe(SemanticOperator operator, List arguments) + implements ExpressionRecipe + { + public OperatorRecipe + { + operator = requireNonNull(operator, "operator is null"); + arguments = copy(arguments, "arguments"); + if (arguments.isEmpty()) { + throw new IllegalArgumentException("operator recipe requires arguments"); + } + int expectedArguments = switch (operator) { + case NOT, NEGATE, IS_NULL, IS_NOT_NULL -> 1; + default -> 2; + }; + if (arguments.size() != expectedArguments) { + throw new IllegalArgumentException("invalid operator arity"); + } + } + + @Override + public ExpressionRecipeKind kind() + { + return ExpressionRecipeKind.OPERATOR; + } + } + + public record CastRecipe(ExpressionRecipe expression, String targetTypeSignature) + implements ExpressionRecipe + { + public CastRecipe + { + expression = requireNonNull(expression, "expression is null"); + targetTypeSignature = definition(targetTypeSignature, "cast target type signature"); + } + + @Override + public ExpressionRecipeKind kind() + { + return ExpressionRecipeKind.CAST; + } + } + + public record ArgumentReferenceRecipe(ExpressionArgument argument) + implements ExpressionRecipe + { + public ArgumentReferenceRecipe + { + argument = requireNonNull(argument, "argument is null"); + } + + @Override + public ExpressionRecipeKind kind() + { + return ExpressionRecipeKind.ARGUMENT_REFERENCE; + } + } + + public record ScopedFieldReferenceRecipe(RelationshipJoinSide side, String field) + implements ExpressionRecipe + { + public ScopedFieldReferenceRecipe + { + side = requireNonNull(side, "side is null"); + field = definition(field, "scoped field reference"); + } + + @Override + public ExpressionRecipeKind kind() + { + return ExpressionRecipeKind.SCOPED_FIELD_REFERENCE; + } + } + + public record PropertyLookupRecipe(String table, String property, ExpressionRecipe key) + implements ExpressionRecipe + { + public PropertyLookupRecipe + { + table = definition(table, "property lookup table"); + property = definition(property, "property lookup name"); + key = requireNonNull(key, "property lookup key is null"); + } + + @Override + public ExpressionRecipeKind kind() + { + return ExpressionRecipeKind.PROPERTY_LOOKUP; + } + } + + public record TypedLiteral(String typeSignature, LiteralEncoding encoding, String value) + { + private static final Pattern DECIMAL = Pattern.compile("[+-]?(0|[1-9][0-9]*)(\\.[0-9]+)?"); + + public TypedLiteral + { + typeSignature = definition(typeSignature, "literal type signature"); + encoding = requireNonNull(encoding, "encoding is null"); + value = requireNonNull(value, "value is null"); + if (value.indexOf('\0') >= 0) { + throw new IllegalArgumentException("literal value contains NUL"); + } + try { + switch (encoding) { + case NULL -> { + if (!value.isEmpty()) { + throw new IllegalArgumentException(); + } + } + case STRING -> {} + case BOOLEAN -> { + if (!value.equals("true") && !value.equals("false")) { + throw new IllegalArgumentException(); + } + } + case INTEGER -> Long.parseLong(value); + case DECIMAL -> { + if (!DECIMAL.matcher(value).matches()) { + throw new IllegalArgumentException(); + } + } + case FLOAT -> { + if (!Double.isFinite(Double.parseDouble(value))) { + throw new IllegalArgumentException(); + } + } + case JSON -> validateJson(value); + case BASE64 -> Base64.getDecoder().decode(value); + } + } + catch (IOException | IllegalArgumentException e) { + throw new IllegalArgumentException("invalid typed literal"); + } + } + + private static void validateJson(String value) + throws IOException + { + try (JsonParser parser = JSON_MAPPER.createParser(value)) { + if (JSON_MAPPER.readTree(parser) == null || parser.nextToken() != null) { + throw new IllegalArgumentException(); + } + } + } + } + + public record VirtualTableDefinition(String name, RelationReference source, List projections) + { + public VirtualTableDefinition + { + name = definition(name, "virtual table name"); + source = requireNonNull(source, "source is null"); + projections = copy(projections, "projections"); + } + } + + public record RelationReference(RelationKind kind, String name) + { + public RelationReference + { + kind = requireNonNull(kind, "kind is null"); + name = definition(name, "semantic relation source"); + } + } + + public record VirtualProjection(String name, String sourceField, boolean starVisible) + { + public VirtualProjection + { + name = definition(name, "virtual projection name"); + sourceField = definition(sourceField, "virtual projection source field"); + } + } + + public record SavedQueryReference(String name, String queryId, RelationReference target, List fields) + { + public SavedQueryReference + { + name = definition(name, "saved query name"); + queryId = definition(queryId, "saved query ID"); + target = requireNonNull(target, "target is null"); + fields = copy(fields, "fields"); + } + } + + public record MaterializedViewReference(String name, PhysicalQualifiedName physicalView, List fields) + { + public MaterializedViewReference + { + name = definition(name, "materialized view name"); + physicalView = requireNonNull(physicalView, "physicalView is null"); + fields = copy(fields, "fields"); + } + } + + public record ReferencedField(String name, String trinoTypeSignature, LogicalType logicalType, boolean nullable, boolean starVisible) + { + public ReferencedField + { + name = definition(name, "referenced field name"); + trinoTypeSignature = definition(trinoTypeSignature, "referenced field type signature"); + logicalType = requireNonNull(logicalType, "logicalType is null"); + } + } + + public record FunctionCapabilityDefinition(String name, FunctionKind kind, FunctionImplementation implementation, List trinoName, Optional rewrite, List signatures, boolean deterministic, boolean supportsDistinct, boolean supportsOrderBy, boolean supportsFilter, boolean supportsWindow) + { + public FunctionCapabilityDefinition + { + name = definition(name, "function name"); + kind = requireNonNull(kind, "kind is null"); + implementation = requireNonNull(implementation, "implementation is null"); + trinoName = copy(trinoName, "trinoName"); + rewrite = requireNonNull(rewrite, "rewrite is null"); + signatures = copy(signatures, "signatures"); + if (signatures.isEmpty()) { + throw new IllegalArgumentException("function must include signatures"); + } + } + + public FunctionCapabilityDefinition( + String name, + FunctionKind kind, + FunctionImplementation implementation, + List trinoName, + List signatures, + boolean deterministic, + boolean supportsDistinct, + boolean supportsOrderBy, + boolean supportsFilter, + boolean supportsWindow) + { + this(name, kind, implementation, trinoName, Optional.empty(), signatures, deterministic, supportsDistinct, supportsOrderBy, supportsFilter, supportsWindow); + } + } + + public record FunctionSignature(List argumentTypes, String returnType, boolean variadic) + { + public FunctionSignature + { + argumentTypes = copy(argumentTypes, "argumentTypes").stream().map(value -> definition(value, "function argument type")).toList(); + returnType = definition(returnType, "function return type"); + if (variadic && argumentTypes.isEmpty()) { + throw new IllegalArgumentException("variadic function signature must declare an argument"); + } + } + } + + public record SemanticModifierDefault(String name, ModifierBehavior behavior, TypedLiteral defaultValue, List sessionProperty) + { + public SemanticModifierDefault + { + name = definition(name, "modifier name"); + behavior = requireNonNull(behavior, "behavior is null"); + defaultValue = requireNonNull(defaultValue, "defaultValue is null"); + sessionProperty = copy(sessionProperty, "sessionProperty"); + } + } + + public record LazyTableDefinition(String table, String name, List relationshipPath, List projections) + { + public LazyTableDefinition + { + table = definition(table, "lazy table owner"); + name = definition(name, "lazy table name"); + relationshipPath = copy(relationshipPath, "relationshipPath").stream() + .map(value -> definition(value, "lazy table relationship path")) + .toList(); + projections = copy(projections, "projections"); + } + } + + public record LazyProjectionDefinition(String name, String trinoTypeSignature, LogicalType logicalType, boolean nullable, boolean starVisible, ExpressionRecipe recipe) + { + public LazyProjectionDefinition + { + name = definition(name, "lazy projection name"); + trinoTypeSignature = definition(trinoTypeSignature, "lazy projection type signature"); + logicalType = requireNonNull(logicalType, "logicalType is null"); + recipe = requireNonNull(recipe, "recipe is null"); + } + } + + public record ActionReference(String name, String actionId, String table, SemanticEntityRepresentation representation) + { + public ActionReference + { + name = definition(name, "action name"); + actionId = definition(actionId, "action ID"); + table = definition(table, "action table"); + representation = requireNonNull(representation, "representation is null"); + } + } + + public record CohortReference(String name, String cohortId, String table, SemanticEntityRepresentation representation) + { + public CohortReference + { + name = definition(name, "cohort name"); + cohortId = definition(cohortId, "cohort ID"); + table = definition(table, "cohort table"); + representation = requireNonNull(representation, "representation is null"); + } + } + + public sealed interface SemanticEntityRepresentation + permits PredicateRepresentation, RelationMembershipRepresentation + { + SemanticEntityKind kind(); + } + + public record PredicateRepresentation(ExpressionRecipe predicate) + implements SemanticEntityRepresentation + { + public PredicateRepresentation + { + predicate = requireNonNull(predicate, "predicate is null"); + } + + @Override + public SemanticEntityKind kind() + { + return SemanticEntityKind.PREDICATE; + } + } + + public record RelationMembershipRepresentation(RelationMembershipRecipe relation) + implements SemanticEntityRepresentation + { + public RelationMembershipRepresentation + { + relation = requireNonNull(relation, "relation is null"); + } + + @Override + public SemanticEntityKind kind() + { + return SemanticEntityKind.RELATION; + } + } + + public record RelationMembershipRecipe(RelationReference relation, String sourceField, String targetField) + { + public RelationMembershipRecipe + { + relation = requireNonNull(relation, "relation is null"); + sourceField = definition(sourceField, "membership source field"); + targetField = definition(targetField, "membership target field"); + } + } + + public record PhysicalQualifiedName(PhysicalIdentifier catalog, PhysicalIdentifier schema, PhysicalIdentifier table) + { + public PhysicalQualifiedName + { + catalog = requireNonNull(catalog, "catalog is null"); + schema = requireNonNull(schema, "schema is null"); + table = requireNonNull(table, "table is null"); + } + } + + public record PhysicalIdentifier(String value, boolean delimited) + { + private static final Pattern UNDELIMITED = Pattern.compile("[A-Za-z_][A-Za-z0-9_]*"); + + public PhysicalIdentifier + { + value = definition(value, "physical identifier"); + if (!delimited && !UNDELIMITED.matcher(value).matches()) { + throw new IllegalArgumentException("invalid physical identifier"); + } + if (!delimited) { + value = value.toLowerCase(Locale.ENGLISH); + } + } + } + + public enum ExpressionRecipeKind + { + FIELD_REFERENCE, LITERAL, FUNCTION_CALL, OPERATOR, CAST, ARGUMENT_REFERENCE, SCOPED_FIELD_REFERENCE, PROPERTY_LOOKUP + } + + public enum SemanticOperator + { + ADD, SUBTRACT, MULTIPLY, DIVIDE, MODULUS, EQUAL, NOT_EQUAL, LESS_THAN, LESS_THAN_OR_EQUAL, GREATER_THAN, GREATER_THAN_OR_EQUAL, AND, OR, NOT, NEGATE, IS_NULL, IS_NOT_NULL, SUBSCRIPT, JSON_OBJECT_LOOKUP + } + + public enum LiteralEncoding + { + NULL, STRING, BOOLEAN, INTEGER, DECIMAL, FLOAT, JSON, BASE64 + } + + public enum RelationKind + { + LOGICAL_TABLE, VIRTUAL_TABLE, SAVED_QUERY, MATERIALIZED_VIEW + } + + public enum FunctionKind + { + SCALAR, AGGREGATE, WINDOW, TABLE + } + + public enum FunctionImplementation + { + STOCK, UDF, REWRITE + } + + public enum FunctionRewrite + { + CAST_DATE, + CAST_DOUBLE, + CAST_SMALLINT, + CAST_BIGINT, + CAST_TIMESTAMP, + CAST_UUID, + CAST_VARCHAR, + ADD_DAYS, + ADD_MONTHS, + AND, + ANY_IF, + ARG_MAX_IF, + ARG_MIN_IF, + ARRAY_ELEMENT, + ARRAY_ENUMERATE, + ARRAY_FILTER, + ARRAY_FIRST, + ARRAY_MAP, + ARRAY_SLICE, + ARRAY_SORT, + ARRAY_SUM, + ASSUME_NOT_NULL, + AVG_IF, + DATE_ADD, + DATE_PART, + DATE_TRUNC_DAY, + DATE_TRUNC_HOUR, + DATE_TRUNC_MONTH, + DATE_TRUNC_WEEK, + COUNT_IF, + COUNT_DISTINCT, + CONVERT_CURRENCY, + DECIMAL_CAST, + DIVIDE_DECIMAL, + EMPTY, + EQUALS, + FLOAT_OR_DEFAULT, + FLOAT_OR_ZERO, + GROUP_UNIQ_ARRAY, + GREATER, + GREATER_OR_EQUAL, + GROUP_ARRAY_IF, + GROUP_UNIQ_ARRAY_IF, + HAS, + IN_ARRAY, + INTERVAL_DAY, + INT_DIV, + INT_OR_ZERO, + INTERVAL_MONTH, + IS_NULL, + IS_NOT_NULL, + JSON_EXTRACT_ARRAY_RAW, + JSON_EXTRACT_BOOL, + JSON_EXTRACT_FLOAT, + JSON_EXTRACT_INT, + JSON_EXTRACT_KEYS, + JSON_EXTRACT_RAW, + JSON_EXTRACT_STRING, + JSON_EXTRACT_TYPED, + JSON_EXTRACT_UINT, + JSON_HAS, + JSON_KEYS_AND_VALUES, + JSON_KEYS_AND_VALUES_RAW, + JSON_LENGTH, + JSON_VALUE, + LIKE, + LESS_OR_EQUAL, + MAX_IF, + MAP_CONSTRUCTOR, + MD5, + MEDIAN_IF, + MIN_IF, + MINUS, + MULTIPLY, + MULTIPLY_DECIMAL, + MULTI_IF, + NOT, + OR, + NOT_EQUALS, + NOT_EMPTY, + PARSE_TIMESTAMP, + PLUS, + QUANTILE, + QUANTILE_EXACT, + QUANTILE_IF, + REGEX_EXTRACT, + REGEX_EXTRACT_ALL, + REGEX_REPLACE_ALL, + REGEX_REPLACE_ONE, + RANGE, + SPLIT_CHAR, + SPLIT_STRING, + START_WEEK, + SUBTRACT_DAYS, + SUBTRACT_MONTHS, + SUBTRACT_YEARS, + SUM_IF, + SURVEY_RESPONSE, + TODAY, + TO_JSON_STRING, + TO_UNIX_TIMESTAMP, + TUPLE, + TUPLE_ELEMENT, + UNIQ_EXACT, + UNIQ_EXACT_IF, + UNIQ_IF + } + + public enum ModifierBehavior + { + COMPILER, TRINO_SESSION_PROPERTY, SAFE_NOOP, UNSUPPORTED + } + + public enum LogicalType + { + UNKNOWN, BOOLEAN, INTEGER, FLOAT, DECIMAL, STRING, DATE, TIMESTAMP, INTERVAL, UUID, JSON, ARRAY, MAP, ROW + } + + public enum PropertyStorage + { + JSON_OBJECT, MAP + } + + public enum RelationshipCardinality + { + ONE_TO_ONE, ONE_TO_MANY, MANY_TO_ONE, MANY_TO_MANY + } + + public enum ExpressionArgument + { + PROPERTY_SOURCE, PROPERTY_KEY + } + + public enum RelationshipJoinSide + { + SOURCE, TARGET + } + + public enum SemanticEntityKind + { + PREDICATE, RELATION + } + + private enum VisitState + { + VISITING, VISITED + } + + private static final class RecipeCounter + { + private int nodes; + + private void add() + { + if (++nodes > MAX_RECIPE_NODES) { + throw new IllegalArgumentException("expression recipes exceed node limit"); + } + } + } + + private record RecipeValidationContext( + String ownerTable, + Map tables, + Map expressions, + Map functions, + List dependencies, + boolean allowFieldReferences, + boolean allowPropertyLookups, + Map arguments, + Map scopedTables) + { + private RecipeValidationContext + { + ownerTable = definition(ownerTable, "recipe owner table"); + tables = requireNonNull(tables, "tables is null"); + expressions = requireNonNull(expressions, "expressions is null"); + functions = requireNonNull(functions, "functions is null"); + scopedTables = Map.copyOf(requireNonNull(scopedTables, "scopedTables is null")); + } + } + + private static final class SemanticRelation + { + private final RelationKind kind; + private Set fields; + private final VirtualTableDefinition virtualTable; + private final SavedQueryReference savedQuery; + + private SemanticRelation(RelationKind kind, Set fields, VirtualTableDefinition virtualTable, SavedQueryReference savedQuery) + { + this.kind = requireNonNull(kind, "kind is null"); + this.fields = fields == null ? null : Set.copyOf(fields); + this.virtualTable = virtualTable; + this.savedQuery = savedQuery; + } + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotCache.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotCache.java new file mode 100644 index 000000000000..fb50feaaf8d8 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotCache.java @@ -0,0 +1,33 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; + +import java.util.Optional; +import java.util.OptionalLong; + +import static java.util.Objects.requireNonNull; + +@FunctionalInterface +public interface HogQlSemanticCatalogSnapshotCache +{ + Optional currentSnapshot(PhysicalIdentifier catalog); + + default Optional currentSnapshot(PhysicalIdentifier catalog, OptionalLong expectedGeneration) + { + requireNonNull(expectedGeneration, "expectedGeneration is null"); + return currentSnapshot(catalog); + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotJsonDecoder.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotJsonDecoder.java new file mode 100644 index 000000000000..e9417cacca22 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotJsonDecoder.java @@ -0,0 +1,899 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.StreamReadConstraints; +import com.fasterxml.jackson.core.StreamReadFeature; +import com.fasterxml.jackson.core.exc.StreamConstraintsException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ActionReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ArgumentReferenceRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.CastRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.CohortReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionArgument; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionRecipeKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FieldReferenceRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCallRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCapabilityDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionImplementation; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionRewrite; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionSignature; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.JoinKey; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LazyProjectionDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LazyTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralEncoding; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.MaterializedViewReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ModifierBehavior; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.OperatorRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PredicateRepresentation; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyLookupRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyStorage; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ReferencedField; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationMembershipRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationMembershipRepresentation; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipCardinality; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipJoinSide; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SavedQueryReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ScopedFieldReferenceRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticEntityKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticModifierDefault; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticOperator; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.TypedLiteral; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.VirtualProjection; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.VirtualTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotLoader.LoadRequest; +import io.trino.hogql.parser.HogQlLanguageVersion; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +import static java.util.Objects.requireNonNull; + +public final class HogQlSemanticCatalogSnapshotJsonDecoder +{ + public static final int PROTOCOL_VERSION = 1; + public static final int MAXIMUM_PAYLOAD_BYTES = 8 * 1024 * 1024; + + private static final int SCHEMA_VERSION = 2; + private static final Limits DEFAULT_LIMITS = new Limits(MAXIMUM_PAYLOAD_BYTES, 256, 100_000); + + private static final Set SNAPSHOT_FIELDS = Set.of( + "protocolVersion", + "schemaVersion", + "languageVersion", + "catalog", + "generation", + "logicalTables", + "expressionFields", + "virtualTables", + "savedQueries", + "materializedViews", + "functions", + "modifierDefaults", + "lazyTables", + "actions", + "cohorts"); + private static final Set REQUIRED_SNAPSHOT_FIELDS = Set.of( + "protocolVersion", + "schemaVersion", + "languageVersion", + "catalog", + "generation", + "logicalTables", + "expressionFields", + "virtualTables", + "savedQueries", + "materializedViews", + "functions", + "modifierDefaults"); + private static final Set IDENTIFIER_FIELDS = Set.of("value", "delimited"); + private static final Set QUALIFIED_NAME_FIELDS = Set.of("catalog", "schema", "table"); + private static final Set TABLE_FIELDS = Set.of("name", "physicalTable", "fields", "properties", "relationships"); + private static final Set LOGICAL_FIELD_FIELDS = Set.of("name", "physicalColumn", "trinoTypeSignature", "logicalType", "nullable", "starVisible"); + private static final Set PROPERTY_FIELDS = Set.of("name", "sourceField", "storage", "logicalType", "nullable", "keyTypeSignature", "valueTypeSignature", "lookupRecipe"); + private static final Set REQUIRED_PROPERTY_FIELDS = Set.of("name", "sourceField", "storage", "logicalType", "nullable"); + private static final Set RELATIONSHIP_FIELDS = Set.of("name", "targetTable", "cardinality", "joinKeys", "joinPredicate"); + private static final Set REQUIRED_RELATIONSHIP_FIELDS = Set.of("name", "targetTable", "cardinality", "joinKeys"); + private static final Set JOIN_KEY_FIELDS = Set.of("sourceField", "targetField"); + private static final Set EXPRESSION_FIELD_FIELDS = Set.of("table", "name", "trinoTypeSignature", "logicalType", "nullable", "starVisible", "recipe"); + private static final Set RECIPE_FIELD_REFERENCE_FIELDS = Set.of("kind", "fieldReference"); + private static final Set RECIPE_LITERAL_FIELDS = Set.of("kind", "literal"); + private static final Set RECIPE_FUNCTION_CALL_FIELDS = Set.of("kind", "functionCall"); + private static final Set RECIPE_OPERATOR_FIELDS = Set.of("kind", "operator"); + private static final Set RECIPE_CAST_FIELDS = Set.of("kind", "cast"); + private static final Set RECIPE_ARGUMENT_REFERENCE_FIELDS = Set.of("kind", "argumentReference"); + private static final Set RECIPE_SCOPED_FIELD_REFERENCE_FIELDS = Set.of("kind", "scopedFieldReference"); + private static final Set RECIPE_PROPERTY_LOOKUP_FIELDS = Set.of("kind", "propertyLookup"); + private static final Set FIELD_REFERENCE_FIELDS = Set.of("table", "field"); + private static final Set TYPED_LITERAL_FIELDS = Set.of("typeSignature", "encoding", "value"); + private static final Set FUNCTION_CALL_FIELDS = Set.of("name", "arguments"); + private static final Set OPERATOR_FIELDS = Set.of("operator", "arguments"); + private static final Set CAST_FIELDS = Set.of("expression", "targetTypeSignature"); + private static final Set ARGUMENT_REFERENCE_FIELDS = Set.of("argument"); + private static final Set SCOPED_FIELD_REFERENCE_FIELDS = Set.of("side", "field"); + private static final Set PROPERTY_LOOKUP_FIELDS = Set.of("table", "property", "key"); + private static final Set VIRTUAL_TABLE_FIELDS = Set.of("name", "source", "projections"); + private static final Set RELATION_REFERENCE_FIELDS = Set.of("kind", "name"); + private static final Set VIRTUAL_PROJECTION_FIELDS = Set.of("name", "sourceField", "starVisible"); + private static final Set SAVED_QUERY_FIELDS = Set.of("name", "queryId", "target", "fields"); + private static final Set MATERIALIZED_VIEW_FIELDS = Set.of("name", "physicalView", "fields"); + private static final Set REFERENCED_FIELD_FIELDS = Set.of("name", "trinoTypeSignature", "logicalType", "nullable", "starVisible"); + private static final Set FUNCTION_FIELDS = Set.of("name", "kind", "implementation", "trinoName", "rewrite", "signatures", "deterministic", "supportsDistinct", "supportsOrderBy", "supportsFilter", "supportsWindow"); + private static final Set REQUIRED_FUNCTION_FIELDS = Set.of("name", "kind", "implementation", "trinoName", "signatures", "deterministic", "supportsDistinct", "supportsOrderBy", "supportsFilter", "supportsWindow"); + private static final Set FUNCTION_SIGNATURE_FIELDS = Set.of("argumentTypes", "returnType", "variadic"); + private static final Set MODIFIER_FIELDS = Set.of("name", "behavior", "defaultValue", "sessionProperty"); + private static final Set MODIFIER_FIELDS_WITHOUT_SESSION_PROPERTY = Set.of("name", "behavior", "defaultValue"); + private static final Set LAZY_TABLE_FIELDS = Set.of("table", "name", "relationshipPath", "projections"); + private static final Set LAZY_PROJECTION_FIELDS = Set.of("name", "trinoTypeSignature", "logicalType", "nullable", "starVisible", "recipe"); + private static final Set ACTION_FIELDS = Set.of("name", "actionId", "table", "representation"); + private static final Set COHORT_FIELDS = Set.of("name", "cohortId", "table", "representation"); + private static final Set PREDICATE_REPRESENTATION_FIELDS = Set.of("kind", "predicate"); + private static final Set RELATION_REPRESENTATION_FIELDS = Set.of("kind", "relation"); + private static final Set RELATION_MEMBERSHIP_FIELDS = Set.of("relation", "sourceField", "targetField"); + + private final Limits limits; + private final ObjectMapper objectMapper; + + public HogQlSemanticCatalogSnapshotJsonDecoder() + { + this(DEFAULT_LIMITS); + } + + public HogQlSemanticCatalogSnapshotJsonDecoder(Limits limits) + { + this.limits = requireNonNull(limits, "limits is null"); + JsonFactory jsonFactory = JsonFactory.builder() + .streamReadConstraints(StreamReadConstraints.builder() + .maxNestingDepth(limits.maximumNestingDepth()) + .maxStringLength(limits.maximumPayloadBytes()) + .maxNumberLength(64) + .build()) + .enable(StreamReadFeature.STRICT_DUPLICATE_DETECTION) + .build(); + this.objectMapper = new ObjectMapper(jsonFactory); + } + + public HogQlSemanticCatalogSnapshot decode(byte[] payload, LoadRequest request) + { + requireNonNull(request, "request is null"); + if (payload == null) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + if (payload.length > limits.maximumPayloadBytes()) { + throw failure(DecodeFailure.LIMIT_EXCEEDED); + } + + try { + ObjectNode root = parse(payload); + validateFields(root, SNAPSHOT_FIELDS, REQUIRED_SNAPSHOT_FIELDS); + + int protocolVersion = integer(root, "protocolVersion"); + if (protocolVersion != PROTOCOL_VERSION) { + throw failure(DecodeFailure.UNSUPPORTED_PROTOCOL); + } + int schemaVersion = integer(root, "schemaVersion"); + if (schemaVersion != SCHEMA_VERSION) { + throw failure(DecodeFailure.UNSUPPORTED_SCHEMA); + } + + HogQlLanguageVersion languageVersion = languageVersion(root); + if (!languageVersion.equals(request.languageVersion())) { + throw failure(DecodeFailure.LANGUAGE_VERSION_MISMATCH); + } + + PhysicalIdentifier catalog = physicalIdentifier(required(root, "catalog")); + if (!catalog.equals(request.catalog())) { + throw failure(DecodeFailure.CATALOG_MISMATCH); + } + + long generation = positiveGeneration(root); + if (request.expectedGeneration().isPresent() && request.expectedGeneration().orElseThrow() != generation) { + throw failure(DecodeFailure.GENERATION_MISMATCH); + } + + CollectionBudget budget = new CollectionBudget(limits.maximumCollectionEntries()); + List tables = logicalTables(required(root, "logicalTables"), budget); + List expressionFields = expressionFields(required(root, "expressionFields"), budget); + List virtualTables = virtualTables(required(root, "virtualTables"), budget); + List savedQueries = savedQueries(required(root, "savedQueries"), budget); + List materializedViews = materializedViews(required(root, "materializedViews"), budget); + List functions = functions(required(root, "functions"), budget); + List modifierDefaults = modifierDefaults(required(root, "modifierDefaults"), budget); + List lazyTables = root.has("lazyTables") ? lazyTables(required(root, "lazyTables"), budget) : List.of(); + List actions = root.has("actions") ? actions(required(root, "actions"), budget) : List.of(); + List cohorts = root.has("cohorts") ? cohorts(required(root, "cohorts"), budget) : List.of(); + return new HogQlSemanticCatalogSnapshot( + protocolVersion, + schemaVersion, + languageVersion, + catalog, + generation, + tables, + expressionFields, + virtualTables, + savedQueries, + materializedViews, + functions, + modifierDefaults, + lazyTables, + actions, + cohorts); + } + catch (DecodeException e) { + throw e; + } + catch (StreamConstraintsException e) { + throw failure(DecodeFailure.LIMIT_EXCEEDED); + } + catch (IOException | RuntimeException e) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + } + + private ObjectNode parse(byte[] payload) + throws IOException + { + try (JsonParser parser = objectMapper.createParser(payload)) { + JsonNode root = objectMapper.readTree(parser); + if (parser.nextToken() != null) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + return object(root); + } + } + + private static HogQlLanguageVersion languageVersion(ObjectNode root) + { + return HogQlLanguageVersion.valueOf(text(root, "languageVersion")); + } + + private static long positiveGeneration(ObjectNode root) + { + JsonNode node = required(root, "generation"); + if (!node.isIntegralNumber() || !node.canConvertToLong() || node.longValue() <= 0) { + throw failure(DecodeFailure.GENERATION_MISMATCH); + } + return node.longValue(); + } + + private static List logicalTables(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List tables = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode table = object(element); + validateFields(table, TABLE_FIELDS); + tables.add(new LogicalTableDefinition( + text(table, "name"), + qualifiedName(required(table, "physicalTable")), + logicalFields(required(table, "fields"), budget), + properties(required(table, "properties"), budget), + relationships(required(table, "relationships"), budget))); + } + return List.copyOf(tables); + } + + private static List logicalFields(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List fields = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode field = object(element); + validateFields(field, LOGICAL_FIELD_FIELDS); + fields.add(new LogicalFieldDefinition( + text(field, "name"), + physicalIdentifier(required(field, "physicalColumn")), + text(field, "trinoTypeSignature"), + enumValue(field, "logicalType", LogicalType.class), + bool(field, "nullable"), + bool(field, "starVisible"))); + } + return List.copyOf(fields); + } + + private static List properties(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List properties = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode property = object(element); + validateFields(property, PROPERTY_FIELDS, REQUIRED_PROPERTY_FIELDS); + properties.add(new PropertyDefinition( + text(property, "name"), + text(property, "sourceField"), + enumValue(property, "storage", PropertyStorage.class), + enumValue(property, "logicalType", LogicalType.class), + bool(property, "nullable"), + property.has("keyTypeSignature") ? Optional.of(text(property, "keyTypeSignature")) : Optional.empty(), + property.has("valueTypeSignature") ? Optional.of(text(property, "valueTypeSignature")) : Optional.empty(), + property.has("lookupRecipe") ? Optional.of(expressionRecipe(required(property, "lookupRecipe"), budget)) : Optional.empty())); + } + return List.copyOf(properties); + } + + private static List relationships(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List relationships = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode relationship = object(element); + validateFields(relationship, RELATIONSHIP_FIELDS, REQUIRED_RELATIONSHIP_FIELDS); + relationships.add(new RelationshipDefinition( + text(relationship, "name"), + text(relationship, "targetTable"), + enumValue(relationship, "cardinality", RelationshipCardinality.class), + joinKeys(required(relationship, "joinKeys"), budget), + relationship.has("joinPredicate") ? Optional.of(expressionRecipe(required(relationship, "joinPredicate"), budget)) : Optional.empty())); + } + return List.copyOf(relationships); + } + + private static List joinKeys(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List joinKeys = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode joinKey = object(element); + validateFields(joinKey, JOIN_KEY_FIELDS); + joinKeys.add(new JoinKey(text(joinKey, "sourceField"), text(joinKey, "targetField"))); + } + return List.copyOf(joinKeys); + } + + private static List expressionFields(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List definitions = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode field = object(element); + validateFields(field, EXPRESSION_FIELD_FIELDS); + definitions.add(new ExpressionFieldDefinition( + text(field, "table"), + text(field, "name"), + text(field, "trinoTypeSignature"), + enumValue(field, "logicalType", LogicalType.class), + bool(field, "nullable"), + bool(field, "starVisible"), + expressionRecipe(required(field, "recipe"), budget))); + } + return List.copyOf(definitions); + } + + private static ExpressionRecipe expressionRecipe(JsonNode node, CollectionBudget budget) + { + ObjectNode recipe = object(node); + ExpressionRecipeKind kind = enumValue(recipe, "kind", ExpressionRecipeKind.class); + return switch (kind) { + case FIELD_REFERENCE -> { + validateFields(recipe, RECIPE_FIELD_REFERENCE_FIELDS); + ObjectNode reference = object(required(recipe, "fieldReference")); + validateFields(reference, FIELD_REFERENCE_FIELDS); + yield new FieldReferenceRecipe(text(reference, "table"), text(reference, "field")); + } + case LITERAL -> { + validateFields(recipe, RECIPE_LITERAL_FIELDS); + yield new LiteralRecipe(typedLiteral(required(recipe, "literal"))); + } + case FUNCTION_CALL -> { + validateFields(recipe, RECIPE_FUNCTION_CALL_FIELDS); + ObjectNode call = object(required(recipe, "functionCall")); + validateFields(call, FUNCTION_CALL_FIELDS); + yield new FunctionCallRecipe(text(call, "name"), expressionRecipes(required(call, "arguments"), budget)); + } + case OPERATOR -> { + validateFields(recipe, RECIPE_OPERATOR_FIELDS); + ObjectNode operator = object(required(recipe, "operator")); + validateFields(operator, OPERATOR_FIELDS); + yield new OperatorRecipe(enumValue(operator, "operator", SemanticOperator.class), expressionRecipes(required(operator, "arguments"), budget)); + } + case CAST -> { + validateFields(recipe, RECIPE_CAST_FIELDS); + ObjectNode cast = object(required(recipe, "cast")); + validateFields(cast, CAST_FIELDS); + yield new CastRecipe(expressionRecipe(required(cast, "expression"), budget), text(cast, "targetTypeSignature")); + } + case ARGUMENT_REFERENCE -> { + validateFields(recipe, RECIPE_ARGUMENT_REFERENCE_FIELDS); + ObjectNode reference = object(required(recipe, "argumentReference")); + validateFields(reference, ARGUMENT_REFERENCE_FIELDS); + yield new ArgumentReferenceRecipe(enumValue(reference, "argument", ExpressionArgument.class)); + } + case SCOPED_FIELD_REFERENCE -> { + validateFields(recipe, RECIPE_SCOPED_FIELD_REFERENCE_FIELDS); + ObjectNode reference = object(required(recipe, "scopedFieldReference")); + validateFields(reference, SCOPED_FIELD_REFERENCE_FIELDS); + yield new ScopedFieldReferenceRecipe( + enumValue(reference, "side", RelationshipJoinSide.class), + text(reference, "field")); + } + case PROPERTY_LOOKUP -> { + validateFields(recipe, RECIPE_PROPERTY_LOOKUP_FIELDS); + ObjectNode lookup = object(required(recipe, "propertyLookup")); + validateFields(lookup, PROPERTY_LOOKUP_FIELDS); + yield new PropertyLookupRecipe( + text(lookup, "table"), + text(lookup, "property"), + expressionRecipe(required(lookup, "key"), budget)); + } + }; + } + + private static List expressionRecipes(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List recipes = new ArrayList<>(array.size()); + for (JsonNode element : array) { + recipes.add(expressionRecipe(element, budget)); + } + return List.copyOf(recipes); + } + + private static TypedLiteral typedLiteral(JsonNode node) + { + ObjectNode literal = object(node); + validateFields(literal, TYPED_LITERAL_FIELDS); + return new TypedLiteral( + text(literal, "typeSignature"), + enumValue(literal, "encoding", LiteralEncoding.class), + text(literal, "value")); + } + + private static List virtualTables(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List definitions = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode table = object(element); + validateFields(table, VIRTUAL_TABLE_FIELDS); + definitions.add(new VirtualTableDefinition( + text(table, "name"), + relationReference(required(table, "source")), + virtualProjections(required(table, "projections"), budget))); + } + return List.copyOf(definitions); + } + + private static RelationReference relationReference(JsonNode node) + { + ObjectNode reference = object(node); + validateFields(reference, RELATION_REFERENCE_FIELDS); + return new RelationReference(enumValue(reference, "kind", RelationKind.class), text(reference, "name")); + } + + private static List virtualProjections(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List projections = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode projection = object(element); + validateFields(projection, VIRTUAL_PROJECTION_FIELDS); + projections.add(new VirtualProjection(text(projection, "name"), text(projection, "sourceField"), bool(projection, "starVisible"))); + } + return List.copyOf(projections); + } + + private static List savedQueries(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List definitions = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode savedQuery = object(element); + validateFields(savedQuery, SAVED_QUERY_FIELDS); + definitions.add(new SavedQueryReference( + text(savedQuery, "name"), + text(savedQuery, "queryId"), + relationReference(required(savedQuery, "target")), + referencedFields(required(savedQuery, "fields"), budget))); + } + return List.copyOf(definitions); + } + + private static List materializedViews(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List definitions = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode view = object(element); + validateFields(view, MATERIALIZED_VIEW_FIELDS); + definitions.add(new MaterializedViewReference( + text(view, "name"), + qualifiedName(required(view, "physicalView")), + referencedFields(required(view, "fields"), budget))); + } + return List.copyOf(definitions); + } + + private static List referencedFields(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List fields = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode field = object(element); + validateFields(field, REFERENCED_FIELD_FIELDS); + fields.add(new ReferencedField( + text(field, "name"), + text(field, "trinoTypeSignature"), + enumValue(field, "logicalType", LogicalType.class), + bool(field, "nullable"), + bool(field, "starVisible"))); + } + return List.copyOf(fields); + } + + private static List functions(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List definitions = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode function = object(element); + validateFields(function, FUNCTION_FIELDS, REQUIRED_FUNCTION_FIELDS); + definitions.add(new FunctionCapabilityDefinition( + text(function, "name"), + enumValue(function, "kind", FunctionKind.class), + enumValue(function, "implementation", FunctionImplementation.class), + physicalIdentifiers(required(function, "trinoName"), budget), + function.has("rewrite") ? Optional.of(enumValue(function, "rewrite", FunctionRewrite.class)) : Optional.empty(), + functionSignatures(required(function, "signatures"), budget), + bool(function, "deterministic"), + bool(function, "supportsDistinct"), + bool(function, "supportsOrderBy"), + bool(function, "supportsFilter"), + bool(function, "supportsWindow"))); + } + return List.copyOf(definitions); + } + + private static List physicalIdentifiers(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List identifiers = new ArrayList<>(array.size()); + for (JsonNode element : array) { + identifiers.add(physicalIdentifier(element)); + } + return List.copyOf(identifiers); + } + + private static List functionSignatures(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List signatures = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode signature = object(element); + validateFields(signature, FUNCTION_SIGNATURE_FIELDS); + signatures.add(new FunctionSignature( + strings(required(signature, "argumentTypes"), budget), + text(signature, "returnType"), + bool(signature, "variadic"))); + } + return List.copyOf(signatures); + } + + private static List strings(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List values = new ArrayList<>(array.size()); + for (JsonNode element : array) { + if (!element.isTextual()) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + values.add(element.textValue()); + } + return List.copyOf(values); + } + + private static List modifierDefaults(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List modifiers = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode modifier = object(element); + if (modifier.has("sessionProperty")) { + validateFields(modifier, MODIFIER_FIELDS); + } + else { + validateFields(modifier, MODIFIER_FIELDS_WITHOUT_SESSION_PROPERTY); + } + modifiers.add(new SemanticModifierDefault( + text(modifier, "name"), + enumValue(modifier, "behavior", ModifierBehavior.class), + typedLiteral(required(modifier, "defaultValue")), + modifier.has("sessionProperty") ? physicalIdentifiers(required(modifier, "sessionProperty"), budget) : List.of())); + } + return List.copyOf(modifiers); + } + + private static List lazyTables(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List definitions = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode table = object(element); + validateFields(table, LAZY_TABLE_FIELDS); + definitions.add(new LazyTableDefinition( + text(table, "table"), + text(table, "name"), + strings(required(table, "relationshipPath"), budget), + lazyProjections(required(table, "projections"), budget))); + } + return List.copyOf(definitions); + } + + private static List lazyProjections(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List projections = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode projection = object(element); + validateFields(projection, LAZY_PROJECTION_FIELDS); + projections.add(new LazyProjectionDefinition( + text(projection, "name"), + text(projection, "trinoTypeSignature"), + enumValue(projection, "logicalType", LogicalType.class), + bool(projection, "nullable"), + bool(projection, "starVisible"), + expressionRecipe(required(projection, "recipe"), budget))); + } + return List.copyOf(projections); + } + + private static List actions(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List actions = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode action = object(element); + validateFields(action, ACTION_FIELDS); + actions.add(new ActionReference( + text(action, "name"), + text(action, "actionId"), + text(action, "table"), + semanticEntityRepresentation(required(action, "representation"), budget))); + } + return List.copyOf(actions); + } + + private static List cohorts(JsonNode node, CollectionBudget budget) + { + ArrayNode array = array(node, budget); + List cohorts = new ArrayList<>(array.size()); + for (JsonNode element : array) { + ObjectNode cohort = object(element); + validateFields(cohort, COHORT_FIELDS); + cohorts.add(new CohortReference( + text(cohort, "name"), + text(cohort, "cohortId"), + text(cohort, "table"), + semanticEntityRepresentation(required(cohort, "representation"), budget))); + } + return List.copyOf(cohorts); + } + + private static HogQlSemanticCatalogSnapshot.SemanticEntityRepresentation semanticEntityRepresentation(JsonNode node, CollectionBudget budget) + { + ObjectNode representation = object(node); + SemanticEntityKind kind = enumValue(representation, "kind", SemanticEntityKind.class); + return switch (kind) { + case PREDICATE -> { + validateFields(representation, PREDICATE_REPRESENTATION_FIELDS); + yield new PredicateRepresentation(expressionRecipe(required(representation, "predicate"), budget)); + } + case RELATION -> { + validateFields(representation, RELATION_REPRESENTATION_FIELDS); + ObjectNode membership = object(required(representation, "relation")); + validateFields(membership, RELATION_MEMBERSHIP_FIELDS); + yield new RelationMembershipRepresentation(new RelationMembershipRecipe( + relationReference(required(membership, "relation")), + text(membership, "sourceField"), + text(membership, "targetField"))); + } + }; + } + + private static PhysicalQualifiedName qualifiedName(JsonNode node) + { + ObjectNode name = object(node); + validateFields(name, QUALIFIED_NAME_FIELDS); + return new PhysicalQualifiedName( + physicalIdentifier(required(name, "catalog")), + physicalIdentifier(required(name, "schema")), + physicalIdentifier(required(name, "table"))); + } + + private static PhysicalIdentifier physicalIdentifier(JsonNode node) + { + ObjectNode identifier = object(node); + validateFields(identifier, IDENTIFIER_FIELDS); + return new PhysicalIdentifier(text(identifier, "value"), bool(identifier, "delimited")); + } + + private static ObjectNode object(JsonNode node) + { + if (!(node instanceof ObjectNode object)) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + return object; + } + + private static ArrayNode array(JsonNode node, CollectionBudget budget) + { + if (!(node instanceof ArrayNode array)) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + budget.add(array.size()); + return array; + } + + private static JsonNode required(ObjectNode object, String name) + { + JsonNode node = object.get(name); + if (node == null || node.isNull()) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + return node; + } + + private static String text(ObjectNode object, String name) + { + JsonNode node = required(object, name); + if (!node.isTextual()) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + return node.textValue(); + } + + private static int integer(ObjectNode object, String name) + { + JsonNode node = required(object, name); + if (!node.isIntegralNumber() || !node.canConvertToInt()) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + return node.intValue(); + } + + private static boolean bool(ObjectNode object, String name) + { + JsonNode node = required(object, name); + if (!node.isBoolean()) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + return node.booleanValue(); + } + + private static > E enumValue(ObjectNode object, String name, Class enumType) + { + return Enum.valueOf(enumType, text(object, name)); + } + + private static void validateFields(ObjectNode object, Set allowedFields) + { + Iterator fieldNames = object.fieldNames(); + while (fieldNames.hasNext()) { + if (!allowedFields.contains(fieldNames.next())) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + } + if (object.size() != allowedFields.size()) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + } + + private static void validateFields(ObjectNode object, Set allowedFields, Set requiredFields) + { + Iterator fieldNames = object.fieldNames(); + while (fieldNames.hasNext()) { + if (!allowedFields.contains(fieldNames.next())) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + } + if (!requiredFields.stream().allMatch(object::has)) { + throw failure(DecodeFailure.INVALID_PAYLOAD); + } + } + + private static DecodeException failure(DecodeFailure failure) + { + return new DecodeException(failure); + } + + public record Limits(int maximumPayloadBytes, int maximumNestingDepth, int maximumCollectionEntries) + { + public Limits + { + if (maximumPayloadBytes <= 0 || maximumNestingDepth <= 0 || maximumCollectionEntries <= 0) { + throw new IllegalArgumentException("HogQL semantic catalog decoder limits must be positive"); + } + } + } + + public static final class DecodeException + extends RuntimeException + { + private final DecodeFailure failure; + + private DecodeException(DecodeFailure failure) + { + super(requireNonNull(failure, "failure is null").message()); + this.failure = failure; + } + + public DecodeFailure failure() + { + return failure; + } + } + + public enum DecodeFailure + { + INVALID_PAYLOAD("Invalid HogQL semantic catalog payload"), + LIMIT_EXCEEDED("HogQL semantic catalog payload limit exceeded"), + UNSUPPORTED_PROTOCOL("Unsupported HogQL semantic catalog protocol"), + UNSUPPORTED_SCHEMA("Unsupported HogQL semantic catalog schema"), + LANGUAGE_VERSION_MISMATCH("HogQL semantic catalog language version mismatch"), + CATALOG_MISMATCH("HogQL semantic catalog identifier mismatch"), + GENERATION_MISMATCH("HogQL semantic catalog generation mismatch"); + + private final String message; + + DecodeFailure(String message) + { + this.message = message; + } + + private String message() + { + return message; + } + } + + private static final class CollectionBudget + { + private final int maximumEntries; + private int entries; + + private CollectionBudget(int maximumEntries) + { + this.maximumEntries = maximumEntries; + } + + private void add(int size) + { + if (size > maximumEntries - entries) { + throw failure(DecodeFailure.LIMIT_EXCEEDED); + } + entries += size; + } + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotLoader.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotLoader.java new file mode 100644 index 000000000000..c70ce34fe85c --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotLoader.java @@ -0,0 +1,70 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.parser.HogQlLanguageVersion; + +import java.util.OptionalLong; +import java.util.concurrent.CompletionStage; + +import static java.util.Objects.requireNonNull; + +@FunctionalInterface +public interface HogQlSemanticCatalogSnapshotLoader +{ + CompletionStage load(LoadRequest request); + + static HogQlSemanticCatalogSnapshotLoader fromJsonTransport( + JsonTransport transport, + HogQlSemanticCatalogSnapshotJsonDecoder decoder) + { + requireNonNull(transport, "transport is null"); + requireNonNull(decoder, "decoder is null"); + return request -> { + requireNonNull(request, "request is null"); + CompletionStage response = requireNonNull(transport.load(request), "transport returned null"); + return response.thenApply(payload -> decoder.decode(payload, request)); + }; + } + + @FunctionalInterface + interface JsonTransport + { + CompletionStage load(LoadRequest request); + } + + record LoadRequest(PhysicalIdentifier catalog, HogQlLanguageVersion languageVersion, OptionalLong expectedGeneration) + { + public LoadRequest + { + catalog = requireNonNull(catalog, "catalog is null"); + languageVersion = requireNonNull(languageVersion, "languageVersion is null"); + expectedGeneration = requireNonNull(expectedGeneration, "expectedGeneration is null"); + if (expectedGeneration.isPresent() && expectedGeneration.orElseThrow() <= 0) { + throw new IllegalArgumentException("expected generation must be positive"); + } + } + + public static LoadRequest latest(PhysicalIdentifier catalog, HogQlLanguageVersion languageVersion) + { + return new LoadRequest(catalog, languageVersion, OptionalLong.empty()); + } + + public static LoadRequest pinned(PhysicalIdentifier catalog, HogQlLanguageVersion languageVersion, long generation) + { + return new LoadRequest(catalog, languageVersion, OptionalLong.of(generation)); + } + } +} diff --git a/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotProvider.java b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotProvider.java new file mode 100644 index 000000000000..1bd5d8dd2f57 --- /dev/null +++ b/core/trino-hogql-compiler/src/main/java/io/trino/hogql/compiler/catalog/HogQlSemanticCatalogSnapshotProvider.java @@ -0,0 +1,80 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogException.Failure; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.parser.HogQlLanguageVersion; + +import java.util.Optional; +import java.util.OptionalLong; + +import static java.util.Objects.requireNonNull; + +@FunctionalInterface +public interface HogQlSemanticCatalogSnapshotProvider +{ + PinnedSnapshot pin(PinRequest request); + + static HogQlSemanticCatalogSnapshotProvider fromCache(HogQlSemanticCatalogSnapshotCache cache) + { + requireNonNull(cache, "cache is null"); + return request -> { + HogQlSemanticCatalogSnapshot snapshot = cache.currentSnapshot(request.catalog(), request.expectedGeneration()) + .orElseThrow(() -> new HogQlSemanticCatalogException(Failure.UNAVAILABLE, "HogQL semantic catalog snapshot is unavailable")); + if (!snapshot.catalog().equals(request.catalog())) { + throw new HogQlSemanticCatalogException(Failure.CATALOG_MISMATCH, "HogQL semantic catalog snapshot does not match the requested catalog"); + } + if (!snapshot.languageVersion().equals(request.languageVersion())) { + throw new HogQlSemanticCatalogException(Failure.LANGUAGE_VERSION_MISMATCH, "HogQL semantic catalog snapshot language version does not match the compiler"); + } + if (request.expectedGeneration().isPresent() && request.expectedGeneration().orElseThrow() != snapshot.generation()) { + throw new HogQlSemanticCatalogException(Failure.GENERATION_MISMATCH, "HogQL semantic catalog snapshot generation does not match the request"); + } + return new PinnedSnapshot(snapshot); + }; + } + + record PinRequest(PhysicalIdentifier catalog, HogQlLanguageVersion languageVersion, OptionalLong expectedGeneration) + { + public PinRequest + { + catalog = requireNonNull(catalog, "catalog is null"); + languageVersion = requireNonNull(languageVersion, "languageVersion is null"); + expectedGeneration = requireNonNull(expectedGeneration, "expectedGeneration is null"); + if (expectedGeneration.isPresent() && expectedGeneration.orElseThrow() <= 0) { + throw new IllegalArgumentException("expected HogQL semantic catalog generation must be positive"); + } + } + } + + record PinnedSnapshot(HogQlSemanticCatalogSnapshot snapshot) + { + public PinnedSnapshot + { + snapshot = requireNonNull(snapshot, "snapshot is null"); + } + + public long generation() + { + return snapshot.generation(); + } + + public Optional logicalTable(String name) + { + return snapshot.logicalTable(name); + } + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/BenchmarkHogQlCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/BenchmarkHogQlCompiler.java new file mode 100644 index 000000000000..6e9edeb9569d --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/BenchmarkHogQlCompiler.java @@ -0,0 +1,169 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FieldReferenceRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCallRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCapabilityDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionImplementation; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionSignature; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.sql.tree.Statement; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.profile.GCProfiler; +import org.openjdk.jmh.runner.RunnerException; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.TimeUnit; + +import static io.trino.jmh.Benchmarks.benchmark; + +@State(Scope.Thread) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@BenchmarkMode(Mode.AverageTime) +@Fork(1) +@Warmup(iterations = 5, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 10, time = 500, timeUnit = TimeUnit.MILLISECONDS) +public class BenchmarkHogQlCompiler +{ + private static final String SIMPLE_QUERY = "SELECT event, distinct_id FROM events WHERE team_id = 42"; + private static final String JOIN_WINDOW_QUERY = + """ + SELECT e.event, p.person_id, + row_number() OVER (PARTITION BY e.distinct_id ORDER BY e.timestamp) AS row_number + FROM events AS e + LEFT JOIN persons AS p ON e.distinct_id = p.distinct_id + WHERE e.event = 'signup' + """; + private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier("ducklake", false); + private static final HogQlSemanticCatalogSnapshot SEMANTIC_SNAPSHOT = semanticSnapshot(); + private static final Optional SEMANTIC_CONTEXT = Optional.of( + new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SEMANTIC_SNAPSHOT))); + private static final HogQlCompileEnvelope SEMANTIC_QUERY = new HogQlCompileEnvelope( + "SELECT lowerEvent FROM events WHERE event = 'signup'", + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + Map.of(), + Map.of(), + Map.of(), + Map.of(), + OptionalLong.of(1)); + + private final HogQlCompiler compiler = new HogQlCompiler(); + + @Benchmark + public Statement compileSimpleQuery() + { + return compiler.compile(SIMPLE_QUERY); + } + + @Benchmark + public Statement compileJoinWindowQuery() + { + return compiler.compile(JOIN_WINDOW_QUERY); + } + + @Benchmark + public HogQlCompilationResult compileSemanticCatalogQuery() + { + return compiler.compile(SEMANTIC_QUERY, SEMANTIC_CONTEXT); + } + + static void main() + throws RunnerException + { + benchmark(BenchmarkHogQlCompiler.class) + .withOptions(options -> options.addProfiler(GCProfiler.class)) + .run(); + } + + private static HogQlSemanticCatalogSnapshot semanticSnapshot() + { + LogicalFieldDefinition event = new LogicalFieldDefinition( + "event", + new PhysicalIdentifier("event_name", false), + "varchar", + LogicalType.STRING, + false, + true); + LogicalFieldDefinition distinctId = new LogicalFieldDefinition( + "distinct_id", + new PhysicalIdentifier("distinct_id", false), + "varchar", + LogicalType.STRING, + false, + true); + LogicalTableDefinition events = new LogicalTableDefinition( + "events", + new PhysicalQualifiedName( + CATALOG, + new PhysicalIdentifier("analytics", false), + new PhysicalIdentifier("events_data", false)), + List.of(event, distinctId), + List.of(), + List.of()); + ExpressionFieldDefinition lowerEvent = new ExpressionFieldDefinition( + "events", + "lowerEvent", + "varchar", + LogicalType.STRING, + false, + true, + new FunctionCallRecipe("lower", List.of(new FieldReferenceRecipe("events", "event")))); + FunctionCapabilityDefinition lower = new FunctionCapabilityDefinition( + "lower", + FunctionKind.SCALAR, + FunctionImplementation.STOCK, + List.of(new PhysicalIdentifier("lower", false)), + List.of(new FunctionSignature(List.of("varchar"), "varchar", false)), + true, + false, + false, + false, + false); + return new HogQlSemanticCatalogSnapshot( + HogQlSemanticCatalogSnapshot.PROTOCOL_VERSION, + HogQlSemanticCatalogSnapshot.SCHEMA_VERSION, + HogQlLanguageContract.current().languageVersion(), + CATALOG, + 1, + List.of(events), + List.of(lowerEvent), + List.of(), + List.of(), + List.of(), + List.of(lower), + List.of()); + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompileEnvelope.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompileEnvelope.java new file mode 100644 index 000000000000..9aafff73725d --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompileEnvelope.java @@ -0,0 +1,162 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.HogQlTypedValue.ArrayValue; +import io.trino.hogql.compiler.HogQlTypedValue.NumberValue; +import io.trino.hogql.compiler.HogQlTypedValue.ObjectValue; +import io.trino.hogql.compiler.HogQlTypedValue.StringValue; +import io.trino.hogql.compiler.HogQlTypedValue.Value; +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.hogql.parser.HogQlLanguageVersion; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.OptionalLong; +import java.util.function.Supplier; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.params.provider.Arguments.arguments; + +public class TestHogQlCompileEnvelope +{ + private static final HogQlLanguageVersion LANGUAGE_VERSION = HogQlLanguageContract.current().languageVersion(); + + @ParameterizedTest + @ValueSource(ints = {-1, 0, 2, Integer.MAX_VALUE}) + public void testRejectsUnknownProtocolVersions(int protocolVersion) + { + assertThatThrownBy(() -> envelope(protocolVersion, LANGUAGE_VERSION, Map.of(), Map.of(), Map.of(), Map.of())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("unsupported HogQL protocol version"); + } + + @ParameterizedTest + @ValueSource(strings = {"0.0.0", "1.0.1", "2.0.0"}) + public void testRejectsUnknownLanguageVersions(String languageVersion) + { + assertThatThrownBy(() -> envelope(HogQlCompileEnvelope.PROTOCOL_VERSION, HogQlLanguageVersion.valueOf(languageVersion), Map.of(), Map.of(), Map.of(), Map.of())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("unsupported HogQL language version"); + } + + @ParameterizedTest + @MethodSource("ambiguousTypedValues") + public void testRejectsAmbiguousTypedValues(Supplier invalidValue, String sensitiveValue) + { + assertThatThrownBy(invalidValue::get) + .isInstanceOfAny(IllegalArgumentException.class, NullPointerException.class) + .message() + .doesNotContain(sensitiveValue); + } + + @ParameterizedTest + @ValueSource(strings = {"actions", "cohorts", "savedQueries"}) + public void testRejectsUnknownSemanticFields(String semanticField) + { + Map> fields = Map.of( + semanticField, Map.of("input", new HogQlTypedValue("varchar", new StringValue("sensitive-value")))); + + assertThatThrownBy(() -> HogQlCompileEnvelope.fromSemanticFields( + "SELECT 1", + HogQlCompileEnvelope.PROTOCOL_VERSION, + LANGUAGE_VERSION, + fields, + OptionalLong.empty())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("unknown HogQL semantic field"); + } + + @ParameterizedTest + @MethodSource("sensitiveValues") + public void testTypedValuesAreRedacted(Value value, String sensitiveValue) + { + HogQlTypedValue typedValue = new HogQlTypedValue("varchar", value); + + assertThat(value.toString()).doesNotContain(sensitiveValue); + assertThat(typedValue.toString()).doesNotContain(sensitiveValue); + } + + @ParameterizedTest + @ValueSource(strings = {"parameters", "variables", "filters", "modifiers"}) + public void testEnvelopeRenderingRedactsEveryValueCollection(String semanticField) + { + String sensitiveValue = "sensitive-value-" + semanticField; + Map> fields = Map.of( + semanticField, Map.of("input", new HogQlTypedValue("varchar", new StringValue(sensitiveValue)))); + HogQlCompileEnvelope envelope = HogQlCompileEnvelope.fromSemanticFields( + "SELECT 'sensitive-query'", + HogQlCompileEnvelope.PROTOCOL_VERSION, + LANGUAGE_VERSION, + fields, + OptionalLong.of(42)); + + assertThat(envelope.toString()) + .doesNotContain(sensitiveValue, "sensitive-query") + .contains("query="); + } + + private static Stream ambiguousTypedValues() + { + String sensitiveValue = "sensitive-value"; + List arrayWithMissingValue = new ArrayList<>(); + arrayWithMissingValue.add(new StringValue(sensitiveValue)); + arrayWithMissingValue.add(null); + Map objectWithMissingValue = new HashMap<>(); + objectWithMissingValue.put("input", new StringValue(sensitiveValue)); + objectWithMissingValue.put("missing", null); + return Stream.of( + arguments((Supplier) () -> new HogQlTypedValue(null, new StringValue(sensitiveValue)), sensitiveValue), + arguments((Supplier) () -> new HogQlTypedValue("", new StringValue(sensitiveValue)), sensitiveValue), + arguments((Supplier) () -> new HogQlTypedValue("varchar", null), sensitiveValue), + arguments((Supplier) () -> new ArrayValue(arrayWithMissingValue), sensitiveValue), + arguments((Supplier) () -> new ObjectValue(objectWithMissingValue), sensitiveValue)); + } + + private static Stream sensitiveValues() + { + return Stream.of( + arguments(new StringValue("sensitive-string"), "sensitive-string"), + arguments(new NumberValue("98765432101234567890"), "98765432101234567890"), + arguments(new ArrayValue(List.of(new StringValue("sensitive-array"))), "sensitive-array"), + arguments(new ObjectValue(Map.of("sensitive-key", new StringValue("sensitive-object"))), "sensitive-object")); + } + + private static HogQlCompileEnvelope envelope( + int protocolVersion, + HogQlLanguageVersion languageVersion, + Map parameters, + Map variables, + Map filters, + Map modifiers) + { + return new HogQlCompileEnvelope( + "SELECT 1", + protocolVersion, + languageVersion, + parameters, + variables, + filters, + modifiers, + OptionalLong.empty()); + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java new file mode 100644 index 000000000000..f327dd5885b7 --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCompiler.java @@ -0,0 +1,906 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.HogQlTypedValue.StringValue; +import io.trino.spi.Location; +import io.trino.spi.TrinoException; +import io.trino.sql.SqlFormatter; +import io.trino.sql.parser.SqlParser; +import io.trino.sql.tree.AliasedRelation; +import io.trino.sql.tree.AllColumns; +import io.trino.sql.tree.Identifier; +import io.trino.sql.tree.InPredicate; +import io.trino.sql.tree.Join; +import io.trino.sql.tree.JoinOn; +import io.trino.sql.tree.Node; +import io.trino.sql.tree.NodeLocation; +import io.trino.sql.tree.Parameter; +import io.trino.sql.tree.Pivot; +import io.trino.sql.tree.Predicated; +import io.trino.sql.tree.Query; +import io.trino.sql.tree.QuerySpecification; +import io.trino.sql.tree.SingleColumn; +import io.trino.sql.tree.Statement; +import io.trino.sql.tree.SubqueryExpression; +import io.trino.sql.tree.SubscriptExpression; +import io.trino.sql.tree.Table; +import io.trino.sql.tree.TableSubquery; +import io.trino.sql.tree.Union; +import io.trino.sql.tree.Values; +import io.trino.sql.tree.WithQuery; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Deque; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowableOfType; +import static org.junit.jupiter.params.provider.Arguments.arguments; + +public class TestHogQlCompiler +{ + private final HogQlCompiler compiler = new HogQlCompiler(); + private final SqlParser sqlParser = new SqlParser(); + + @ParameterizedTest + @ValueSource(strings = { + "SELECT 1", + "select 'event'", + "SELECT TRUE, FALSE, NULL", + "SELECT * FROM events", + "SELECT event, properties FROM ducklake.default.events", + "SELECT * FROM \"MiXeD\".\"Event Table\"", + "SELECT event + 1 * 2 FROM events WHERE event >= 3 AND NOT false", + "SELECT lower(event) AS lowered FROM events", + "SELECT 01", + "SELECT -1, +2", + }) + public void testLowersToEquivalentStockTrinoAst(String hogql) + { + Statement statement = compiler.compile(hogql); + + assertThat(statement).isEqualTo(sqlParser.createStatement(hogql)); + assertThat(sqlParser.createStatement(SqlFormatter.formatSql(statement))).isEqualTo(statement); + } + + @Test + public void testQuotesHogQlPropertyIdentifiersForTrino() + { + assertThat(compiler.compile("SELECT properties.$user_id, properties.$host FROM events")) + .isEqualTo(sqlParser.createStatement("SELECT properties.\"$user_id\", properties.\"$host\" FROM events")); + } + + @Test + public void testExpandsSelectAliasesInClausesOutsideTrinoAliasScope() + { + assertThat(compiler.compile( + "SELECT number + 1 AS next FROM numbers(3) WHERE next > 1 GROUP BY next HAVING next < 4 ORDER BY next")) + .isEqualTo(compiler.compile( + "SELECT number + 1 AS next FROM numbers(3) WHERE number + 1 > 1 GROUP BY number + 1 HAVING number + 1 < 4 ORDER BY number + 1")); + } + + @Test + public void testDoesNotExpandQualifiedReferencesMatchingSelectAliases() + { + Statement statement = compiler.compile("SELECT number + 1 AS next FROM numbers(3) AS source WHERE source.next > 1"); + + assertThat(SqlFormatter.formatSql(statement)).contains("source.next > 1"); + } + + @Test + public void testLowersDistinctOrderingAndClickHousePaginationOrder() + { + Statement statement = compiler.compile("SELECT DISTINCT event FROM events ORDER BY event DESC NULLS FIRST LIMIT 10 OFFSET 2"); + + assertThat(statement).isEqualTo(sqlParser.createStatement("SELECT DISTINCT event FROM events ORDER BY event DESC NULLS FIRST OFFSET 2 LIMIT 10")); + } + + @Test + public void testLowersLimitByToPartitionedRowNumber() + { + Statement statement = compiler.compile( + "SELECT event, created_at AS ts FROM events ORDER BY ts DESC LIMIT 1, 2 BY event LIMIT 10"); + + assertThat(statement).isEqualTo(sqlParser.createStatement( + "SELECT __hogql_limit_by_ranked.__hogql_limit_by_column_0 AS event, " + + "__hogql_limit_by_ranked.__hogql_limit_by_column_1 AS ts " + + "FROM (" + + "SELECT __hogql_limit_by_base.__hogql_limit_by_column_0 AS __hogql_limit_by_column_0, " + + "__hogql_limit_by_base.__hogql_limit_by_column_1 AS __hogql_limit_by_column_1, " + + "row_number() OVER (PARTITION BY __hogql_limit_by_base.__hogql_limit_by_column_0 " + + "ORDER BY __hogql_limit_by_base.__hogql_limit_by_column_1 DESC) AS __hogql_limit_by_row_number " + + "FROM (SELECT event AS __hogql_limit_by_column_0, created_at AS __hogql_limit_by_column_1 FROM events) " + + "AS __hogql_limit_by_base" + + ") AS __hogql_limit_by_ranked " + + "WHERE __hogql_limit_by_ranked.__hogql_limit_by_row_number > 1 " + + "AND __hogql_limit_by_ranked.__hogql_limit_by_row_number <= 1 + 2 " + + "ORDER BY __hogql_limit_by_ranked.__hogql_limit_by_column_1 DESC LIMIT 10")); + } + + @Test + public void testWrapsSetOperandsWithLocalClauses() + { + Statement statement = compiler.compile("SELECT 1 ORDER BY 1 LIMIT 1 UNION ALL SELECT 2"); + + assertThat(statement).isEqualTo(sqlParser.createStatement("(SELECT 1 ORDER BY 1 LIMIT 1) UNION ALL SELECT 2")); + } + + @Test + public void testPreservesUnqualifiedOutputReferencesInDerivedSetQueries() + { + Statement statement = compiler.compile( + "SELECT * FROM (SELECT 1 AS value UNION ALL SELECT 2 ORDER BY value) AS nested"); + + assertThat(statement).isEqualTo(sqlParser.createStatement( + "SELECT * FROM (SELECT 1 AS value UNION ALL SELECT 2 ORDER BY value) AS nested")); + } + + @Test + public void testLowersNumbersTableFunction() + { + Statement statement = compiler.compile("SELECT number FROM numbers(5)"); + + assertThat(statement).isEqualTo(sqlParser.createStatement( + "SELECT number FROM UNNEST(if(5 <= 0, CAST(ARRAY[] AS array(bigint)), sequence(0, 5 - 1))) AS numbers(number)")); + } + + @ParameterizedTest + @CsvSource(delimiter = '|', textBlock = """ + SELECT [1, 2, 3] | SELECT ARRAY[1, 2, 3] + SELECT ARRAY[] | SELECT ARRAY[] + SELECT (1, 'example') | SELECT (1, 'example') + SELECT (1,) | SELECT ROW(1) + SELECT [1, 2, 3][1] | SELECT ARRAY[1, 2, 3][1] + SELECT (1, 'example').2 | SELECT ROW(1, 'example')[2] + SELECT attributes['plan'] | SELECT attributes['plan'] + SELECT nested[position[1]] | SELECT nested[position[1]] + SELECT [payload][1].plan | SELECT ARRAY[payload][1].plan + SELECT value BETWEEN 1 AND 10 | SELECT value BETWEEN 1 AND 10 + SELECT value NOT BETWEEN 1 AND 10| SELECT value NOT BETWEEN 1 AND 10 + SELECT value IS NULL | SELECT value IS NULL + SELECT value IS NOT NULL | SELECT value IS NOT NULL + SELECT value IN (1, 2, 3) | SELECT value IN (1, 2, 3) + SELECT value IN (1) | SELECT value IN (1) + SELECT value NOT IN (1, 2, 3) | SELECT value NOT IN (1, 2, 3) + SELECT value IN [1, 2, 3] | SELECT value IN (1, 2, 3) + SELECT value NOT IN [1] | SELECT value NOT IN (1) + SELECT value IN [] | SELECT IF(value IS NULL, NULL, false) + SELECT value NOT IN [] | SELECT IF(value IS NULL, NULL, true) + SELECT value ?? fallback | SELECT coalesce(value, fallback) + SELECT first ?? second ?? third | SELECT coalesce(coalesce(first, second), third) + SELECT value LIKE 'pro%' | SELECT value LIKE 'pro%' + SELECT value NOT LIKE 'pro%' | SELECT value NOT LIKE 'pro%' + SELECT value ILIKE 'Pro%' | SELECT lower(value) LIKE lower('Pro%') + SELECT value NOT ILIKE 'Pro%' | SELECT lower(value) NOT LIKE lower('Pro%') + SELECT 1.5 | SELECT 1.5E0 + SELECT .5 | SELECT .5E0 + SELECT 1. | SELECT 1E0 + SELECT -1.5 | SELECT -1.5E0 + SELECT +1.5 | SELECT +1.5E0 + SELECT 1e3 | SELECT 1E3 + SELECT map(x -> x + 1, [1]) | SELECT map(x -> x + 1, ARRAY[1]) + SELECT map((x, y) -> x + y, [1]) | SELECT map((x, y) -> x + y, ARRAY[1]) + SELECT map(lambda x: x + 1, [1]) | SELECT map(x -> x + 1, ARRAY[1]) + """) + public void testLowersCollectionAndPredicateExpressions(String hogql, String trinoSql) + { + assertThat(compiler.compile(hogql)).isEqualTo(sqlParser.createStatement(trinoSql)); + } + + @Test + public void testLowersConcatenationOperator() + { + assertThat(compiler.compile("SELECT first || second")) + .isEqualTo(sqlParser.createStatement("SELECT first || second")); + } + + @ParameterizedTest + @CsvSource(delimiter = '|', textBlock = """ + SELECT item FROM events ARRAY JOIN attributes AS item | SELECT item FROM events CROSS JOIN UNNEST(attributes) AS __hogql_array_join(item) + SELECT item FROM events LEFT ARRAY JOIN attributes AS item | SELECT item FROM events LEFT JOIN UNNEST(attributes) AS __hogql_array_join(item) ON TRUE + SELECT item, key FROM events ARRAY JOIN attributes AS item, keys AS key | SELECT item, key FROM events CROSS JOIN UNNEST(attributes, keys) AS __hogql_array_join(item, key) + SELECT item ARRAY JOIN [1] AS item | SELECT item FROM UNNEST(ARRAY[1]) AS __hogql_array_join(item) + """) + public void testLowersArrayJoin(String hogql, String trinoSql) + { + assertThat(compiler.compile(hogql)).isEqualTo(sqlParser.createStatement(trinoSql)); + } + + @ParameterizedTest + @CsvSource(delimiter = '|', textBlock = """ + SELECT INTERVAL 2 SECOND | SELECT 2 * INTERVAL '1' SECOND + SELECT INTERVAL value MINUTE | SELECT value * INTERVAL '1' MINUTE + SELECT INTERVAL 2 HOUR | SELECT 2 * INTERVAL '1' HOUR + SELECT INTERVAL 2 DAY | SELECT 2 * INTERVAL '1' DAY + SELECT INTERVAL 2 WEEK | SELECT 2 * INTERVAL '7' DAY + SELECT INTERVAL 2 MONTH | SELECT 2 * INTERVAL '1' MONTH + SELECT INTERVAL 2 QUARTER | SELECT 2 * INTERVAL '3' MONTH + SELECT INTERVAL value YEAR | SELECT value * INTERVAL '1' YEAR + SELECT INTERVAL '5 months' | SELECT 5 * INTERVAL '1' MONTH + """) + public void testLowersCanonicalIntervals(String hogql, String trinoSql) + { + Statement statement = compiler.compile(hogql); + + assertThat(statement).isEqualTo(sqlParser.createStatement(trinoSql)); + assertThat(sqlParser.createStatement(SqlFormatter.formatSql(statement))).isEqualTo(statement); + } + + @Test + public void testBindsPlaceholdersInsideCollectionSubscriptsBySourceOrder() + { + HogQlCompilationResult result = compiler.compile( + "SELECT [{element}][{index}], ({left}, {right}).2, {object}.field", + Map.of( + "element", typedValue("element"), + "index", typedValue("index"), + "left", typedValue("left"), + "right", typedValue("right"), + "object", typedValue("object"))); + + assertThat(result.parameterNames()).containsExactly("element", "index", "left", "right", "object"); + assertThat(parameters(result.statement())) + .extracting(Parameter::getId) + .containsExactly(0, 1, 2, 3, 4); + } + + @Test + public void testPreservesCollectionSubscriptSourceLocations() + { + Query query = (Query) compiler.compile("SELECT [10, 20][2], (1, 'two').2"); + QuerySpecification querySpecification = (QuerySpecification) query.getQueryBody(); + SubscriptExpression arrayAccess = (SubscriptExpression) ((SingleColumn) querySpecification.getSelect().getSelectItems().getFirst()).getExpression(); + SubscriptExpression tupleAccess = (SubscriptExpression) ((SingleColumn) querySpecification.getSelect().getSelectItems().get(1)).getExpression(); + + assertThat(arrayAccess.getLocation()).contains(new NodeLocation(1, 8)); + assertThat(arrayAccess.getIndex().getLocation()).contains(new NodeLocation(1, 17)); + assertThat(tupleAccess.getLocation()).contains(new NodeLocation(1, 21)); + assertThat(tupleAccess.getIndex().getLocation()).contains(new NodeLocation(1, 32)); + } + + @ParameterizedTest + @ValueSource(strings = { + "SELECT CASE WHEN enabled THEN 1 ELSE 0 END", + "SELECT CASE value WHEN 1 THEN 'one' WHEN 2 THEN 'two' ELSE 'other' END", + }) + public void testLowersCaseAndCastExpressions(String hogql) + { + assertThat(compiler.compile(hogql)).isEqualTo(sqlParser.createStatement(hogql)); + } + + @ParameterizedTest + @MethodSource("representableHogQlCastTypes") + public void testLowersRepresentableHogQlCastTypes(String hogql, String trinoSql) + { + assertThat(compiler.compile(hogql)).isEqualTo(sqlParser.createStatement(trinoSql)); + } + + @ParameterizedTest + @MethodSource("unsupportedHogQlCastTypes") + public void testRejectsHogQlCastTypesWithoutExactTrinoRepresentation(String type, String reason) + { + String hogql = "SELECT CAST(value AS " + type + ")"; + + TrinoException exception = catchThrowableOfType(TrinoException.class, () -> compiler.compile(hogql)); + + assertThat(exception.getErrorCode()).isEqualTo(HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE.toErrorCode()); + assertThat(exception.getLocation()).contains(new Location(1, hogql.indexOf(type) + 1)); + assertThat(exception) + .hasMessageContaining("HogQL cast type cannot be represented exactly in Trino") + .hasMessageContaining(reason); + } + + @ParameterizedTest + @ValueSource(strings = { + "SELECT event, count(*) AS total FROM events GROUP BY event HAVING count(*) > 1", + "SELECT lower(event), count(DISTINCT person_id), sum(revenue) FROM events GROUP BY lower(event) HAVING sum(revenue) > 10", + "SELECT count(*) FILTER (WHERE enabled) FROM events", + "SELECT array_agg(event ORDER BY timestamp DESC) FROM events", + }) + public void testLowersOrdinaryGroupingHavingAndAggregateFunctions(String hogql) + { + Statement statement = compiler.compile(hogql); + + assertThat(statement).isEqualTo(sqlParser.createStatement(hogql)); + assertThat(sqlParser.createStatement(SqlFormatter.formatSql(statement))).isEqualTo(statement); + } + + @Test + public void testBindsPlaceholdersAcrossGroupingHavingAndAggregateModifiers() + { + String hogql = "SELECT array_agg({value} ORDER BY {sort}) FILTER (WHERE {filter}) FROM events " + + "GROUP BY {group} HAVING sum({having}) > {threshold}"; + Map bindings = Map.of( + "value", typedValue("value"), + "sort", typedValue("sort"), + "filter", typedValue("filter"), + "group", typedValue("group"), + "having", typedValue("having"), + "threshold", typedValue("threshold")); + + HogQlCompilationResult result = compiler.compile(hogql, bindings); + + assertThat(result.parameterNames()).containsExactly("value", "sort", "filter", "group", "having", "threshold"); + assertThat(parameters(result.statement())) + .extracting(Parameter::getId) + .containsExactly(0, 1, 2, 3, 4, 5); + } + + @ParameterizedTest + @ValueSource(strings = { + "SELECT e.id FROM events AS e", + "SELECT date.id FROM events AS date", + "SELECT \"e\".id FROM events AS \"e\"", + "SELECT e.id FROM events e JOIN persons p ON e.person_id = p.id", + "SELECT * FROM events LEFT OUTER JOIN persons USING (id)", + "SELECT * FROM events RIGHT JOIN persons ON events.id = persons.id", + "SELECT * FROM events FULL OUTER JOIN persons USING (id, team_id)", + "SELECT * FROM events CROSS JOIN persons", + "SELECT * FROM (events CROSS JOIN persons)", + }) + public void testLowersAliasesAndStockJoins(String hogql) + { + Statement statement = compiler.compile(hogql); + + assertThat(statement).isEqualTo(sqlParser.createStatement(hogql)); + assertThat(sqlParser.createStatement(SqlFormatter.formatSql(statement))).isEqualTo(statement); + } + + @Test + public void testLowersUnparenthesizedUsing() + { + assertThat(compiler.compile("SELECT * FROM events JOIN persons USING id")) + .isEqualTo(sqlParser.createStatement("SELECT * FROM events JOIN persons USING (id)")); + } + + @Test + public void testLowersPivotToEquivalentStockAstWithSourceLocations() + { + String hogql = "SELECT * FROM orders PIVOT (sum(totalprice) AS total FOR (orderstatus, custkey) " + + "IN (('F', 1) AS filled, ('O', 2) AS open) GROUP BY clerk)"; + + Query query = (Query) compiler.compile(hogql); + Pivot pivot = (Pivot) ((QuerySpecification) query.getQueryBody()).getFrom().orElseThrow(); + + assertThat(query).isEqualTo(sqlParser.createStatement(hogql)); + assertThat(sqlParser.createStatement(SqlFormatter.formatSql(query))).isEqualTo(query); + assertThat(pivot.getLocation()).contains(new NodeLocation(1, hogql.indexOf("orders") + 1)); + assertThat(pivot.getAggregations()).singleElement().satisfies(aggregation -> { + assertThat(aggregation.getLocation()).contains(new NodeLocation(1, hogql.indexOf("sum(totalprice)") + 1)); + assertThat(aggregation.getAlias()).get().satisfies(alias -> + assertThat(alias.getLocation()).contains(new NodeLocation(1, hogql.indexOf("total FOR") + 1))); + }); + assertThat(pivot.getPivotColumns()).hasSize(2); + assertThat(pivot.getValueGroups()).hasSize(2); + assertThat(pivot.getValueGroups().getFirst().getLocation()) + .contains(new NodeLocation(1, hogql.indexOf("('F', 1)") + 1)); + assertThat(pivot.getValueGroups().getFirst().getAlias()).get().satisfies(alias -> + assertThat(alias.getLocation()).contains(new NodeLocation(1, hogql.indexOf("filled") + 1))); + assertThat(pivot.getGroupBy()).isPresent(); + } + + @Test + public void testBindsPlaceholdersInsidePivot() + { + HogQlCompilationResult result = compiler.compile( + "SELECT * FROM orders PIVOT (sum({amount}) FOR orderstatus IN ({status}))", + Map.of("amount", typedValue("amount"), "status", typedValue("status"))); + + assertThat(result.parameterNames()).containsExactly("amount", "status"); + assertThat(parameters(result.statement())).extracting(Parameter::getId).containsExactly(0, 1); + } + + @ParameterizedTest + @ValueSource(strings = { + "WITH base AS (SELECT 1 AS id) SELECT id FROM base", + "WITH base(id) AS (SELECT 1), next AS (SELECT id FROM base) SELECT id FROM next", + "WITH base AS (WITH base AS (SELECT 1 AS id) SELECT id FROM base) SELECT id FROM base", + "SELECT derived.id FROM (SELECT id FROM events) AS derived", + "WITH base AS (SELECT id FROM events) SELECT derived.id FROM (SELECT id FROM base) derived", + }) + public void testLowersCtesAndDerivedTables(String hogql) + { + Statement statement = compiler.compile(hogql); + + assertThat(statement).isEqualTo(sqlParser.createStatement(hogql)); + assertThat(sqlParser.createStatement(SqlFormatter.formatSql(statement))).isEqualTo(statement); + } + + @Test + public void testLowersCorrelatedInSubqueryWithSourceLocations() + { + String hogql = + """ + SELECT o.orderkey + FROM orders o + WHERE o.custkey IN ( + SELECT c.custkey + FROM customer c + WHERE c.custkey = o.custkey + ) + """; + + Query query = (Query) compiler.compile(hogql); + Predicated where = (Predicated) ((QuerySpecification) query.getQueryBody()).getWhere().orElseThrow(); + InPredicate predicate = (InPredicate) where.getPredicate(); + SubqueryExpression subquery = (SubqueryExpression) predicate.getValueList(); + + assertThat(query).isEqualTo(sqlParser.createStatement(hogql)); + assertThat(where.getLocation()).contains(new NodeLocation(3, 17)); + assertThat(subquery.getLocation()).contains(new NodeLocation(4, 5)); + assertThat(subquery.getQuery().getLocation()).contains(new NodeLocation(4, 5)); + } + + @Test + public void testLowersCorrelatedScalarSubqueryWithSourceLocations() + { + String hogql = "SELECT o.orderkey, (SELECT c.custkey FROM customer c WHERE c.custkey = o.custkey) AS matched FROM orders o"; + + Query query = (Query) compiler.compile(hogql); + SingleColumn projection = (SingleColumn) ((QuerySpecification) query.getQueryBody()).getSelect().getSelectItems().get(1); + SubqueryExpression subquery = (SubqueryExpression) projection.getExpression(); + + assertThat(query).isEqualTo(sqlParser.createStatement(hogql)); + assertThat(subquery.getLocation()).contains(new NodeLocation(1, 20)); + assertThat(subquery.getQuery().getLocation()).contains(new NodeLocation(1, 21)); + } + + @Test + public void testBindsPlaceholdersAcrossCteAndDerivedQueryScopes() + { + HogQlCompilationResult result = compiler.compile( + "WITH base AS (SELECT {cte}) SELECT {outer} FROM (SELECT {derived} FROM base) nested WHERE {where}", + Map.of( + "cte", typedValue("cte"), + "outer", typedValue("outer"), + "derived", typedValue("derived"), + "where", typedValue("where"))); + + assertThat(result.parameterNames()).containsExactly("cte", "outer", "derived", "where"); + assertThat(parameters(result.statement())) + .extracting(Parameter::getId) + .containsExactly(0, 1, 2, 3); + } + + @Test + public void testPreservesCteAndDerivedTableSourceLocations() + { + Query query = (Query) compiler.compile("WITH base AS (SELECT id FROM events)\nSELECT d.id FROM (SELECT id FROM base) AS d"); + WithQuery commonTable = query.getWith().orElseThrow().getQueries().getFirst(); + AliasedRelation aliasedRelation = (AliasedRelation) ((QuerySpecification) query.getQueryBody()).getFrom().orElseThrow(); + TableSubquery derivedTable = (TableSubquery) aliasedRelation.getRelation(); + + assertThat(query.getWith().orElseThrow().getLocation()).contains(new NodeLocation(1, 1)); + assertThat(commonTable.getLocation()).contains(new NodeLocation(1, 6)); + assertThat(commonTable.getQuery().getLocation()).contains(new NodeLocation(1, 15)); + assertThat(aliasedRelation.getLocation()).contains(new NodeLocation(2, 18)); + assertThat(derivedTable.getLocation()).contains(new NodeLocation(2, 18)); + assertThat(derivedTable.getQuery().getLocation()).contains(new NodeLocation(2, 19)); + } + + @ParameterizedTest + @ValueSource(strings = { + "SELECT 1 UNION SELECT 2", + "SELECT 1 UNION ALL SELECT 2", + "SELECT 1 UNION DISTINCT SELECT 2", + "SELECT 1 INTERSECT SELECT 2", + "SELECT 1 INTERSECT ALL SELECT 2", + "SELECT 1 INTERSECT DISTINCT SELECT 2", + "SELECT 1 EXCEPT SELECT 2", + "SELECT 1 EXCEPT ALL SELECT 2", + "SELECT 1 UNION SELECT 2 INTERSECT SELECT 2", + "SELECT 1 EXCEPT SELECT 2 UNION SELECT 3", + "SELECT 1 UNION (SELECT 2 EXCEPT SELECT 3)", + "WITH base AS (SELECT 1 AS id) SELECT id FROM base UNION SELECT id FROM base", + "WITH base AS (SELECT 1 UNION ALL SELECT 2) SELECT * FROM base", + "SELECT * FROM (SELECT 1 UNION ALL SELECT 2) AS data", + "SELECT * FROM (VALUES (1, 'first'), (2, 'second')) AS data(id, label)", + }) + public void testLowersSetOperationsAndValuesToStockTrinoAst(String hogql) + { + Statement statement = compiler.compile(hogql); + + assertThat(statement).isEqualTo(sqlParser.createStatement(hogql)); + assertThat(sqlParser.createStatement(SqlFormatter.formatSql(statement))).isEqualTo(statement); + } + + @Test + public void testLowersSetOperationOrderLimitAndOffset() + { + Statement statement = compiler.compile("SELECT 1 UNION SELECT 2 ORDER BY 1 LIMIT 3 OFFSET 1"); + + assertThat(statement).isEqualTo(sqlParser.createStatement("SELECT 1 UNION SELECT 2 ORDER BY 1 OFFSET 1 ROW LIMIT 3")); + assertThat(sqlParser.createStatement(SqlFormatter.formatSql(statement))).isEqualTo(statement); + } + + @Test + public void testLowersLargeSetOperation() + { + int operandCount = 3_000; + StringBuilder hogql = new StringBuilder("SELECT 0"); + for (int operand = 1; operand < operandCount; operand++) { + hogql.append(" UNION ALL SELECT ").append(operand); + } + + Query query = (Query) compiler.compile(hogql.toString()); + Deque pending = new ArrayDeque<>(); + pending.add(new NodeDepth(query, 0)); + int querySpecificationCount = 0; + int maximumUnionDepth = 0; + while (!pending.isEmpty()) { + NodeDepth current = pending.removeFirst(); + Node node = current.node(); + if (node instanceof QuerySpecification) { + querySpecificationCount++; + } + if (node instanceof Union) { + maximumUnionDepth = Math.max(maximumUnionDepth, current.depth()); + } + for (Node child : node.getChildren()) { + pending.add(new NodeDepth(child, current.depth() + 1)); + } + } + + assertThat(querySpecificationCount).isEqualTo(operandCount); + assertThat(maximumUnionDepth).isLessThan(20); + } + + @Test + public void testBindsPlaceholdersAcrossSetBranchesAndValuesRows() + { + HogQlCompilationResult result = compiler.compile( + "SELECT {left} UNION ALL SELECT {right} FROM (VALUES ({first}), ({second})) AS data(value)", + Map.of( + "left", typedValue("left"), + "right", typedValue("right"), + "first", typedValue("first"), + "second", typedValue("second"))); + + assertThat(result.parameterNames()).containsExactly("left", "right", "first", "second"); + assertThat(parameters(result.statement())) + .extracting(Parameter::getId) + .containsExactly(0, 1, 2, 3); + } + + @Test + public void testPreservesSetOperationAndValuesSourceLocations() + { + Query query = (Query) compiler.compile("SELECT 1\nUNION ALL\nSELECT * FROM (VALUES (2), (3)) data(value)"); + Union union = (Union) query.getQueryBody(); + QuerySpecification right = (QuerySpecification) union.getRelations().get(1); + AliasedRelation alias = (AliasedRelation) right.getFrom().orElseThrow(); + TableSubquery valuesSubquery = (TableSubquery) alias.getRelation(); + Values values = (Values) valuesSubquery.getQuery().getQueryBody(); + + assertThat(union.getLocation()).contains(new NodeLocation(2, 1)); + assertThat(alias.getLocation()).contains(new NodeLocation(3, 15)); + assertThat(valuesSubquery.getLocation()).contains(new NodeLocation(3, 15)); + assertThat(values.getLocation()).contains(new NodeLocation(3, 16)); + } + + @Test + public void testLowersLeftAnyJoinToCorrelatedLateralLimit() + { + Statement statement = compiler.compile( + "SELECT e.id, p.name FROM events e LEFT ANY JOIN persons p ON e.person_id = p.id"); + Statement expected = sqlParser.createStatement( + "SELECT e.id, p.name FROM events e " + + "LEFT JOIN LATERAL (SELECT * FROM persons p WHERE e.person_id = p.id ORDER BY p.id LIMIT 1) p ON true"); + + assertThat(statement).isEqualTo(expected); + assertThat(sqlParser.createStatement(SqlFormatter.formatSql(statement))).isEqualTo(statement); + } + + @Test + public void testLowersInnerAnyJoinUsingToCorrelatedLateralLimit() + { + Statement statement = compiler.compile( + "SELECT e.id, p.name FROM events e ANY INNER JOIN persons p USING (person_id)"); + Statement expected = sqlParser.createStatement( + "SELECT e.id, p.name FROM events e " + + "INNER JOIN LATERAL (SELECT * FROM persons p WHERE e.person_id = p.person_id ORDER BY p.person_id LIMIT 1) p ON true"); + + assertThat(statement).isEqualTo(expected); + } + + @Test + public void testLowersLeftAnyJoinUsingToCorrelatedLateralLimit() + { + Statement statement = compiler.compile( + "SELECT e.id, p.name FROM events e LEFT ANY JOIN persons p USING (person_id)"); + Statement expected = sqlParser.createStatement( + "SELECT e.id, p.name FROM events e " + + "LEFT JOIN LATERAL (SELECT * FROM persons p WHERE e.person_id = p.person_id ORDER BY p.person_id LIMIT 1) p ON true"); + + assertThat(statement).isEqualTo(expected); + } + + @Test + public void testPreservesAliasedJoinSourceLocations() + { + Query query = (Query) compiler.compile("SELECT e.id\nFROM events AS e\nLEFT JOIN persons AS p ON e.person_id = p.id"); + Join join = (Join) ((QuerySpecification) query.getQueryBody()).getFrom().orElseThrow(); + AliasedRelation left = (AliasedRelation) join.getLeft(); + JoinOn criteria = (JoinOn) join.getCriteria().orElseThrow(); + + assertThat(join.getLocation()).contains(new NodeLocation(2, 6)); + assertThat(left.getLocation()).contains(new NodeLocation(2, 6)); + assertThat(left.getAlias().getLocation()).contains(new NodeLocation(2, 16)); + assertThat(criteria.getExpression().getLocation()).contains(new NodeLocation(3, 27)); + } + + @Test + public void testBindsPlaceholdersInJoinCriteriaBySourceOrder() + { + HogQlCompilationResult result = compiler.compile( + "SELECT {projection} FROM events e JOIN persons p ON e.id = {join_value} WHERE {where_value}", + Map.of( + "projection", typedValue("projection"), + "join_value", typedValue("join"), + "where_value", typedValue("where"))); + + assertThat(result.parameterNames()).containsExactly("projection", "join_value", "where_value"); + assertThat(parameters(result.statement())) + .extracting(Parameter::getId) + .containsExactly(0, 1, 2); + } + + @ParameterizedTest + @ValueSource(strings = { + "SELECT value BETWEEN 1 AND 10", + "SELECT value NOT BETWEEN 1 AND 10", + "SELECT value IS NULL", + "SELECT value IS NOT NULL", + "SELECT value IN (1, 2, 3)", + "SELECT value NOT IN (1, 2, 3)", + }) + public void testPreservesPredicateSourceLocations(String hogql) + { + Query query = (Query) compiler.compile(hogql); + QuerySpecification querySpecification = (QuerySpecification) query.getQueryBody(); + SingleColumn column = (SingleColumn) querySpecification.getSelect().getSelectItems().getFirst(); + Predicated predicated = (Predicated) column.getExpression(); + + assertThat(predicated.getLocation()).contains(new NodeLocation(1, 14)); + assertThat(predicated.getPredicate().getLocation()).contains(new NodeLocation(1, 14)); + } + + @Test + public void testPreservesQuotedIdentifiersAndSourceLocations() + { + Statement statement = compiler.compile( + """ + SELECT + `Event Name`, + * + FROM + `Analytics`.`Event Table` + """); + + Query query = (Query) statement; + QuerySpecification querySpecification = (QuerySpecification) query.getQueryBody(); + SingleColumn column = (SingleColumn) querySpecification.getSelect().getSelectItems().getFirst(); + Identifier columnIdentifier = (Identifier) column.getExpression(); + AllColumns allColumns = (AllColumns) querySpecification.getSelect().getSelectItems().get(1); + Table table = (Table) querySpecification.getFrom().orElseThrow(); + + assertThat(query.getLocation()).contains(new NodeLocation(1, 1)); + assertThat(column.getLocation()).contains(new NodeLocation(2, 5)); + assertThat(columnIdentifier.getValue()).isEqualTo("Event Name"); + assertThat(columnIdentifier.isDelimited()).isTrue(); + assertThat(allColumns.getLocation()).contains(new NodeLocation(3, 5)); + assertThat(table.getLocation()).contains(new NodeLocation(5, 5)); + assertThat(table.getName().getOriginalParts()) + .extracting(Identifier::getValue) + .containsExactly("Analytics", "Event Table"); + assertThat(table.getName().getOriginalParts()) + .allMatch(Identifier::isDelimited); + } + + @Test + public void testLowersNamedPlaceholdersToOrderedPositionalParameters() + { + String firstValue = "sensitive-first-value"; + String laterValue = "sensitive-later-value"; + Map bindings = new LinkedHashMap<>(); + bindings.put("first", typedValue(firstValue)); + bindings.put("later", typedValue(laterValue)); + + HogQlCompilationResult result = compiler.compile("SELECT {later},\n {first} + {later}", bindings); + List parameters = parameters(result.statement()); + + assertThat(result.parameterNames()).containsExactly("later", "first", "later"); + assertThat(parameters) + .extracting(Parameter::getId) + .containsExactly(0, 1, 2); + assertThat(parameters) + .extracting(Parameter::getLocation) + .containsExactly( + Optional.of(new NodeLocation(1, 8)), + Optional.of(new NodeLocation(2, 2)), + Optional.of(new NodeLocation(2, 12))); + assertThat(SqlFormatter.formatSql(result.statement())) + .contains("?") + .doesNotContain(firstValue, laterValue, "{first}", "{later}"); + } + + @ParameterizedTest + @MethodSource("bindingErrors") + public void testReportsStableSourceLocatedBindingErrors( + String hogql, + Map bindings, + String expectedMessage, + Location expectedLocation, + String sensitiveValue) + { + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile(hogql, bindings)); + + assertThat(exception.getErrorCode()).isEqualTo(HogQlErrorCode.HOGQL_BINDING_ERROR.toErrorCode()); + assertThat(exception.getLocation()).contains(expectedLocation); + assertThat(exception) + .hasMessage(expectedMessage) + .message() + .doesNotContain(sensitiveValue); + } + + @Test + public void testReturnsOnlyStockTrinoTreeNodes() + { + Statement statement = compiler.compile("SELECT event, * FROM ducklake.default.events"); + Deque pending = new ArrayDeque<>(); + pending.add(statement); + + while (!pending.isEmpty()) { + Node node = pending.removeFirst(); + assertThat(node.getClass().getPackageName()).isEqualTo("io.trino.sql.tree"); + pending.addAll(node.getChildren()); + } + } + + @Test + public void testReportsSourceLocatedHogQlSyntaxError() + { + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile("SELECT 1\nGROUP BY ALL")); + + assertThat(exception.getErrorCode()).isEqualTo(HogQlErrorCode.HOGQL_SYNTAX_ERROR.toErrorCode()); + assertThat(exception.getLocation()).contains(new Location(2, 1)); + assertThat(exception).hasMessageStartingWith("line 2:1:"); + } + + private static Stream bindingErrors() + { + return Stream.of( + arguments( + "SELECT\n {missing}", + Map.of(), + "line 2:2: Missing HogQL parameter bindings: missing", + new Location(2, 2), + "sensitive-missing-value"), + arguments( + "SELECT 1", + Map.of("extra", typedValue("sensitive-extra-value")), + "line 1:1: Unused HogQL parameter bindings: extra", + new Location(1, 1), + "sensitive-extra-value"), + arguments( + "SELECT *\nFROM {source}", + Map.of("source", typedValue("sensitive-table-value")), + "line 2:6: HogQL parameter placeholders are not supported in table positions: source", + new Location(2, 6), + "sensitive-table-value")); + } + + private static Stream representableHogQlCastTypes() + { + return Stream.of( + arguments("SELECT CAST(value AS Int8)", "SELECT CAST(value AS tinyint)"), + arguments("SELECT CAST(value AS Int16)", "SELECT CAST(value AS smallint)"), + arguments("SELECT CAST(value AS Int32)", "SELECT CAST(value AS integer)"), + arguments("SELECT CAST(value AS Int64)", "SELECT CAST(value AS bigint)"), + arguments("SELECT CAST(value AS Int)", "SELECT CAST(value AS integer)"), + arguments("SELECT CAST(value AS Integer)", "SELECT CAST(value AS integer)"), + arguments("SELECT CAST(value AS Float32)", "SELECT CAST(value AS real)"), + arguments("SELECT CAST(value AS Float64)", "SELECT CAST(value AS double)"), + arguments("SELECT CAST(value AS Float)", "SELECT CAST(value AS real)"), + arguments("SELECT CAST(value AS Real)", "SELECT CAST(value AS real)"), + arguments("SELECT CAST(value AS String)", "SELECT CAST(value AS varchar)"), + arguments("SELECT TRY_CAST(value AS VARCHAR)", "SELECT TRY_CAST(value AS varchar)"), + arguments("SELECT CAST(value AS Bool)", "SELECT CAST(value AS boolean)"), + arguments("SELECT CAST(value AS Date)", "SELECT CAST(value AS date)"), + arguments("SELECT CAST(value AS UUID)", "SELECT CAST(value AS uuid)"), + arguments("SELECT CAST(value AS JSON)", "SELECT CAST(value AS json)"), + arguments("SELECT CAST(value AS Decimal(18, 4))", "SELECT CAST(value AS decimal(18, 4))"), + arguments("SELECT CAST(value AS Decimal32(2))", "SELECT CAST(value AS decimal(9, 2))"), + arguments("SELECT CAST(value AS Decimal64(4))", "SELECT CAST(value AS decimal(18, 4))"), + arguments("SELECT CAST(value AS Decimal128(8))", "SELECT CAST(value AS decimal(38, 8))"), + arguments("SELECT TRY_CAST(value AS Nullable(Decimal(18, 4)))", "SELECT TRY_CAST(value AS decimal(18, 4))"), + arguments("SELECT CAST(value AS Array(Nullable(Int32)))", "SELECT CAST(value AS array(integer))"), + arguments("SELECT CAST(value AS Map(String, Float64))", "SELECT CAST(value AS map(varchar, double))"), + arguments("SELECT CAST(value AS Tuple(Int64, String))", "SELECT CAST(value AS row(bigint, varchar))"), + arguments("SELECT CAST(value AS Tuple(id Int64, label String))", "SELECT CAST(value AS row(id bigint, label varchar))"), + arguments("SELECT CAST(value AS Timestamp(6))", "SELECT CAST(value AS timestamp(6))"), + arguments("SELECT CAST(value AS Timestamp WITH TIME ZONE)", "SELECT CAST(value AS timestamp(3) with time zone)"), + arguments("SELECT CAST(value AS Time(12))", "SELECT CAST(value AS time(12))"), + arguments("SELECT CAST(value AS Interval Day To Second)", "SELECT CAST(value AS interval day to second)"), + arguments("SELECT CAST(value AS Interval Year To Month)", "SELECT CAST(value AS interval year to month)")); + } + + private static Stream unsupportedHogQlCastTypes() + { + return Stream.of( + arguments("UInt8", "unsigned integer range"), + arguments("UInt16", "unsigned integer range"), + arguments("UInt32", "unsigned integer range"), + arguments("UInt64", "unsigned integer range"), + arguments("UInt128", "unsigned integer range"), + arguments("UInt256", "unsigned integer range"), + arguments("Int128", "integer width exceeds Trino bigint"), + arguments("Int256", "integer width exceeds Trino bigint"), + arguments("Decimal256(2)", "decimal precision exceeds Trino"), + arguments("Decimal(39, 2)", "precision <= 38"), + arguments("Decimal(4, 5)", "scale <= precision"), + arguments("Nullable(Array(Int32))", "Nullable cannot wrap"), + arguments("Nullable(Map(String, Int32))", "Nullable cannot wrap"), + arguments("Nullable(Tuple(Int32))", "Nullable cannot wrap"), + arguments("FixedString(16)", "padding semantics"), + arguments("Date32", "Date32 range"), + arguments("DateTime", "time-zone and range semantics"), + arguments("DateTime64(3, 'UTC')", "time-zone and range semantics"), + arguments("Timestamp(13)", "precision exceeds 12"), + arguments("Time(13)", "precision exceeds 12"), + arguments("Timestamp WITH LOCAL TIME ZONE", "WITH LOCAL TIME ZONE semantics"), + arguments("Interval", "interval qualifier is required")); + } + + private static HogQlTypedValue typedValue(String value) + { + return new HogQlTypedValue("varchar", new StringValue(value)); + } + + private static List parameters(Statement statement) + { + Deque pending = new ArrayDeque<>(); + List parameters = new ArrayList<>(); + pending.add(statement); + while (!pending.isEmpty()) { + Node node = pending.removeFirst(); + if (node instanceof Parameter parameter) { + parameters.add(parameter); + } + pending.addAll(node.getChildren()); + } + return parameters.stream() + .sorted(Comparator.comparingInt(Parameter::getId)) + .toList(); + } + + private record NodeDepth(Node node, int depth) {} +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCurrencyCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCurrencyCompiler.java new file mode 100644 index 000000000000..6dcddc58b7d8 --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlCurrencyCompiler.java @@ -0,0 +1,81 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshot; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshot.ExchangeRate; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotProvider; +import io.trino.spi.TrinoException; +import io.trino.sql.parser.SqlParser; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; + +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_RESOLUTION_ERROR; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowableOfType; + +public class TestHogQlCurrencyCompiler +{ + private final SqlParser sqlParser = new SqlParser(); + + @Test + public void testPinsOneGenerationAndLowersAllCurrencyCalls() + { + AtomicInteger pins = new AtomicInteger(); + HogQlExchangeRateSnapshotProvider provider = expectedGeneration -> { + assertThat(expectedGeneration).isEmpty(); + pins.incrementAndGet(); + return new HogQlExchangeRateSnapshotProvider.PinnedSnapshot(snapshot(42)); + }; + + HogQlCompilationResult result = new HogQlCompiler(provider).compile( + "SELECT convertCurrency('USD', 'EUR', 100), " + + "convertCurrency(source_currency, target_currency, amount, event_date), " + + "convertCurrency('USD', 'EUR', 100, _toDate('2024-01-01')) FROM events", + java.util.Map.of()); + + assertThat(pins).hasValue(1); + assertThat(result.exchangeRateGeneration()).hasValue(42); + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT " + + "hogql_convert_currency(42, CAST('USD' AS varchar), CAST('EUR' AS varchar), CAST(100 AS decimal(38, 10)), CAST(now() AS date)), " + + "hogql_convert_currency(42, CAST(source_currency AS varchar), CAST(target_currency AS varchar), CAST(amount AS decimal(38, 10)), CAST(event_date AS date)), " + + "hogql_convert_currency(42, CAST('USD' AS varchar), CAST('EUR' AS varchar), CAST(100 AS decimal(38, 10)), CAST(CAST('2024-01-01' AS date) AS date)) " + + "FROM events")); + } + + @Test + public void testFailsClosedWithoutExchangeRateProvider() + { + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> new HogQlCompiler().compile("SELECT convertCurrency('USD', 'EUR', 100)")); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(exception).hasMessage("line 1:8: HogQL function convertCurrency requires an exchange-rate snapshot"); + } + + private static HogQlExchangeRateSnapshot snapshot(long generation) + { + return new HogQlExchangeRateSnapshot( + HogQlExchangeRateSnapshot.PROTOCOL_VERSION, + HogQlExchangeRateSnapshot.SCHEMA_VERSION, + generation, + HogQlExchangeRateSnapshot.BASE_CURRENCY, + HogQlExchangeRateSnapshot.DECIMAL_SCALE, + List.of(new ExchangeRate("USD", "1970-01-01", "10000000000"))); + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java new file mode 100644 index 000000000000..d00ec25238eb --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlFunctionResolver.java @@ -0,0 +1,816 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCapabilityDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionImplementation; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionRewrite; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionSignature; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.hogql.parser.HogQlParser; +import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.spi.Location; +import io.trino.spi.TrinoException; +import io.trino.sql.parser.SqlParser; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; + +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_RESOLUTION_ERROR; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowableOfType; + +public class TestHogQlFunctionResolver +{ + private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier("analytics", false); + + private final HogQlParser parser = new HogQlParser(); + private final SqlParser sqlParser = new SqlParser(); + + @Test + public void testMapsStockAndCompatibilityFunctionsToQualifiedTrinoNames() + { + HogQlQuery query = resolve( + "SELECT hogUpper('one'), hogCompat('two')", + function("hogUpper", FunctionKind.SCALAR, FunctionImplementation.STOCK, List.of("system", "builtin", "upper"), signature(1), false, false, false, false), + function("hogCompat", FunctionKind.SCALAR, FunctionImplementation.UDF, List.of("ducklake", "compat", "hog_compat"), signature(1), false, false, false, false)); + + assertThat(TrinoAstFactory.createStatement(query, Map.of())).isEqualTo(sqlParser.createStatement( + "SELECT system.builtin.upper('one'), ducklake.compat.hog_compat('two')")); + } + + @Test + public void testCompilerPinsOnceAndResolvesUserFunctions() + { + AtomicInteger pins = new AtomicInteger(); + HogQlSemanticCatalogSnapshot snapshot = snapshot(List.of( + function("hogUpper", FunctionKind.SCALAR, FunctionImplementation.STOCK, List.of("system", "builtin", "upper"), signature(1), false, false, false, false))); + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> { + pins.incrementAndGet(); + return new PinnedSnapshot(snapshot); + }); + + HogQlCompilationResult result = new HogQlCompiler().compile(envelope("SELECT [hogUpper('one')][1]"), Optional.of(context)); + + assertThat(pins).hasValue(1); + assertThat(result.catalogGeneration()).hasValue(7); + assertThat(result.statement()).isEqualTo(sqlParser.createStatement("SELECT ARRAY[system.builtin.upper('one')][1]")); + } + + @Test + public void testV0CatalogFunctionsCannotExpandFrozenRegistry() + { + HogQlSemanticCatalogSnapshot snapshot = snapshot(List.of( + function("manifestOnly", FunctionKind.SCALAR, FunctionImplementation.STOCK, List.of("upper"), signature(1), false, false, false, false))); + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(snapshot)); + + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> new HogQlCompiler().compileV0(envelope("SELECT manifestOnly('value')"), Optional.of(context))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(exception).hasMessage("line 1:8: Unknown HogQL function: manifestOnly"); + } + + @Test + public void testCompilerRewritesNullPredicatesAndPreservesPlaceholders() + { + HogQlSemanticCatalogSnapshot snapshot = snapshot(List.of( + rewriteFunction("hogIsNull", FunctionRewrite.IS_NULL), + rewriteFunction("hogIsNotNull", FunctionRewrite.IS_NOT_NULL))); + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(snapshot)); + + HogQlCompilationResult result = new HogQlCompiler().compile( + envelope( + "SELECT hogIsNull({first}), hogIsNotNull({second})", + Map.of( + "first", new HogQlTypedValue("varchar", new HogQlTypedValue.StringValue("one")), + "second", new HogQlTypedValue("varchar", new HogQlTypedValue.StringValue("two")))), + Optional.of(context)); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement("SELECT ? IS NULL, ? IS NOT NULL")); + assertThat(result.parameterNames()).containsExactly("first", "second"); + } + + @Test + public void testCompilerResolvesV0FunctionsWithoutCatalogContext() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT coalesce(NULL, 'fallback'), any(value), argMin(value, timestamp), " + + "arrayDistinct([1, 1]), dateDiff('day', start_time, end_time), date_diff('second', start_time, end_time), rank() OVER (), " + + "nullIf(value, ''), ifNull(value, 'fallback'), trim(value), round(score, 2), " + + "now(), current_timestamp(), isNotNull(value), groupArray(value), uniq(value), " + + "toInt(value), toFloat(value), toString(value), toDate(timestamp), toDateTime(timestamp), " + + "toStartOfMonth(timestamp), toStartOfDay(timestamp), toStartOfHour(timestamp), toMonday(timestamp), " + + "countIf(active), sumIf(value, active), maxIf(value, active), uniqIf(value, active), " + + "uniqExact(value), groupUniqArray(value), multiIf(first, 'one', second, 'two', 'other') FROM metrics")); + + assertThat(result.catalogGeneration()).isEmpty(); + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT coalesce(NULL, 'fallback'), arbitrary(value), min_by(value, timestamp), " + + "array_distinct(ARRAY[1, 1]), date_diff('day', start_time, end_time), date_diff('second', start_time, end_time), rank() OVER (), " + + "nullif(value, ''), coalesce(value, 'fallback'), \"trim\"(value), round(score, 2), " + + "now(), now(), value IS NOT NULL, array_agg(value), approx_distinct(value), " + + "CAST(value AS bigint), CAST(value AS double), CAST(value AS varchar), CAST(timestamp AS date), CAST(timestamp AS timestamp(0)), " + + "date_trunc('month', timestamp), date_trunc('day', timestamp), date_trunc('hour', timestamp), date_trunc('week', timestamp), " + + "count(*) FILTER (WHERE active), sum(value) FILTER (WHERE active), max(value) FILTER (WHERE active), " + + "approx_distinct(value) FILTER (WHERE active), count(DISTINCT value), array_agg(DISTINCT value), " + + "CASE WHEN first THEN 'one' WHEN second THEN 'two' ELSE 'other' END FROM metrics")); + } + + @Test + public void testToIntConvertsDateToUnixDayNumber() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT toInt(toDate('2022-01-01')), toInt(value) FROM metrics")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT date_diff('day', CAST('1970-01-01' AS date), CAST('2022-01-01' AS date)), CAST(value AS bigint) FROM metrics")); + } + + @Test + public void testCompilerResolvesExponentialAndWeekStartModes() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT exp(value), toStartOfWeek(timestamp, 1), toStartOfWeek(timestamp, 3) FROM metrics")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT exp(value), date_trunc('week', timestamp), date_trunc('week', timestamp) FROM metrics")); + } + + @Test + public void testCompilerRejectsUnsupportedWeekStartMode() + { + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> new HogQlCompiler().compile(envelope("SELECT toStartOfWeek(timestamp, 2) FROM metrics"))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_UNSUPPORTED_FEATURE.toErrorCode()); + assertThat(exception).hasMessageContaining("mode must be 0, 1, or 3"); + } + + @Test + public void testCompilerRejectsUnknownFunctionsWithoutCatalogContext() + { + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> new HogQlCompiler().compile(envelope("SELECT ordinary(1)"))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(exception).hasMessage("line 1:8: Unknown HogQL function: ordinary"); + } + + @Test + public void testCompilerLowersJsonExtractionFunctions() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT JSONExtractString(payload, 'name'), JSONExtractInt(payload, 'items', 0), " + + "JSONExtractFloat(payload, 'score'), JSONExtractRaw(payload, 'object'), " + + "JSONLength(payload), JSONLength(payload, 'items'), " + + "JSONExtract(payload_text, 'Map(String, Float64)'), " + + "JSONExtractKeysAndValues(payload_text, 'Float64'), JSONExtractKeysAndValuesRaw(payload_text) FROM records")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT coalesce(json_extract_scalar(payload, '$[\"name\"]'), ''), " + + "coalesce(TRY_CAST(json_extract_scalar(payload, '$[\"items\"][0]') AS bigint), 0), " + + "coalesce(TRY_CAST(json_extract_scalar(payload, '$[\"score\"]') AS double), 0E0), " + + "coalesce(json_format(json_extract(payload, '$[\"object\"]')), ''), " + + "coalesce(json_size(payload, '$'), 0), coalesce(json_size(payload, '$[\"items\"]'), 0), " + + "coalesce(TRY_CAST(json_parse(payload_text) AS map(varchar, double)), CAST(map(ARRAY[], ARRAY[]) AS map(varchar, double))), " + + "map_entries(coalesce(TRY_CAST(json_parse(payload_text) AS map(varchar, double)), CAST(map(ARRAY[], ARRAY[]) AS map(varchar, double)))), " + + "map_entries(transform_values(coalesce(TRY_CAST(json_parse(payload_text) AS map(varchar, json)), " + + "CAST(map(ARRAY[], ARRAY[]) AS map(varchar, json))), (key, value) -> json_format(value))) FROM records")); + + HogQlCompilationResult dynamicPathResult = new HogQlCompiler().compile(envelope( + "SELECT JSONExtractString(payload, dynamic_key), JSONExtractInt(payload, 'items', dynamic_index) FROM records")); + + assertThat(dynamicPathResult.statement()).isEqualTo(sqlParser.createStatement( + "SELECT coalesce(json_extract_scalar(payload, concat('$', '[', json_format(CAST(dynamic_key AS json)), ']')), ''), " + + "coalesce(TRY_CAST(json_extract_scalar(payload, concat('$[\"items\"]', '[', json_format(CAST(dynamic_index AS json)), ']')) AS bigint), 0) FROM records")); + + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> new HogQlCompiler().compile(envelope("SELECT JSONExtractString(payload, true)"))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_UNSUPPORTED_FEATURE.toErrorCode()); + assertThat(exception).hasMessageContaining("JSON path segments must be string or integer literals"); + } + + @Test + public void testCompilerLowersDateTimeCompatibilityFunctions() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT today(), toIntervalDay(2), addDays(timestamp, 3), subtractDays(timestamp, 3), addMonths(timestamp, -2), " + + "dateAdd(timestamp, INTERVAL 4 DAY), fromUnixTimestamp(epoch), toUnixTimestamp(timestamp), " + + "toDateTime(timestamp, 'UTC'), formatDateTime(timestamp, '%Y-%m-%d'), " + + "toTimeZone(timestamp, 'UTC'), parseDateTimeBestEffort(text) FROM records")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT CAST(now() AS date), 2 * INTERVAL '1' DAY, date_add('day', 3, timestamp), " + + "date_add('day', -(3), timestamp), date_add('month', -2, timestamp), timestamp + 4 * INTERVAL '1' DAY, " + + "from_unixtime(epoch), CAST(to_unixtime(timestamp) AS bigint), " + + "with_timezone(CAST(timestamp AS timestamp(0)), 'UTC'), date_format(timestamp, '%Y-%m-%d'), " + + "at_timezone(timestamp, 'UTC'), TRY_CAST(text AS timestamp(3)) FROM records")); + } + + @Test + public void testCompilerLowersScalarCompatibilityFunctions() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT not(active), and(active, ready), or(active, ready, pending), greater(score, 10), like(name, 'pro%'), " + + "least(first, second), greatest(first, second), position(name, 'needle'), " + + "startsWith(name, 'prefix'), substring(name, 2, 4), log10(score) FROM records")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT NOT active, active AND ready, (active OR ready) OR pending, score > 10, name LIKE 'pro%', " + + "least(first, second), greatest(first, second), strpos(name, 'needle'), " + + "starts_with(name, 'prefix'), substring(name, 2, 4), log10(score) FROM records")); + } + + @Test + public void testCompilerLowersRegexCompatibilityFunctions() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT extract(name, '([a-z]+)'), extract(name, '[a-z]+'), match(name, '^prefix'), " + + "replaceRegexpAll(name, '([a-z])', 'x_') FROM records")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT coalesce(regexp_extract(name, '([a-z]+)', 1), ''), " + + "coalesce(regexp_extract(name, '[a-z]+', 0), ''), regexp_like(name, '^prefix'), " + + "regexp_replace(name, '([a-z])', 'x_') FROM records")); + } + + @Test + public void testCompilerLowersExtendedRegexCompatibilityFunctions() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT extractAll(name, '([a-z]+)'), extractAll(name, '[a-z]+'), " + + "replaceRegexpOne(name, '([a-z])', 'x_') FROM records")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT regexp_extract_all(name, '([a-z]+)', 1), regexp_extract_all(name, '[a-z]+', 0), " + + "regexp_replace(name, '(?s)^(.*?)(([a-z]))', '$1x_') FROM records")); + } + + @Test + public void testCompilerLowersExtendedJsonCompatibilityFunctions() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT JSONExtractBool(payload, 'active'), JSONExtractUInt(payload, 'count'), " + + "JSONExtractArrayRaw(payload_text), JSONExtractArrayRaw(payload, 'items'), " + + "JSONExtractKeysAndValuesRaw(payload, 'object') FROM records")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT coalesce(TRY_CAST(json_extract_scalar(payload, '$[\"active\"]') AS boolean), false), " + + "coalesce(TRY_CAST(json_extract_scalar(payload, '$[\"count\"]') AS bigint), 0), " + + "transform(coalesce(TRY_CAST(json_parse(payload_text) AS array(json)), CAST(ARRAY[] AS array(json))), " + + "_hogql_json_item -> json_format(_hogql_json_item)), " + + "transform(coalesce(TRY_CAST(json_extract(payload, '$[\"items\"]') AS array(json)), CAST(ARRAY[] AS array(json))), " + + "_hogql_json_item -> json_format(_hogql_json_item)), " + + "map_entries(transform_values(coalesce(TRY_CAST(json_extract(payload, '$[\"object\"]') AS map(varchar, json)), " + + "CAST(map(ARRAY[], ARRAY[]) AS map(varchar, json))), (key, value) -> json_format(value))) FROM records")); + } + + @Test + public void testCompilerLowersAggregateCombinators() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT argMaxIf(value, timestamp, active), anyIf(value, active), minIf(value, active), " + + "avgIf(value, active), groupArrayIf(value, active), uniqExactIf(value, active), " + + "groupUniqArrayIf(value, active), countDistinct(value) FROM records")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT max_by(value, timestamp) FILTER (WHERE active), arbitrary(value) FILTER (WHERE active), " + + "min(value) FILTER (WHERE active), avg(value) FILTER (WHERE active), " + + "array_agg(value) FILTER (WHERE active), count(DISTINCT value) FILTER (WHERE active), " + + "array_agg(DISTINCT value) FILTER (WHERE active), count(DISTINCT value) FROM records")); + } + + @Test + public void testCompilerPreservesWindowsOnAggregateRewrites() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT countIf(active) OVER (PARTITION BY account_id), " + + "countDistinct(value) OVER (PARTITION BY account_id) FROM records")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT count(*) FILTER (WHERE active) OVER (PARTITION BY account_id), " + + "count(DISTINCT value) OVER (PARTITION BY account_id) FROM records")); + } + + @Test + public void testCompilerHoistsArrayJoinToCrossJoinUnnest() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT arrayJoin(items) AS item, upper(arrayJoin(names)) FROM records " + + "WHERE arrayJoin(flags) ORDER BY arrayJoin(scores)")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT __hogql_array_join_0.__hogql_value_0 AS item, upper(__hogql_array_join_1.__hogql_value_1) " + + "FROM records " + + "CROSS JOIN UNNEST(items) AS __hogql_array_join_0 (__hogql_value_0) " + + "CROSS JOIN UNNEST(names) AS __hogql_array_join_1 (__hogql_value_1) " + + "CROSS JOIN UNNEST(flags) AS __hogql_array_join_2 (__hogql_value_2) " + + "CROSS JOIN UNNEST(scores) AS __hogql_array_join_3 (__hogql_value_3) " + + "WHERE __hogql_array_join_2.__hogql_value_2 " + + "ORDER BY __hogql_array_join_3.__hogql_value_3")); + + assertThat(new HogQlCompiler().compile(envelope("SELECT arrayJoin([1, 2])")).statement()) + .isEqualTo(sqlParser.createStatement( + "SELECT __hogql_array_join_0.__hogql_value_0 " + + "FROM UNNEST(ARRAY[1, 2]) AS __hogql_array_join_0 (__hogql_value_0)")); + } + + @Test + public void testCompilerLowersNumericConversions() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT toFloatOrZero(text), toFloatOrDefault(text, 1), toDecimal(text, 4), intDiv(total, 1000), intDiv(-5, 2) FROM records")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT coalesce(TRY_CAST(text AS double), 0E0), " + + "coalesce(TRY_CAST(text AS double), CAST(1 AS double)), " + + "TRY_CAST(text AS decimal(18, 4)), " + + "CAST(total AS bigint) / CAST(1000 AS bigint) - if(CAST(total AS bigint) % CAST(1000 AS bigint) <> 0 AND " + + "(CAST(total AS bigint) < 0 AND CAST(1000 AS bigint) > 0 OR CAST(total AS bigint) > 0 AND CAST(1000 AS bigint) < 0), 1, 0), " + + "CAST(-5 AS bigint) / CAST(2 AS bigint) - if(CAST(-5 AS bigint) % CAST(2 AS bigint) <> 0 AND " + + "(CAST(-5 AS bigint) < 0 AND CAST(2 AS bigint) > 0 OR CAST(-5 AS bigint) > 0 AND CAST(2 AS bigint) < 0), 1, 0) FROM records")); + + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> new HogQlCompiler().compile(envelope("SELECT toDecimal(value, scale)"))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_UNSUPPORTED_FEATURE.toErrorCode()); + assertThat(exception).hasMessageContaining("decimal scale must be an integer literal"); + } + + @Test + public void testCompilerLowersArrayCompatibilityFunctions() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT arrayElement(items, -1), arrayFilter(x -> x > 0, items), arrayFirst(x -> x > 0, items), " + + "arrayMap(x -> x + 1, items), arraySum(items), range(3), range(0), range(2, 5), tupleElement(item, 2), " + + "splitByChar(',', text), has(items, 3), has(items, NULL) FROM records")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT element_at(items, -1), filter(items, x -> x > 0), element_at(filter(items, x -> x > 0), 1), " + + "transform(items, x -> x + 1), reduce(items, 0, (_hogql_sum, _hogql_item) -> _hogql_sum + _hogql_item, _hogql_sum -> _hogql_sum), " + + "if(3 <= 0, CAST(ARRAY[] AS array(bigint)), sequence(0, 3 - 1)), " + + "if(0 <= 0, CAST(ARRAY[] AS array(bigint)), sequence(0, 0 - 1)), " + + "if(5 <= 2, CAST(ARRAY[] AS array(bigint)), sequence(2, 5 - 1)), item[2], split(text, ','), " + + "if(3 IS NULL, any_match(items, _hogql_item -> _hogql_item IS NULL), coalesce(contains(items, 3), false)), " + + "if(NULL IS NULL, any_match(items, _hogql_item -> _hogql_item IS NULL), coalesce(contains(items, NULL), false)) FROM records")); + } + + @Test + public void testCompilerLowersArrayExtremaAndKeySorting() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT arrayMin(items), arraySort(item -> item.1, tuples) FROM records")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT array_min(items), array_sort(tuples, (__hogql_array_sort_left, __hogql_array_sort_right) -> " + + "CASE WHEN __hogql_array_sort_left[1] < __hogql_array_sort_right[1] THEN -(1) " + + "WHEN __hogql_array_sort_left[1] > __hogql_array_sort_right[1] THEN 1 ELSE 0 END) FROM records")); + } + + @Test + public void testCompilerLowersFinalStockCompatibilityFunctions() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT plus(total, surcharge), minus(total, discount), divide(total, count), notEquals(status, 'ignored'), " + + "greaterOrEquals(total, minimum), lessOrEquals(total, maximum), " + + "toMonth(timestamp), toYear(timestamp), toDayOfWeek(timestamp), ceil(score), _toInt16(value), cityHash64(value), " + + "formatReadableTimeDelta(duration), formatReadableTimeDelta(duration, 'days'), " + + "formatReadableTimeDelta(duration, 'hours', 'minutes'), " + + "JSONExtractKeys(payload), JSONExtractKeys(payload, 'nested') FROM records")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT total + surcharge, total - discount, total / count, status <> 'ignored', total >= minimum, total <= maximum, " + + "month(timestamp), year(timestamp), day_of_week(timestamp), ceiling(score), CAST(value AS smallint), cityhash64(value), " + + "format_readable_time_delta(duration), format_readable_time_delta(duration, 'days'), " + + "format_readable_time_delta(duration, 'hours', 'minutes'), " + + "map_keys(coalesce(TRY_CAST(json_parse(payload) AS map(varchar, json)), CAST(map(ARRAY[], ARRAY[]) AS map(varchar, json)))), " + + "map_keys(coalesce(TRY_CAST(json_extract(payload, '$[\"nested\"]') AS map(varchar, json)), CAST(map(ARRAY[], ARRAY[]) AS map(varchar, json)))) " + + "FROM records")); + } + + @Test + public void testCompilerLowersStaticSurveyResponse() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT getSurveyResponse(0, 'question-id'), getSurveyResponse(2, 'another-id') FROM events")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT coalesce(" + + "nullif(coalesce(json_extract_scalar(properties, '$[\"$survey_response_question-id\"]'), ''), ''), " + + "nullif(coalesce(json_extract_scalar(properties, '$[\"$survey_response\"]'), ''), '')), " + + "coalesce(" + + "nullif(coalesce(json_extract_scalar(properties, '$[\"$survey_response_another-id\"]'), ''), ''), " + + "nullif(coalesce(json_extract_scalar(properties, '$[\"$survey_response_2\"]'), ''), '')) " + + "FROM events")); + + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> new HogQlCompiler().compile(envelope("SELECT getSurveyResponse(question_index, 'question-id') FROM events"))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_UNSUPPORTED_FEATURE.toErrorCode()); + assertThat(exception).hasMessageContaining("question index must be an integer literal"); + } + + @Test + public void testCompilerLowersParametricAggregates() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT quantile(0.25)(value), quantileExact(0.5)(value), " + + "quantileIf(0.75)(value, active), groupArrayIf(20)(value, active), " + + "argMinIf(value, timestamp, active), replaceAll(name, 'old', 'new') FROM records")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT approx_percentile(value, 2.5E-1), approx_percentile(value, 5E-1), " + + "approx_percentile(value, 7.5E-1) FILTER (WHERE active), " + + "slice(array_agg(value) FILTER (WHERE active), 1, 20), " + + "min_by(value, timestamp) FILTER (WHERE active), replace(name, 'old', 'new') FROM records")); + } + + @Test + public void testCompilerLowersExpressionAndDateAliases() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT assumeNotNull(name), empty(name), notEmpty(name), equals(first, second), multiply(amount, 2), " + + "multiplyDecimal(amount, rate), divideDecimal(amount, rate), in(kind, ['a', 'b']), tuple(name, amount), " + + "subtractMonths(timestamp, 2), toIntervalMonth(3), toStartOfWeek(timestamp), " + + "splitByString('::', name), hasAny(first_array, second_array), parseDateTime(text, '%Y-%m-%d'), " + + "toLastDayOfMonth(timestamp), anyLast(name), lag(name, 2) OVER (), lagInFrame(name) OVER () FROM records")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT name, coalesce(hogql_empty(name), true), NOT coalesce(hogql_empty(name), true), " + + "first = second, amount * 2, amount * rate, amount / rate, contains(ARRAY['a', 'b'], kind), ROW(name, amount), " + + "date_add('month', -(2), timestamp), 3 * INTERVAL '1' MONTH, " + + "date_add('day', -1, date_trunc('week', date_add('day', 1, timestamp))), split(name, '::'), " + + "arrays_overlap(first_array, second_array), date_parse(text, '%Y-%m-%d'), last_day_of_month(timestamp), " + + "arbitrary(name), lag(name, 2) OVER (), lag(name) OVER () FROM records")); + } + + @Test + public void testCompilerLowersRemainingScalarAliases() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT arraySlice(items, 2, 3), arrayEnumerate(items), pow(2, exponent), substringUTF8(name, 2, 3), " + + "arrayConcat(first_array, second_array), subtractYears(timestamp, 2), toIntOrZero(text), toUUID(uuid_text), " + + "toJSONString(payload), JSONHas(payload, 'key'), JSON_VALUE(payload, '$.key'), md5(name), roundBankers(score, 2), " + + "and(first, second, third), JSONExtractString(payload), medianIf(score, active) FROM records")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT slice(items, 2, 3), " + + "if(cardinality(items) = 0, CAST(ARRAY[] AS array(bigint)), sequence(1, cardinality(items))), " + + "power(2, exponent), substring(name, 2, 3), concat(first_array, second_array), " + + "date_add('year', -(2), timestamp), coalesce(TRY_CAST(text AS bigint), 0), TRY_CAST(uuid_text AS uuid), " + + "json_format(CAST(payload AS json)), json_extract(payload, '$[\"key\"]') IS NOT NULL, " + + "json_extract_scalar(payload, '$.key'), md5(to_utf8(CAST(name AS varchar))), round(score, 2), " + + "(first AND second) AND third, coalesce(json_extract_scalar(payload, '$'), ''), " + + "approx_percentile(score, 5E-1) FILTER (WHERE active) FROM records")); + } + + @Test + public void testCompilerLowersFinalStockAliases() + { + HogQlCompilationResult result = new HogQlCompiler().compile(envelope( + "SELECT floor(score), toDayOfMonth(timestamp), mapFromArrays(keys, values_array), mapUpdate(existing, updates), " + + "map(), map('a', 1, 'b', 2), date_part('hour', timestamp), isNull(value) FROM records")); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT floor(score), day(timestamp), map(keys, values_array), map_concat(existing, updates), " + + "map(), map(ARRAY['a', 'b'], ARRAY[1, 2]), hour(timestamp), value IS NULL FROM records")); + + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> new HogQlCompiler().compile(envelope("SELECT map('a', 1, 'b')"))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(exception).hasMessageContaining("requires key/value argument pairs"); + } + + @Test + public void testCompilerEnforcesV0FunctionArities() + { + assertThat(new HogQlCompiler().compile(envelope("SELECT coalesce('value')")).statement()) + .isEqualTo(sqlParser.createStatement("SELECT 'value'")); + + for (String query : List.of("SELECT if(true, 1)", "SELECT concat('value')")) { + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> new HogQlCompiler().compile(envelope(query))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(exception).hasMessageContaining("does not accept"); + } + + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> new HogQlCompiler().compile(envelope("SELECT multiIf(first, 1, second, 2)"))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(exception).hasMessageContaining("requires condition/result pairs followed by a default value"); + } + + @ParameterizedTest + @MethodSource("nestedFunctionQueries") + public void testResolvesFunctionsThroughoutQueryTree(String query, String expected) + { + HogQlQuery resolved = resolve( + query, + function("hogUpper", FunctionKind.SCALAR, FunctionImplementation.STOCK, List.of("system", "builtin", "upper"), signature(1), false, false, false, false)); + + assertThat(TrinoAstFactory.createStatement(resolved, Map.of())).isEqualTo(sqlParser.createStatement(expected)); + } + + private static Stream nestedFunctionQueries() + { + return Stream.of( + Arguments.of( + "WITH cte AS (SELECT hogUpper('cte') AS value) " + + "SELECT hogUpper(value) FROM cte WHERE hogUpper(value) = 'x' " + + "GROUP BY hogUpper(value) HAVING hogUpper(value) = 'y' ORDER BY hogUpper(value)", + "WITH cte AS (SELECT system.builtin.upper('cte') AS value) " + + "SELECT system.builtin.upper(value) FROM cte WHERE system.builtin.upper(value) = 'x' " + + "GROUP BY system.builtin.upper(value) HAVING system.builtin.upper(value) = 'y' ORDER BY system.builtin.upper(value)"), + Arguments.of( + "SELECT hogUpper('left') UNION ALL SELECT hogUpper('right')", + "SELECT system.builtin.upper('left') UNION ALL SELECT system.builtin.upper('right')"), + Arguments.of( + "SELECT * FROM (VALUES (hogUpper('value')))", + "SELECT * FROM (VALUES (system.builtin.upper('value')))"), + Arguments.of( + "SELECT [hogUpper('value')][hogUpper('1')]", + "SELECT ARRAY[system.builtin.upper('value')][system.builtin.upper('1')]"), + Arguments.of( + "SELECT hogUpper('value').field", + "SELECT system.builtin.upper('value').field")); + } + + @Test + public void testAcceptsExactAndVariadicArities() + { + FunctionCapabilityDefinition exact = function("exact", FunctionKind.SCALAR, FunctionImplementation.STOCK, List.of("exact"), signature(2), false, false, false, false); + FunctionCapabilityDefinition variadic = function( + "variadic", + FunctionKind.SCALAR, + FunctionImplementation.STOCK, + List.of("variadic"), + new FunctionSignature(List.of("varchar", "varchar"), "varchar", true), + false, + false, + false, + false); + + resolve("SELECT exact('one', 'two'), variadic('one'), variadic('one', 'two', 'three')", exact, variadic); + } + + @Test + public void testMapsWindowFunctionsAndPreservesWindowSpecifications() + { + HogQlQuery query = resolve( + "SELECT hogRank(value) OVER (PARTITION BY team_id ORDER BY timestamp ROWS CURRENT ROW)", + function("hogRank", FunctionKind.WINDOW, FunctionImplementation.STOCK, List.of("analytics", "rank_value"), signature(1), false, false, false, true)); + + assertThat(TrinoAstFactory.createStatement(query, Map.of())).isEqualTo(sqlParser.createStatement( + "SELECT analytics.rank_value(value) OVER (PARTITION BY team_id ORDER BY timestamp ROWS CURRENT ROW)")); + } + + @Test + public void testRejectsCallsOutsideEveryDeclaredArity() + { + FunctionCapabilityDefinition function = function( + "exact", + FunctionKind.SCALAR, + FunctionImplementation.STOCK, + List.of("exact"), + List.of(signature(1), signature(3)), + false, + false, + false, + false); + + assertResolutionError("SELECT exact('one', 'two')", function, "HogQL function exact does not accept 2 arguments"); + assertResolutionError( + "SELECT hogIsNull('one', 'two')", + rewriteFunction("hogIsNull", FunctionRewrite.IS_NULL), + "HogQL function hogIsNull does not accept 2 arguments"); + } + + @ParameterizedTest + @MethodSource("unsupportedInvocationFeatures") + public void testRejectsUnsupportedInvocationFeatures( + String query, + FunctionCapabilityDefinition function, + String message) + { + assertUnsupportedError(query, function, message); + } + + private static Stream unsupportedInvocationFeatures() + { + return Stream.of( + Arguments.of( + "SELECT aggregate(DISTINCT value)", + function("aggregate", FunctionKind.AGGREGATE, FunctionImplementation.STOCK, List.of("aggregate"), signature(1), false, false, false, true), + "HogQL function aggregate does not support DISTINCT"), + Arguments.of( + "SELECT aggregate(value ORDER BY value)", + function("aggregate", FunctionKind.AGGREGATE, FunctionImplementation.STOCK, List.of("aggregate"), signature(1), true, false, false, true), + "HogQL function aggregate does not support ORDER BY"), + Arguments.of( + "SELECT aggregate(value) FILTER (WHERE true)", + function("aggregate", FunctionKind.AGGREGATE, FunctionImplementation.STOCK, List.of("aggregate"), signature(1), true, true, false, true), + "HogQL function aggregate does not support FILTER"), + Arguments.of( + "SELECT windowOnly(value)", + function("windowOnly", FunctionKind.WINDOW, FunctionImplementation.STOCK, List.of("window_only"), signature(1), false, false, false, true), + "HogQL window function windowOnly requires an OVER clause"), + Arguments.of( + "SELECT aggregate(value) OVER ()", + function("aggregate", FunctionKind.AGGREGATE, FunctionImplementation.STOCK, List.of("aggregate"), signature(1), false, false, false, false), + "HogQL function aggregate does not support OVER"), + Arguments.of( + "SELECT tableOnly(value)", + function("tableOnly", FunctionKind.TABLE, FunctionImplementation.STOCK, List.of("table_only"), signature(1), false, false, false, false), + "HogQL table function tableOnly cannot be used as an expression"), + Arguments.of( + "SELECT hogIsNull(DISTINCT value)", + rewriteFunction("hogIsNull", FunctionRewrite.IS_NULL), + "HogQL function hogIsNull does not support DISTINCT"), + Arguments.of( + "SELECT hogIsNull(value ORDER BY value)", + rewriteFunction("hogIsNull", FunctionRewrite.IS_NULL), + "HogQL function hogIsNull does not support ORDER BY"), + Arguments.of( + "SELECT hogIsNull(value) FILTER (WHERE true)", + rewriteFunction("hogIsNull", FunctionRewrite.IS_NULL), + "HogQL function hogIsNull does not support FILTER"), + Arguments.of( + "SELECT hogIsNull(value) OVER ()", + rewriteFunction("hogIsNull", FunctionRewrite.IS_NULL), + "HogQL function hogIsNull does not support OVER"), + Arguments.of( + "SELECT hogIsNull(value) OVER () IGNORE NULLS", + rewriteFunction("hogIsNull", FunctionRewrite.IS_NULL), + "HogQL function hogIsNull does not support null treatment")); + } + + @Test + public void testRejectsUnknownFunctionsAtCallLocation() + { + assertResolutionError("SELECT missing('one')", "Unknown HogQL function: missing"); + } + + private HogQlQuery resolve(String query, FunctionCapabilityDefinition... functions) + { + return HogQlFunctionResolver.resolve(new PinnedSnapshot(snapshot(List.of(functions))), parser.parseStatement(query)); + } + + private void assertResolutionError(String query, FunctionCapabilityDefinition function, String message) + { + TrinoException exception = catchThrowableOfType(TrinoException.class, () -> resolve(query, function)); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(exception.getLocation()).contains(new Location(1, 8)); + assertThat(exception).hasMessage("line 1:8: " + message); + } + + private void assertResolutionError(String query, String message) + { + TrinoException exception = catchThrowableOfType(TrinoException.class, () -> resolve(query)); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(exception.getLocation()).contains(new Location(1, 8)); + assertThat(exception).hasMessage("line 1:8: " + message); + } + + private void assertUnsupportedError(String query, FunctionCapabilityDefinition function, String message) + { + TrinoException exception = catchThrowableOfType(TrinoException.class, () -> resolve(query, function)); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_UNSUPPORTED_FEATURE.toErrorCode()); + assertThat(exception.getLocation()).contains(new Location(1, 8)); + assertThat(exception).hasMessage("line 1:8: " + message); + } + + private static FunctionCapabilityDefinition function( + String name, + FunctionKind kind, + FunctionImplementation implementation, + List trinoName, + FunctionSignature signature, + boolean supportsDistinct, + boolean supportsOrderBy, + boolean supportsFilter, + boolean supportsWindow) + { + return function(name, kind, implementation, trinoName, List.of(signature), supportsDistinct, supportsOrderBy, supportsFilter, supportsWindow); + } + + private static FunctionCapabilityDefinition function( + String name, + FunctionKind kind, + FunctionImplementation implementation, + List trinoName, + List signatures, + boolean supportsDistinct, + boolean supportsOrderBy, + boolean supportsFilter, + boolean supportsWindow) + { + return new FunctionCapabilityDefinition( + name, + kind, + implementation, + trinoName.stream().map(value -> new PhysicalIdentifier(value, false)).toList(), + signatures, + true, + supportsDistinct, + supportsOrderBy, + supportsFilter, + supportsWindow); + } + + private static FunctionSignature signature(int arity) + { + return new FunctionSignature(Stream.generate(() -> "varchar").limit(arity).toList(), "varchar", false); + } + + private static FunctionCapabilityDefinition rewriteFunction(String name, FunctionRewrite rewrite) + { + return new FunctionCapabilityDefinition( + name, + FunctionKind.SCALAR, + FunctionImplementation.REWRITE, + List.of(), + Optional.of(rewrite), + List.of(new FunctionSignature(List.of("varchar"), "boolean", false)), + true, + false, + false, + false, + false); + } + + private static HogQlSemanticCatalogSnapshot snapshot(List functions) + { + return new HogQlSemanticCatalogSnapshot( + 1, + 2, + HogQlLanguageContract.current().languageVersion(), + CATALOG, + 7, + List.of(), + List.of(), + List.of(), + List.of(), + List.of(), + functions, + List.of()); + } + + private static HogQlCompileEnvelope envelope(String query) + { + return envelope(query, Map.of()); + } + + private static HogQlCompileEnvelope envelope(String query, Map parameters) + { + return new HogQlCompileEnvelope( + query, + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + parameters, + Map.of(), + Map.of(), + Map.of(), + OptionalLong.empty()); + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlModifiers.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlModifiers.java new file mode 100644 index 000000000000..c642e4fae5cd --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlModifiers.java @@ -0,0 +1,196 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.HogQlTypedValue.BooleanValue; +import io.trino.hogql.compiler.HogQlTypedValue.StringValue; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogException; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralEncoding; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ModifierBehavior; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticModifierDefault; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.TypedLiteral; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.spi.ErrorCode; +import io.trino.spi.TrinoException; +import io.trino.sql.tree.Query; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Stream; + +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_BINDING_ERROR; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlModifiers +{ + private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier("analytics", false); + private static final SemanticModifierDefault SESSION_MODIFIER = modifier( + "sampling", + ModifierBehavior.TRINO_SESSION_PROPERTY, + List.of(new PhysicalIdentifier("hogql", false), new PhysicalIdentifier("sampling", false))); + private static final SemanticModifierDefault COMPILER_MODIFIER = modifier("compilerMode", ModifierBehavior.COMPILER, List.of()); + private static final SemanticModifierDefault NOOP_MODIFIER = modifier("legacyMode", ModifierBehavior.SAFE_NOOP, List.of()); + private static final SemanticModifierDefault UNSUPPORTED_MODIFIER = modifier("futureMode", ModifierBehavior.UNSUPPORTED, List.of()); + private static final HogQlSemanticCatalogSnapshot SNAPSHOT = snapshot(List.of( + SESSION_MODIFIER, + COMPILER_MODIFIER, + NOOP_MODIFIER, + UNSUPPORTED_MODIFIER)); + + @Test + public void testExplicitModifierPinsLiteralQueryAndOverridesDefault() + { + AtomicInteger pins = new AtomicInteger(); + HogQlCompilationResult result = new HogQlCompiler().compile( + envelope("SELECT 1", Map.of( + "SAMPLING", new HogQlTypedValue("BOOLEAN", new BooleanValue(true)), + "legacyMode", new HogQlTypedValue("boolean", new BooleanValue(true)))), + Optional.of(context(pins))); + + assertThat(pins).hasValue(1); + assertThat(result.catalogGeneration()).hasValue(7); + assertThat(result.modifierBindings()).hasSize(2); + assertThat(result.modifierBindings()).filteredOn(binding -> binding.modifierName().equals("sampling")).singleElement().satisfies(binding -> { + assertThat(binding.sessionProperty().orElseThrow()).extracting(PhysicalIdentifier::value).containsExactly("hogql", "sampling"); + assertThat(binding.value()).isEqualTo(new HogQlTypedValue("BOOLEAN", new BooleanValue(true))); + }); + assertThat(result.modifierBindings()).filteredOn(binding -> binding.modifierName().equals("legacyMode")).singleElement().satisfies(binding -> { + assertThat(binding.sessionProperty()).isEmpty(); + assertThat(binding.value()).isEqualTo(new HogQlTypedValue("boolean", new BooleanValue(true))); + }); + assertThat(((Query) result.statement()).getSessionProperties()).isEmpty(); + } + + @Test + public void testPinnedQueryAppliesSessionDefaultAndLeavesOtherDefaultsInert() + { + HogQlCompilationResult result = new HogQlCompiler().compile( + envelope("SELECT 1", Map.of("legacyMode", new HogQlTypedValue("boolean", new BooleanValue(true)))), + Optional.of(context(new AtomicInteger()))); + + assertThat(result.modifierBindings()).hasSize(2); + assertThat(result.modifierBindings()).filteredOn(binding -> binding.modifierName().equals("sampling")).singleElement() + .extracting(HogQlModifierBinding::value) + .isEqualTo(new HogQlTypedValue("boolean", new BooleanValue(false))); + } + + @Test + public void testUnmodifiedLiteralQueryDoesNotPinSnapshot() + { + AtomicInteger pins = new AtomicInteger(); + HogQlCompilationResult result = new HogQlCompiler().compile( + envelope("SELECT 1", Map.of()), + Optional.of(context(pins))); + + assertThat(pins).hasValue(0); + assertThat(result.catalogGeneration()).isEmpty(); + assertThat(result.modifierBindings()).isEmpty(); + } + + @Test + public void testExplicitModifierRequiresSnapshotContext() + { + assertThatThrownBy(() -> new HogQlCompiler().compile(envelope( + "SELECT 1", + Map.of("sampling", new HogQlTypedValue("boolean", new BooleanValue(true)))))) + .isInstanceOf(HogQlSemanticCatalogException.class) + .hasMessageContaining("required for modifiers"); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("invalidModifiers") + public void testRejectsInvalidExplicitModifiers(String name, Map modifiers, ErrorCode errorCode, String message) + { + assertThatThrownBy(() -> new HogQlCompiler().compile( + envelope("SELECT 1", modifiers), + Optional.of(context(new AtomicInteger())))) + .isInstanceOfSatisfying(TrinoException.class, exception -> { + assertThat(exception.getErrorCode()).isEqualTo(errorCode); + assertThat(exception).hasMessageContaining(message); + }); + } + + private static Stream invalidModifiers() + { + HogQlTypedValue booleanValue = new HogQlTypedValue("boolean", new BooleanValue(true)); + return Stream.of( + Arguments.of("unknown", Map.of("missing", booleanValue), HOGQL_BINDING_ERROR.toErrorCode(), "Unknown HogQL modifier"), + Arguments.of("wrong type", Map.of("sampling", new HogQlTypedValue("varchar", new StringValue("true"))), HOGQL_BINDING_ERROR.toErrorCode(), "incompatible type"), + Arguments.of("compiler", Map.of("compilerMode", booleanValue), HOGQL_UNSUPPORTED_FEATURE.toErrorCode(), "not implemented"), + Arguments.of("unsupported", Map.of("futureMode", booleanValue), HOGQL_UNSUPPORTED_FEATURE.toErrorCode(), "not supported"), + Arguments.of("duplicate canonical name", Map.of("sampling", booleanValue, "SAMPLING", booleanValue), HOGQL_BINDING_ERROR.toErrorCode(), "Duplicate HogQL modifier")); + } + + private static HogQlSemanticCatalogContext context(AtomicInteger pins) + { + return new HogQlSemanticCatalogContext(CATALOG, _ -> { + pins.incrementAndGet(); + return new PinnedSnapshot(SNAPSHOT); + }); + } + + private static HogQlCompileEnvelope envelope(String query, Map modifiers) + { + return new HogQlCompileEnvelope( + query, + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + Map.of(), + Map.of(), + Map.of(), + modifiers, + OptionalLong.empty()); + } + + private static SemanticModifierDefault modifier(String name, ModifierBehavior behavior, List sessionProperty) + { + return new SemanticModifierDefault( + name, + behavior, + new TypedLiteral("boolean", LiteralEncoding.BOOLEAN, "false"), + sessionProperty); + } + + private static HogQlSemanticCatalogSnapshot snapshot(List modifiers) + { + return new HogQlSemanticCatalogSnapshot( + 1, + 2, + HogQlLanguageContract.current().languageVersion(), + CATALOG, + 7, + List.of(), + List.of(), + List.of(), + List.of(), + List.of(), + List.of(), + modifiers, + List.of(), + List.of(), + List.of()); + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlProjectionDemand.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlProjectionDemand.java new file mode 100644 index 000000000000..bbc0ab106958 --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlProjectionDemand.java @@ -0,0 +1,38 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.parser.HogQlParser; +import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.Identifier; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TestHogQlProjectionDemand +{ + private final HogQlParser parser = new HogQlParser(); + + @Test + public void testJoinUsingDemandsTheSharedDerivedColumn() + { + HogQlQuery query = parser.parseStatement("SELECT sub.event FROM (SELECT * FROM events) sub JOIN persons USING (name)"); + + HogQlProjectionDemand.RequiredOutputs outputs = HogQlProjectionDemand.collect(query) + .forAlias(new Identifier("sub", false, query.span())); + + assertThat(outputs.includes("event")).isTrue(); + assertThat(outputs.includes("name")).isTrue(); + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlRelationshipExpansion.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlRelationshipExpansion.java new file mode 100644 index 000000000000..fee73f97a2cf --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlRelationshipExpansion.java @@ -0,0 +1,289 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ArgumentReferenceRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionArgument; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FieldReferenceRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.JoinKey; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LazyProjectionDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LazyTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralEncoding; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.OperatorRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyLookupRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyStorage; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipCardinality; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipJoinSide; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ScopedFieldReferenceRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticOperator; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.TypedLiteral; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.spi.TrinoException; +import io.trino.sql.parser.SqlParser; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.stream.Stream; + +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowableOfType; + +public class TestHogQlRelationshipExpansion +{ + private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier("analytics", false); + private static final HogQlSemanticCatalogSnapshot SNAPSHOT = snapshot(); + + private final HogQlCompiler compiler = new HogQlCompiler(); + private final SqlParser sqlParser = new SqlParser(); + + @Test + public void testExpandsRelationshipAndLazyProjectionThroughOneJoin() + { + HogQlCompilationResult result = compile("SELECT person.name, e.personProfile.name, e.personProfile.plan FROM events e"); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT \"__hogql_lazy_1\".full_name AS name, " + + "\"__hogql_lazy_1\".full_name AS name, " + + "CAST(\"__hogql_lazy_1\".properties_map[CAST(CAST('plan' AS varchar) AS varchar)] AS varchar) AS plan " + + "FROM analytics.data.raw_events e " + + "LEFT JOIN analytics.data.raw_persons \"__hogql_lazy_1\" " + + "ON e.person_id = \"__hogql_lazy_1\".person_id " + + "AND e.workspace_id = \"__hogql_lazy_1\".workspace_id")); + } + + @Test + public void testExpandsOnlyTheDeclaredPathThroughRelationshipCycle() + { + HogQlCompilationResult result = compile("SELECT personEvent.event FROM events"); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT \"__hogql_lazy_2\".event_name AS event " + + "FROM analytics.data.raw_events " + + "LEFT JOIN analytics.data.raw_persons \"__hogql_lazy_1\" " + + "ON raw_events.person_id = \"__hogql_lazy_1\".person_id " + + "AND raw_events.workspace_id = \"__hogql_lazy_1\".workspace_id " + + "LEFT JOIN analytics.data.raw_events \"__hogql_lazy_2\" " + + "ON \"__hogql_lazy_1\".event_id = \"__hogql_lazy_2\".event_id")); + } + + @Test + public void testExpandsJsonObjectPropertyLookup() + { + HogQlCompilationResult result = compile("SELECT jsonProperties.plan FROM persons"); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT CAST(CAST(json_parse(properties_json) AS map(varchar, json))[CAST('plan' AS varchar)] AS varchar) AS plan " + + "FROM analytics.data.raw_persons")); + } + + @Test + public void testPrunesUnusedLazyStarProjectionBeforeExpansion() + { + HogQlCompilationResult result = compile("SELECT sub.event FROM (SELECT e.personProfile.*, e.* FROM events e) sub"); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT sub.\"event\" AS event FROM (SELECT e.event_name AS \"event\" FROM analytics.data.raw_events e) sub")); + } + + @Test + public void testRetainsDemandedLazyStarProjectionAsStockJoin() + { + HogQlCompilationResult result = compile("SELECT sub.name FROM (SELECT e.personProfile.*, e.* FROM events e) sub"); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT sub.\"name\" AS name FROM (" + + "SELECT \"__hogql_lazy_1\".full_name AS \"name\" " + + "FROM analytics.data.raw_events e " + + "LEFT JOIN analytics.data.raw_persons \"__hogql_lazy_1\" " + + "ON e.person_id = \"__hogql_lazy_1\".person_id " + + "AND e.workspace_id = \"__hogql_lazy_1\".workspace_id) sub")); + } + + @ParameterizedTest + @MethodSource("nonProjectionRelationshipQueries") + public void testExpandsRelationshipDemandedOutsideProjection(String hogql, String trinoSql) + { + assertThat(compile(hogql).statement()).isEqualTo(sqlParser.createStatement(trinoSql)); + } + + @Test + public void testFailsExplicitlyWhenRelationshipIsDemandedInsideJoinCriteria() + { + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compile("SELECT e.event FROM events e JOIN persons p ON e.person.name = p.name")); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_UNSUPPORTED_FEATURE.toErrorCode()); + assertThat(exception).hasMessageContaining("HogQL relationship paths are not supported inside explicit join criteria"); + } + + private static Stream nonProjectionRelationshipQueries() + { + String join = "FROM analytics.data.raw_events " + + "LEFT JOIN analytics.data.raw_persons \"__hogql_lazy_1\" " + + "ON raw_events.person_id = \"__hogql_lazy_1\".person_id " + + "AND raw_events.workspace_id = \"__hogql_lazy_1\".workspace_id "; + return Stream.of( + Arguments.of( + "SELECT event FROM events WHERE person.name IS NOT NULL", + "SELECT event_name AS event " + join + "WHERE \"__hogql_lazy_1\".full_name IS NOT NULL"), + Arguments.of( + "SELECT event FROM events ORDER BY person.name", + "SELECT event_name AS event " + join + "ORDER BY \"__hogql_lazy_1\".full_name")); + } + + private HogQlCompilationResult compile(String query) + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + return compiler.compile(new HogQlCompileEnvelope( + query, + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + Map.of(), + Map.of(), + Map.of(), + Map.of(), + OptionalLong.of(7)), Optional.of(context)); + } + + private static HogQlSemanticCatalogSnapshot snapshot() + { + LogicalTableDefinition events = new LogicalTableDefinition( + "events", + physicalName("raw_events"), + List.of( + field("event", "event_name"), + field("eventId", "event_id"), + field("personId", "person_id"), + field("workspaceId", "workspace_id")), + List.of(), + List.of(new RelationshipDefinition( + "person", + "persons", + RelationshipCardinality.MANY_TO_ONE, + List.of(new JoinKey("personId", "personId")), + Optional.of(new OperatorRecipe( + SemanticOperator.EQUAL, + List.of( + new ScopedFieldReferenceRecipe(RelationshipJoinSide.SOURCE, "workspaceId"), + new ScopedFieldReferenceRecipe(RelationshipJoinSide.TARGET, "workspaceId"))))))); + LogicalTableDefinition persons = new LogicalTableDefinition( + "persons", + physicalName("raw_persons"), + List.of( + field("personId", "person_id"), + field("workspaceId", "workspace_id"), + field("eventId", "event_id"), + field("name", "full_name"), + new LogicalFieldDefinition("propertiesJson", new PhysicalIdentifier("properties_json", false), "varchar", LogicalType.STRING, true, false), + new LogicalFieldDefinition("propertiesMap", new PhysicalIdentifier("properties_map", false), "map(varchar, varchar)", LogicalType.MAP, true, false)), + List.of( + new PropertyDefinition( + "properties", + "propertiesMap", + PropertyStorage.MAP, + LogicalType.STRING, + true, + Optional.of("varchar"), + Optional.of("varchar"), + Optional.of(new OperatorRecipe( + SemanticOperator.SUBSCRIPT, + List.of( + new ArgumentReferenceRecipe(ExpressionArgument.PROPERTY_SOURCE), + new ArgumentReferenceRecipe(ExpressionArgument.PROPERTY_KEY))))), + new PropertyDefinition( + "jsonProperties", + "propertiesJson", + PropertyStorage.JSON_OBJECT, + LogicalType.STRING, + true, + Optional.of("varchar"), + Optional.of("varchar"), + Optional.of(new OperatorRecipe( + SemanticOperator.JSON_OBJECT_LOOKUP, + List.of( + new ArgumentReferenceRecipe(ExpressionArgument.PROPERTY_SOURCE), + new ArgumentReferenceRecipe(ExpressionArgument.PROPERTY_KEY)))))), + List.of(new RelationshipDefinition( + "event", + "events", + RelationshipCardinality.MANY_TO_ONE, + List.of(new JoinKey("eventId", "eventId"))))); + List lazyTables = List.of( + new LazyTableDefinition( + "events", + "personProfile", + List.of("person"), + List.of( + new LazyProjectionDefinition("name", "varchar", LogicalType.STRING, true, true, new FieldReferenceRecipe("persons", "name")), + new LazyProjectionDefinition( + "plan", + "varchar", + LogicalType.STRING, + true, + true, + new PropertyLookupRecipe( + "persons", + "properties", + new LiteralRecipe(new TypedLiteral("varchar", LiteralEncoding.STRING, "plan")))))), + new LazyTableDefinition( + "events", + "personEvent", + List.of("person", "event"), + List.of(new LazyProjectionDefinition("event", "varchar", LogicalType.STRING, true, true, new FieldReferenceRecipe("events", "event"))))); + return new HogQlSemanticCatalogSnapshot( + 1, + 2, + HogQlLanguageContract.current().languageVersion(), + CATALOG, + 7, + List.of(events, persons), + List.of(), + List.of(), + List.of(), + List.of(), + List.of(), + List.of(), + lazyTables, + List.of(), + List.of()); + } + + private static LogicalFieldDefinition field(String name, String physicalName) + { + return new LogicalFieldDefinition(name, new PhysicalIdentifier(physicalName, false), "varchar", LogicalType.STRING, false, true); + } + + private static PhysicalQualifiedName physicalName(String table) + { + return new PhysicalQualifiedName(CATALOG, new PhysicalIdentifier("data", false), new PhysicalIdentifier(table, false)); + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlScopedPlaceholders.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlScopedPlaceholders.java new file mode 100644 index 000000000000..54d14930f6c8 --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlScopedPlaceholders.java @@ -0,0 +1,84 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.HogQlTypedValue.NumberValue; +import io.trino.hogql.compiler.HogQlTypedValue.StringValue; +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.spi.TrinoException; +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.OptionalLong; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowableOfType; + +public class TestHogQlScopedPlaceholders +{ + private final HogQlCompiler compiler = new HogQlCompiler(); + + @Test + public void testBindsVariablesAndFiltersWithoutInterpolation() + { + HogQlCompilationResult result = compiler.compile(envelope( + "SELECT {variables.organization_id}, {filters.date_from}, {variables.organization_id}", + Map.of("organization_id", new HogQlTypedValue("bigint", new NumberValue("42"))), + Map.of("date_from", new HogQlTypedValue("varchar", new StringValue("2026-01-01"))))); + + assertThat(result.parameterNames()).containsExactly( + "variables.organization_id", + "filters.date_from", + "variables.organization_id"); + assertThat(result.statement().toString()).doesNotContain("42", "2026-01-01"); + } + + @Test + public void testReportsMissingScopedBinding() + { + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile(envelope("SELECT {variables.organization_id}", Map.of(), Map.of()))); + + assertThat(exception.getErrorCode()).isEqualTo(HogQlErrorCode.HOGQL_BINDING_ERROR.toErrorCode()); + assertThat(exception).hasMessageContaining("Missing HogQL parameter bindings: variables.organization_id"); + } + + @Test + public void testLeavesOtherDottedPlaceholderExpressionsUnsupported() + { + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile(envelope("SELECT {payload.organization_id}", Map.of(), Map.of()))); + + assertThat(exception.getErrorCode()).isEqualTo(HogQlErrorCode.HOGQL_SYNTAX_ERROR.toErrorCode()); + assertThat(exception).hasMessageContaining("non-name placeholder"); + } + + private static HogQlCompileEnvelope envelope( + String query, + Map variables, + Map filters) + { + return new HogQlCompileEnvelope( + query, + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + Map.of(), + variables, + filters, + Map.of(), + OptionalLong.empty()); + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticEntityExpansion.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticEntityExpansion.java new file mode 100644 index 000000000000..5745a8c39d1c --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticEntityExpansion.java @@ -0,0 +1,246 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ActionReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.CohortReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FieldReferenceRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralEncoding; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.MaterializedViewReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.OperatorRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PredicateRepresentation; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ReferencedField; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationMembershipRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationMembershipRepresentation; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticOperator; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.TypedLiteral; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.spi.TrinoException; +import io.trino.sql.parser.SqlParser; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.stream.Stream; + +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_RESOLUTION_ERROR; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowableOfType; + +public class TestHogQlSemanticEntityExpansion +{ + private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier("analytics", false); + private static final HogQlSemanticCatalogSnapshot SNAPSHOT = snapshot(); + + private final HogQlCompiler compiler = new HogQlCompiler(); + private final SqlParser sqlParser = new SqlParser(); + + @ParameterizedTest + @ValueSource(strings = {"'Paid event'", "17"}) + public void testExpandsActionPredicateByNameOrId(String action) + { + assertThat(compile("SELECT event FROM events e WHERE matchesAction(" + action + ")").statement()) + .isEqualTo(sqlParser.createStatement( + "SELECT event_name AS event FROM analytics.data.raw_events e " + + "WHERE e.event_name = CAST('purchase' AS varchar)")); + } + + @ParameterizedTest + @ValueSource(strings = {"IN COHORT 23", "NOT IN COHORT 'Active people'"}) + public void testExpandsCohortMembershipAsInSubquery(String predicate) + { + assertThat(compile("SELECT event FROM events WHERE personId " + predicate).statement()) + .isEqualTo(sqlParser.createStatement( + "SELECT event_name AS event FROM analytics.data.raw_events " + + "WHERE person_id " + (predicate.startsWith("NOT") ? "NOT " : "") + + "IN (SELECT DISTINCT \"personId\" FROM analytics.data.active_people)")); + } + + @Test + public void testExpandsActionRelationMembership() + { + assertThat(compile("SELECT event FROM events WHERE matchesAction(18)").statement()) + .isEqualTo(sqlParser.createStatement( + "SELECT event_name AS event FROM analytics.data.raw_events " + + "WHERE person_id IN (SELECT DISTINCT \"personId\" FROM analytics.data.active_people)")); + } + + @Test + public void testV0ExpandsActionPredicate() + { + assertThat(compileV0("SELECT event FROM events WHERE matchesAction(17)").statement()) + .isEqualTo(sqlParser.createStatement( + "SELECT event_name AS event FROM analytics.data.raw_events " + + "WHERE raw_events.event_name = CAST('purchase' AS varchar)")); + } + + @Test + public void testExpandsCohortPredicate() + { + assertThat(compile("SELECT event FROM events WHERE event NOT IN COHORT 24").statement()) + .isEqualTo(sqlParser.createStatement( + "SELECT event_name AS event FROM analytics.data.raw_events " + + "WHERE NOT (raw_events.event_name = CAST('purchase' AS varchar))")); + } + + @Test + public void testRejectsUnknownActionWithTypedResolutionError() + { + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compile("SELECT event FROM events WHERE matchesAction('Missing action')")); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(exception).hasMessageContaining("Unknown HogQL action"); + } + + @ParameterizedTest + @MethodSource("invalidCohortQueries") + public void testRejectsInvalidCohortUseWithTypedCompatibilityError(String query, String message) + { + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compile(query)); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_UNSUPPORTED_FEATURE.toErrorCode()); + assertThat(exception).hasMessageContaining(message); + } + + private static Stream invalidCohortQueries() + { + return Stream.of( + Arguments.of( + "SELECT event FROM events WHERE personId IN COHORT event", + "HogQL cohort reference must be a string or integer literal"), + Arguments.of( + "SELECT event FROM events WHERE event IN COHORT 23", + "HogQL cohort membership source does not match the catalog")); + } + + private HogQlCompilationResult compile(String query) + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + return compiler.compile(new HogQlCompileEnvelope( + query, + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + Map.of(), + Map.of(), + Map.of(), + Map.of(), + OptionalLong.of(11)), Optional.of(context)); + } + + private HogQlCompilationResult compileV0(String query) + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + return compiler.compileV0(new HogQlCompileEnvelope( + query, + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + Map.of(), + Map.of(), + Map.of(), + Map.of(), + OptionalLong.of(11)), Optional.of(context)); + } + + private static HogQlSemanticCatalogSnapshot snapshot() + { + LogicalTableDefinition events = new LogicalTableDefinition( + "events", + physicalName("raw_events"), + List.of( + field("event", "event_name"), + field("personId", "person_id")), + List.of(), + List.of()); + MaterializedViewReference activePeople = new MaterializedViewReference( + "active_people", + physicalName("active_people"), + List.of(new ReferencedField("personId", "varchar", LogicalType.STRING, false, true))); + PredicateRepresentation purchasePredicate = new PredicateRepresentation(new OperatorRecipe( + SemanticOperator.EQUAL, + List.of( + new FieldReferenceRecipe("events", "event"), + new LiteralRecipe(new TypedLiteral("varchar", LiteralEncoding.STRING, "purchase"))))); + RelationMembershipRepresentation activeMembership = new RelationMembershipRepresentation(new RelationMembershipRecipe( + new RelationReference(RelationKind.MATERIALIZED_VIEW, "active_people"), + "personId", + "personId")); + ActionReference paidEvent = new ActionReference( + "Paid event", + "17", + "events", + purchasePredicate); + ActionReference precomputedAction = new ActionReference( + "Precomputed action", + "18", + "events", + activeMembership); + CohortReference activePeopleCohort = new CohortReference( + "Active people", + "23", + "events", + activeMembership); + CohortReference purchasers = new CohortReference( + "Purchasers", + "24", + "events", + purchasePredicate); + return new HogQlSemanticCatalogSnapshot( + 1, + 2, + HogQlLanguageContract.current().languageVersion(), + CATALOG, + 11, + List.of(events), + List.of(), + List.of(), + List.of(), + List.of(activePeople), + List.of(), + List.of(), + List.of(), + List.of(paidEvent, precomputedAction), + List.of(activePeopleCohort, purchasers)); + } + + private static LogicalFieldDefinition field(String name, String physicalName) + { + return new LogicalFieldDefinition(name, new PhysicalIdentifier(physicalName, false), "varchar", LogicalType.STRING, false, true); + } + + private static PhysicalQualifiedName physicalName(String table) + { + return new PhysicalQualifiedName(CATALOG, new PhysicalIdentifier("data", false), new PhysicalIdentifier(table, false)); + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticExpansion.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticExpansion.java new file mode 100644 index 000000000000..08ca77b2e2d4 --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticExpansion.java @@ -0,0 +1,366 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ArgumentReferenceRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.CastRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionArgument; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FieldReferenceRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCallRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCapabilityDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionImplementation; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionSignature; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralEncoding; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.MaterializedViewReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.OperatorRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyLookupRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyStorage; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ReferencedField; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SavedQueryReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticOperator; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.TypedLiteral; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.VirtualProjection; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.VirtualTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.spi.Location; +import io.trino.spi.TrinoException; +import io.trino.sql.parser.SqlParser; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.atomic.AtomicInteger; + +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_COMPILER_LIMIT_EXCEEDED; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowableOfType; + +public class TestHogQlSemanticExpansion +{ + private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier("analytics", false); + private static final HogQlSemanticCatalogSnapshot SNAPSHOT = snapshot(); + + private final HogQlCompiler compiler = new HogQlCompiler(); + private final SqlParser sqlParser = new SqlParser(); + + @Test + public void testExpandsEveryExpressionRecipeToStockAst() + { + HogQlCompilationResult result = compile("SELECT constant, upperEvent, added, castEvent, missingEvent FROM events"); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT CAST(7 AS bigint) AS constant, " + + "system.builtin.upper(event_name) AS upperEvent, " + + "CAST(7 AS bigint) + CAST(1 AS bigint) AS added, " + + "CAST(event_name AS varchar) AS castEvent, " + + "event_name IS NULL AS missingEvent " + + "FROM analytics.data.raw_events")); + } + + @Test + public void testPreservesCatalogProvidedTrinoCastTypes() + { + HogQlCompilationResult result = compile("SELECT castReal, castInteger, castBigint FROM events"); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT CAST(event_name AS real) AS castReal, " + + "CAST(event_name AS integer) AS castInteger, " + + "CAST(event_name AS bigint) AS castBigint " + + "FROM analytics.data.raw_events")); + } + + @Test + public void testExpandsVirtualSavedAndMaterializedRelationsWithDeclaredOutputs() + { + assertThat(compile("SELECT * FROM event_view").statement()).isEqualTo(sqlParser.createStatement( + "SELECT \"eventTitle\" AS \"eventTitle\", \"constant\" AS \"constant\" " + + "FROM (SELECT system.builtin.upper(event_name) AS \"eventTitle\", CAST(7 AS bigint) AS \"constant\" " + + "FROM analytics.data.raw_events)")); + assertThat(compile("SELECT * FROM saved_view").statement()).isEqualTo(sqlParser.createStatement( + "SELECT \"eventTitle\" AS \"eventTitle\" FROM (" + + "SELECT \"eventTitle\" AS \"eventTitle\" FROM (" + + "SELECT system.builtin.upper(event_name) AS \"eventTitle\", CAST(7 AS bigint) AS \"constant\" " + + "FROM analytics.data.raw_events))")); + assertThat(compile("SELECT * FROM daily_events").statement()).isEqualTo(sqlParser.createStatement( + "SELECT \"day\" AS \"day\" FROM analytics.data.daily_events")); + } + + @Test + public void testPinsOnceAcrossRecursiveExpansionAndPreservesPlaceholderOrder() + { + AtomicInteger pins = new AtomicInteger(); + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> { + pins.incrementAndGet(); + return new PinnedSnapshot(SNAPSHOT); + }); + HogQlCompileEnvelope envelope = new HogQlCompileEnvelope( + "SELECT {first}, eventTitle, {second} FROM saved_view", + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + Map.of( + "first", new HogQlTypedValue("bigint", new HogQlTypedValue.NumberValue("1")), + "second", new HogQlTypedValue("bigint", new HogQlTypedValue.NumberValue("2"))), + Map.of(), + Map.of(), + Map.of(), + OptionalLong.of(7)); + + HogQlCompilationResult result = compiler.compile(envelope, Optional.of(context)); + + assertThat(pins).hasValue(1); + assertThat(result.catalogGeneration()).hasValue(7); + assertThat(result.parameterNames()).containsExactly("first", "second"); + } + + @Test + public void testExpandsStaticAndQualifiedPropertyAccessThroughDeclaredRecipe() + { + HogQlCompilationResult result = compile("SELECT properties.browser, e.properties.browser, (properties).browser FROM events e"); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT CAST(event_properties_blob[CAST('browser' AS varchar)] AS varchar) AS browser, " + + "CAST(e.event_properties_blob[CAST('browser' AS varchar)] AS varchar) AS browser, " + + "CAST(event_properties_blob[CAST('browser' AS varchar)] AS varchar) " + + "FROM analytics.data.raw_events e")); + } + + @Test + public void testExpandsDynamicPropertyAccessAndPreservesPlaceholder() + { + HogQlCompilationResult result = compile( + "SELECT properties[{key}] FROM events", + SNAPSHOT, + Map.of("key", new HogQlTypedValue("varchar", new HogQlTypedValue.StringValue("browser")))); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT CAST(event_properties_blob[CAST(? AS varchar)] AS varchar) FROM analytics.data.raw_events")); + assertThat(result.parameterNames()).containsExactly("key"); + } + + @Test + public void testExpandsPropertyLookupRecipeThroughDeclaredPropertyRecipe() + { + HogQlCompilationResult result = compile("SELECT browserProperty FROM events"); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT CAST(event_properties_blob[CAST(CAST('browser' AS varchar) AS varchar)] AS varchar) AS browserProperty " + + "FROM analytics.data.raw_events")); + } + + @Test + public void testRejectsPropertyAccessWithoutDeclaredRecipeAtOriginalLocation() + { + PropertyDefinition property = new PropertyDefinition( + "properties", + "eventProperties", + PropertyStorage.JSON_OBJECT, + LogicalType.STRING, + true); + + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compile("SELECT properties.browser FROM events", propertySnapshot(property, List.of()))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_UNSUPPORTED_FEATURE.toErrorCode()); + assertThat(exception.getLocation()).contains(new Location(1, 8)); + assertThat(exception).hasMessage("line 1:8: HogQL property lookup has no declared compiler recipe"); + } + + @Test + public void testBoundsPropertyLookupExpansionAtOriginalLocation() + { + List expressions = new ArrayList<>(); + expressions.add(expression("branch0", new LiteralRecipe(new TypedLiteral("bigint", LiteralEncoding.INTEGER, "1")))); + for (int index = 1; index <= 14; index++) { + String previous = "branch" + (index - 1); + expressions.add(expression("branch" + index, new OperatorRecipe(SemanticOperator.ADD, List.of( + new FieldReferenceRecipe("events", previous), + new FieldReferenceRecipe("events", previous))))); + } + expressions.add(expression("lookupBranch", new PropertyLookupRecipe( + "events", + "properties", + new FieldReferenceRecipe("events", "branch14")))); + HogQlSemanticCatalogSnapshot expansiveSnapshot = propertySnapshot(propertiesProperty(), expressions); + + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compile("SELECT lookupBranch FROM events", expansiveSnapshot)); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_COMPILER_LIMIT_EXCEEDED.toErrorCode()); + assertThat(exception.getLocation()).contains(new Location(1, 8)); + assertThat(exception).hasMessage("line 1:8: HogQL semantic expansion exceeded node limit"); + } + + private HogQlCompilationResult compile(String query) + { + return compile(query, SNAPSHOT); + } + + private HogQlCompilationResult compile(String query, HogQlSemanticCatalogSnapshot snapshot) + { + return compile(query, snapshot, Map.of()); + } + + private HogQlCompilationResult compile(String query, HogQlSemanticCatalogSnapshot snapshot, Map parameters) + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(snapshot)); + return compiler.compile(new HogQlCompileEnvelope( + query, + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + parameters, + Map.of(), + Map.of(), + Map.of(), + OptionalLong.of(7)), Optional.of(context)); + } + + private static HogQlSemanticCatalogSnapshot snapshot() + { + LogicalFieldDefinition event = new LogicalFieldDefinition("event", new PhysicalIdentifier("event_name", false), "varchar", LogicalType.STRING, false, true); + LogicalFieldDefinition properties = new LogicalFieldDefinition("eventProperties", new PhysicalIdentifier("event_properties_blob", false), "map(varchar, varchar)", LogicalType.MAP, true, false); + List expressions = List.of( + expression("constant", new LiteralRecipe(new TypedLiteral("bigint", LiteralEncoding.INTEGER, "7"))), + expression("upperEvent", new FunctionCallRecipe("hogUpper", List.of(new FieldReferenceRecipe("events", "event")))), + expression("added", new OperatorRecipe(SemanticOperator.ADD, List.of( + new FieldReferenceRecipe("events", "constant"), + new LiteralRecipe(new TypedLiteral("bigint", LiteralEncoding.INTEGER, "1"))))), + expression("castEvent", new CastRecipe(new FieldReferenceRecipe("events", "event"), "varchar")), + expression("castReal", new CastRecipe(new FieldReferenceRecipe("events", "event"), "real")), + expression("castInteger", new CastRecipe(new FieldReferenceRecipe("events", "event"), "integer")), + expression("castBigint", new CastRecipe(new FieldReferenceRecipe("events", "event"), "bigint")), + expression("missingEvent", new OperatorRecipe(SemanticOperator.IS_NULL, List.of(new FieldReferenceRecipe("events", "event")))), + expression("browserProperty", new PropertyLookupRecipe( + "events", + "properties", + new LiteralRecipe(new TypedLiteral("varchar", LiteralEncoding.STRING, "browser"))))); + ReferencedField eventTitle = new ReferencedField("eventTitle", "varchar", LogicalType.STRING, false, true); + return new HogQlSemanticCatalogSnapshot( + 1, + 2, + HogQlLanguageContract.current().languageVersion(), + CATALOG, + 7, + List.of(new LogicalTableDefinition( + "events", + physicalName("raw_events"), + List.of(event, properties), + List.of(propertiesProperty()), + List.of())), + expressions, + List.of(new VirtualTableDefinition( + "event_view", + new RelationReference(RelationKind.LOGICAL_TABLE, "events"), + List.of( + new VirtualProjection("eventTitle", "upperEvent", true), + new VirtualProjection("constant", "constant", true)))), + List.of(new SavedQueryReference( + "saved_view", + "saved-7", + new RelationReference(RelationKind.VIRTUAL_TABLE, "event_view"), + List.of(eventTitle))), + List.of(new MaterializedViewReference( + "daily_events", + physicalName("daily_events"), + List.of(new ReferencedField("day", "date", LogicalType.DATE, false, true)))), + List.of(new FunctionCapabilityDefinition( + "hogUpper", + FunctionKind.SCALAR, + FunctionImplementation.STOCK, + List.of( + new PhysicalIdentifier("system", false), + new PhysicalIdentifier("builtin", false), + new PhysicalIdentifier("upper", false)), + List.of(new FunctionSignature(List.of("varchar"), "varchar", false)), + true, + false, + false, + false, + false)), + List.of()); + } + + private static HogQlSemanticCatalogSnapshot propertySnapshot(PropertyDefinition property, List expressions) + { + return new HogQlSemanticCatalogSnapshot( + 1, + 2, + HogQlLanguageContract.current().languageVersion(), + CATALOG, + 7, + List.of(new LogicalTableDefinition( + "events", + physicalName("raw_events"), + List.of(new LogicalFieldDefinition( + "eventProperties", + new PhysicalIdentifier("event_properties_blob", false), + "map(varchar, varchar)", + LogicalType.MAP, + true, + false)), + List.of(property), + List.of())), + expressions, + List.of(), + List.of(), + List.of(), + List.of(), + List.of()); + } + + private static PropertyDefinition propertiesProperty() + { + return new PropertyDefinition( + "properties", + "eventProperties", + PropertyStorage.JSON_OBJECT, + LogicalType.STRING, + true, + Optional.of("varchar"), + Optional.of("varchar"), + Optional.of(new OperatorRecipe( + SemanticOperator.SUBSCRIPT, + List.of( + new ArgumentReferenceRecipe(ExpressionArgument.PROPERTY_SOURCE), + new ArgumentReferenceRecipe(ExpressionArgument.PROPERTY_KEY))))); + } + + private static ExpressionFieldDefinition expression(String name, HogQlSemanticCatalogSnapshot.ExpressionRecipe recipe) + { + return new ExpressionFieldDefinition("events", name, "bigint", LogicalType.INTEGER, false, true, recipe); + } + + private static PhysicalQualifiedName physicalName(String table) + { + return new PhysicalQualifiedName(CATALOG, new PhysicalIdentifier("data", false), new PhysicalIdentifier(table, false)); + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java new file mode 100644 index 000000000000..96091de667d4 --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlSemanticResolution.java @@ -0,0 +1,787 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCapabilityDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionImplementation; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionSignature; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinRequest; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.spi.Location; +import io.trino.spi.TrinoException; +import io.trino.sql.parser.SqlParser; +import io.trino.sql.tree.Identifier; +import io.trino.sql.tree.Query; +import io.trino.sql.tree.QuerySpecification; +import io.trino.sql.tree.SingleColumn; +import io.trino.sql.tree.Table; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_RESOLUTION_ERROR; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowableOfType; + +public class TestHogQlSemanticResolution +{ + private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier("analytics", false); + private static final HogQlSemanticCatalogSnapshot SNAPSHOT = new HogQlSemanticCatalogSnapshot( + 1, + 2, + HogQlLanguageContract.current().languageVersion(), + CATALOG, + 7, + List.of( + new LogicalTableDefinition( + "events", + new PhysicalQualifiedName( + CATALOG, + new PhysicalIdentifier("Hog Data", true), + new PhysicalIdentifier("raw-events", true)), + List.of( + new LogicalFieldDefinition("event", new PhysicalIdentifier("event_name", false), "varchar", LogicalType.STRING, false, true), + new LogicalFieldDefinition("personId", new PhysicalIdentifier("Person ID", true), "varchar", LogicalType.STRING, false, true), + new LogicalFieldDefinition("hidden", new PhysicalIdentifier("hidden", false), "bigint", LogicalType.INTEGER, true, false)), + List.of(), + List.of()), + new LogicalTableDefinition( + "persons", + new PhysicalQualifiedName( + CATALOG, + new PhysicalIdentifier("Hog Data", true), + new PhysicalIdentifier("raw-persons", true)), + List.of( + new LogicalFieldDefinition("personId", new PhysicalIdentifier("person_id", false), "varchar", LogicalType.STRING, false, true), + new LogicalFieldDefinition("name", new PhysicalIdentifier("full_name", false), "varchar", LogicalType.STRING, false, true)), + List.of(), + List.of())), + List.of(), + List.of(), + List.of(), + List.of(), + List.of(new FunctionCapabilityDefinition( + "count", + FunctionKind.AGGREGATE, + FunctionImplementation.STOCK, + List.of(new PhysicalIdentifier("count", false)), + List.of(new FunctionSignature(List.of(), "bigint", false)), + true, + true, + true, + true, + true)), + List.of()); + + private final HogQlCompiler compiler = new HogQlCompiler(); + private final SqlParser sqlParser = new SqlParser(); + + @Test + public void testPinsAndResolvesLogicalTableFieldsAndStar() + { + AtomicReference request = new AtomicReference<>(); + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, pinRequest -> { + request.set(pinRequest); + return new PinnedSnapshot(SNAPSHOT); + }); + + HogQlCompilationResult result = compiler.compile( + envelope("SELECT *, event FROM events WHERE personId = 'synthetic' GROUP BY event HAVING count(*) > 0 ORDER BY event", OptionalLong.of(7)), + Optional.of(context)); + + assertThat(request.get()).isEqualTo(new PinRequest(CATALOG, HogQlLanguageContract.current().languageVersion(), OptionalLong.of(7))); + assertThat(result.catalogGeneration()).hasValue(7); + QuerySpecification query = querySpecification(result); + assertThat(((Table) query.getFrom().orElseThrow()).getName().getOriginalParts()) + .extracting(Identifier::getValue) + .containsExactly("analytics", "Hog Data", "raw-events"); + assertThat(((Table) query.getFrom().orElseThrow()).getName().getOriginalParts()) + .extracting(Identifier::isDelimited) + .containsExactly(false, true, true); + assertThat(query.getSelect().getSelectItems()).hasSize(3); + assertThat(query.getSelect().getSelectItems()) + .allSatisfy(item -> assertThat(item).isInstanceOf(SingleColumn.class)); + assertThat(query.getSelect().getSelectItems().subList(0, 2)) + .extracting(item -> ((SingleColumn) item).getAlias().orElseThrow().getValue()) + .containsExactly("event", "personId"); + assertThat(query.getSelect().getSelectItems().subList(0, 2)) + .extracting(item -> ((Identifier) ((SingleColumn) item).getExpression()).getValue()) + .containsExactly("event_name", "Person ID"); + } + + @Test + public void testV0ResolvesDeclaredLogicalTables() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + HogQlCompilationResult result = compiler.compileV0( + envelope("SELECT * FROM persons", OptionalLong.of(7)), + Optional.of(context)); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT person_id AS \"personId\", full_name AS \"name\" FROM analytics.\"Hog Data\".\"raw-persons\"")); + } + + @Test + public void testKeepsOrderingOutputsNeededByLimitByInPrunedSubquery() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + HogQlCompilationResult result = compiler.compileV0( + envelope( + "SELECT personId FROM (SELECT * FROM events ORDER BY event LIMIT 1 BY personId) AS nested", + OptionalLong.of(7)), + Optional.of(context)); + + assertThat(result.statement()).isInstanceOf(Query.class); + assertThat(result.catalogGeneration()).hasValue(7); + } + + @Test + public void testInfersStarsAcrossUnaliasedNestedDerivedTables() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + HogQlCompilationResult result = compiler.compileV0( + envelope( + "WITH nested AS (SELECT * FROM (SELECT * FROM events)) " + + "SELECT event FROM nested", + OptionalLong.of(7)), + Optional.of(context)); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "WITH nested AS (SELECT * FROM (" + + "SELECT event_name AS \"event\" FROM analytics.\"Hog Data\".\"raw-events\")) " + + "SELECT \"event\" AS event FROM nested")); + } + + @Test + public void testExpandsQualifiedLogicalStarsWithExclusionsInManifestOrder() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + HogQlCompilationResult result = compiler.compile( + envelope( + "SELECT e.* EXCLUDE (event), p.* EXCLUDE (personId) " + + "FROM events e JOIN persons p ON e.personId = p.personId", + OptionalLong.of(7)), + Optional.of(context)); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT e.\"Person ID\" AS \"personId\", p.full_name AS \"name\" " + + "FROM analytics.\"Hog Data\".\"raw-events\" e " + + "JOIN analytics.\"Hog Data\".\"raw-persons\" p ON e.\"Person ID\" = p.person_id")); + } + + @Test + public void testExpandsQuotedLogicalStarAndUnqualifiedExclusion() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + HogQlCompilationResult quoted = compiler.compile( + envelope("SELECT \"E\".* EXCLUDE (\"event\") FROM events AS \"E\"", OptionalLong.of(7)), + Optional.of(context)); + HogQlCompilationResult unqualified = compiler.compile( + envelope("SELECT * EXCLUDE (events.personId) FROM events", OptionalLong.of(7)), + Optional.of(context)); + + assertThat(quoted.statement()).isEqualTo(sqlParser.createStatement( + "SELECT \"E\".\"Person ID\" AS \"personId\" FROM analytics.\"Hog Data\".\"raw-events\" AS \"E\"")); + assertThat(unqualified.statement()).isEqualTo(sqlParser.createStatement( + "SELECT event_name AS \"event\" FROM analytics.\"Hog Data\".\"raw-events\"")); + } + + @Test + public void testMatchesLogicalAndPhysicalQualifiedStarSuffixes() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + HogQlCompilationResult result = compiler.compile( + envelope( + "SELECT events.* EXCLUDE (personId), " + + "\"raw-events\".* EXCLUDE (personId), " + + "\"Hog Data\".\"raw-events\".* EXCLUDE (personId), " + + "analytics.\"Hog Data\".\"raw-events\".* EXCLUDE (personId) FROM events", + OptionalLong.of(7)), + Optional.of(context)); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT \"raw-events\".event_name AS \"event\", " + + "\"raw-events\".event_name AS \"event\", " + + "\"raw-events\".event_name AS \"event\", " + + "\"raw-events\".event_name AS \"event\" FROM analytics.\"Hog Data\".\"raw-events\"")); + } + + @Test + public void testExpandsColumnsSelectorsAndReplacements() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + HogQlCompilationResult regex = compiler.compile( + envelope("SELECT COLUMNS('event|person') FROM events", OptionalLong.of(7)), + Optional.of(context)); + HogQlCompilationResult explicit = compiler.compile( + envelope("SELECT COLUMNS(personId, event) FROM events", OptionalLong.of(7)), + Optional.of(context)); + HogQlCompilationResult replaced = compiler.compile( + envelope("SELECT COLUMNS(events.* REPLACE (personId AS event)) FROM events", OptionalLong.of(7)), + Optional.of(context)); + + assertThat(regex.statement()).isEqualTo(sqlParser.createStatement( + "SELECT event_name AS \"event\", \"Person ID\" AS \"personId\" FROM analytics.\"Hog Data\".\"raw-events\"")); + assertThat(explicit.statement()).isEqualTo(sqlParser.createStatement( + "SELECT \"Person ID\" AS personId, event_name AS event FROM analytics.\"Hog Data\".\"raw-events\"")); + assertThat(replaced.statement()).isEqualTo(sqlParser.createStatement( + "SELECT \"Person ID\" AS \"event\", \"raw-events\".\"Person ID\" AS \"personId\" FROM analytics.\"Hog Data\".\"raw-events\"")); + } + + @Test + public void testRejectsInvalidColumnsSelectorsAndReplacementsAtSource() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + assertResolutionFailure( + context, + "SELECT COLUMNS('^missing$') FROM events", + "'^missing$'", + "No HogQL fields matched COLUMNS regex: ^missing$"); + assertResolutionFailure( + context, + "SELECT COLUMNS('(') FROM events", + "'('", + "Invalid HogQL COLUMNS regex: ("); + assertResolutionFailure( + context, + "SELECT COLUMNS(* REPLACE (personId AS event, event AS \"event\")) FROM events", + "\"event\"", + "Duplicate HogQL star replacement: event"); + assertResolutionFailure( + context, + "SELECT COLUMNS(* REPLACE (personId AS \"Event\")) FROM events", + "\"Event\"", + "Unknown HogQL star replacement: Event"); + } + + @Test + public void testFlattensExplicitColumnsWhenRelationSchemaIsUnavailable() + { + List queries = List.of( + "SELECT COLUMNS(event_name, abs(event_id)) FROM analytics.default.raw_events", + "WITH source AS (SELECT event_name FROM analytics.default.raw_events) SELECT COLUMNS(event_name) FROM source", + "SELECT COLUMNS(source.event_name) FROM (SELECT event_name FROM analytics.default.raw_events) source"); + List expected = List.of( + "SELECT event_name, abs(event_id) FROM analytics.default.raw_events", + "WITH source AS (SELECT event_name FROM analytics.default.raw_events) SELECT event_name FROM source", + "SELECT source.event_name FROM (SELECT event_name FROM analytics.default.raw_events) source"); + + for (int index = 0; index < queries.size(); index++) { + HogQlCompilationResult result = compiler.compile(envelope(queries.get(index), OptionalLong.empty()), Optional.empty()); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement(expected.get(index))); + } + } + + @Test + public void testRejectsSchemaDependentColumnsWhenRelationSchemaIsUnavailable() + { + List queries = List.of( + "SELECT COLUMNS('event') FROM analytics.default.raw_events", + "SELECT COLUMNS(* REPLACE (event_name AS event_name)) FROM analytics.default.raw_events"); + + for (String hogql : queries) { + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile(envelope(hogql, OptionalLong.empty()), Optional.empty())); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_UNSUPPORTED_FEATURE.toErrorCode()); + assertThat(exception.getLocation()).isPresent(); + assertThat(exception).hasMessageContaining("requires a logical relation from the semantic catalog"); + } + } + + @Test + public void testRejectsInvalidLogicalStarQualifierAndExclusionsAtSource() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + assertResolutionFailure( + context, + "SELECT e.* EXCLUDE (missing) FROM events e", + "missing", + "Unknown HogQL star exclusion: missing"); + assertResolutionFailure( + context, + "SELECT e.* EXCLUDE (event, \"event\") FROM events e", + "\"event\"", + "Duplicate HogQL star exclusion: event"); + assertResolutionFailure( + context, + "SELECT e.* FROM events e JOIN persons e ON e.personId = e.personId", + "e.*", + "Ambiguous HogQL star qualifier: e"); + assertResolutionFailure( + context, + "SELECT e.* FROM events AS \"E\"", + "e.*", + "Unknown HogQL star qualifier: e"); + assertResolutionFailure( + context, + "SELECT events.* FROM events e", + "events.*", + "Unknown HogQL star qualifier: events"); + assertResolutionFailure( + context, + "SELECT analytics.\"hog data\".\"raw-events\".* FROM events", + "analytics", + "Unknown HogQL star qualifier: analytics.hog data.raw-events"); + } + + @Test + public void testLowersQualifiedPhysicalStarWithoutFetchingSemanticMetadata() + { + AtomicInteger pins = new AtomicInteger(); + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> { + pins.incrementAndGet(); + return new PinnedSnapshot(SNAPSHOT); + }); + + HogQlCompilationResult result = compiler.compile( + envelope("SELECT p.* FROM analytics.default.raw_events p", OptionalLong.empty()), + Optional.of(context)); + + assertThat(pins).hasValue(0); + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT p.* FROM analytics.default.raw_events p")); + } + + @Test + public void testRejectsStarExclusionsWhenRelationSchemaIsUnavailable() + { + AtomicInteger pins = new AtomicInteger(); + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> { + pins.incrementAndGet(); + return new PinnedSnapshot(SNAPSHOT); + }); + List queries = List.of( + "SELECT p.* EXCLUDE (event_name) FROM analytics.default.raw_events p", + "WITH source AS (SELECT 1 AS event_name) SELECT s.* EXCLUDE (event_name) FROM source s"); + + for (String hogql : queries) { + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile(envelope(hogql, OptionalLong.empty()), Optional.of(context))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_UNSUPPORTED_FEATURE.toErrorCode()); + assertThat(exception.getLocation()).contains(new Location(1, hogql.lastIndexOf("event_name") + 1)); + assertThat(exception).hasMessageContaining("HogQL star exclusions require a logical relation from the semantic catalog: event_name"); + } + assertThat(pins).hasValue(0); + } + + @Test + public void testResolvesLogicalFieldsInsideNamedWindows() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + HogQlCompilationResult result = compiler.compile( + envelope( + "SELECT count(*) OVER recent FROM events " + + "WINDOW recent AS (PARTITION BY event ORDER BY personId ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)", + OptionalLong.of(7)), + Optional.of(context)); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT count(*) OVER recent FROM analytics.\"Hog Data\".\"raw-events\" " + + "WINDOW recent AS (PARTITION BY event_name ORDER BY \"Person ID\" ROWS BETWEEN 1 PRECEDING AND CURRENT ROW)")); + } + + @Test + public void testResolvesLogicalFieldsInsideCollectionSubscripts() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + HogQlCompilationResult result = compiler.compile( + envelope("SELECT [event][1], [personId][1], [event][1].label FROM events", OptionalLong.of(7)), + Optional.of(context)); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT ARRAY[event_name][1], ARRAY[\"Person ID\"][1], ARRAY[event_name][1].label " + + "FROM analytics.\"Hog Data\".\"raw-events\"")); + } + + @Test + public void testCatalogIndependentAndPhysicalQueriesDoNotFetchSemanticMetadata() + { + AtomicInteger pins = new AtomicInteger(); + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> { + pins.incrementAndGet(); + return new PinnedSnapshot(SNAPSHOT); + }); + + HogQlCompilationResult literal = compiler.compile(envelope("SELECT 1", OptionalLong.empty()), Optional.of(context)); + HogQlCompilationResult physical = compiler.compile(envelope("SELECT event_name FROM analytics.default.events", OptionalLong.empty()), Optional.of(context)); + + assertThat(pins).hasValue(0); + assertThat(literal.catalogGeneration()).isEmpty(); + assertThat(physical.catalogGeneration()).isEmpty(); + } + + @Test + public void testCteScopeShadowsLogicalCatalogNamesWithoutFetchingMetadata() + { + AtomicInteger pins = new AtomicInteger(); + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> { + pins.incrementAndGet(); + return new PinnedSnapshot(SNAPSHOT); + }); + + HogQlCompilationResult result = compiler.compile( + envelope("WITH events AS (SELECT 1 AS event), next AS (SELECT event FROM events) SELECT event FROM next", OptionalLong.empty()), + Optional.of(context)); + + assertThat(pins).hasValue(0); + assertThat(result.catalogGeneration()).isEmpty(); + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "WITH events AS (SELECT 1 AS event), next AS (SELECT event FROM events) SELECT event FROM next")); + } + + @Test + public void testInfersCteOutputsForColumnsAndStarModifiers() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + HogQlCompilationResult columns = compiler.compile( + envelope( + "WITH source AS (SELECT event AS eventName, personId AS person FROM events) " + + "SELECT COLUMNS('eventName|person') FROM source", + OptionalLong.of(7)), + Optional.of(context)); + HogQlCompilationResult modifiedStar = compiler.compile( + envelope( + "WITH source AS (SELECT event AS eventName, personId AS person FROM events) " + + "SELECT COLUMNS(source.* EXCLUDE (person) REPLACE (person AS eventName)) FROM source", + OptionalLong.of(7)), + Optional.of(context)); + HogQlCompilationResult columnAliases = compiler.compile( + envelope( + "WITH source(renamedEvent, renamedPerson) AS (SELECT event, personId FROM events) " + + "SELECT renamedPerson FROM source", + OptionalLong.of(7)), + Optional.of(context)); + + assertThat(columns.statement()).isEqualTo(sqlParser.createStatement( + "WITH source AS (SELECT event_name AS eventName, \"Person ID\" AS person FROM analytics.\"Hog Data\".\"raw-events\") " + + "SELECT \"eventName\" AS \"eventName\", \"person\" AS \"person\" FROM source")); + assertThat(modifiedStar.statement()).isEqualTo(sqlParser.createStatement( + "WITH source AS (SELECT event_name AS eventName, \"Person ID\" AS person FROM analytics.\"Hog Data\".\"raw-events\") " + + "SELECT \"person\" AS \"eventName\" FROM source")); + assertThat(columnAliases.statement()).isEqualTo(sqlParser.createStatement( + "WITH source(renamedPerson) AS (" + + "SELECT \"Person ID\" AS personId FROM analytics.\"Hog Data\".\"raw-events\") " + + "SELECT \"renamedPerson\" AS renamedPerson FROM source")); + } + + @Test + public void testRejectsCteOutputWithoutAnInferableName() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + String hogql = "WITH source AS (SELECT event + personId FROM events) SELECT * FROM source"; + + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile(envelope(hogql, OptionalLong.of(7)), Optional.of(context))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(exception.getLocation()).contains(new Location(1, hogql.indexOf("event + personId") + 1)); + assertThat(exception).hasMessageContaining("Cannot infer HogQL CTE output name"); + } + + @Test + public void testInfersDerivedOutputsForColumnsAndStarModifiers() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + HogQlCompilationResult columns = compiler.compile( + envelope( + "SELECT COLUMNS('eventName|person') FROM " + + "(SELECT event AS eventName, personId AS person FROM events) source", + OptionalLong.of(7)), + Optional.of(context)); + HogQlCompilationResult modifiedStar = compiler.compile( + envelope( + "SELECT COLUMNS(source.* EXCLUDE (person) REPLACE (person AS eventName)) FROM " + + "(SELECT event AS eventName, personId AS person FROM events) source", + OptionalLong.of(7)), + Optional.of(context)); + HogQlCompilationResult columnAliases = compiler.compile( + envelope( + "SELECT renamedPerson FROM (SELECT event, personId FROM events) source(renamedEvent, renamedPerson)", + OptionalLong.of(7)), + Optional.of(context)); + + assertThat(columns.statement()).isEqualTo(sqlParser.createStatement( + "SELECT \"eventName\" AS \"eventName\", \"person\" AS \"person\" FROM (" + + "SELECT event_name AS eventName, \"Person ID\" AS person FROM analytics.\"Hog Data\".\"raw-events\") source")); + assertThat(modifiedStar.statement()).isEqualTo(sqlParser.createStatement( + "SELECT \"person\" AS \"eventName\" FROM (" + + "SELECT event_name AS eventName, \"Person ID\" AS person FROM analytics.\"Hog Data\".\"raw-events\") source")); + assertThat(columnAliases.statement()).isEqualTo(sqlParser.createStatement( + "SELECT \"renamedPerson\" AS renamedPerson FROM (" + + "SELECT \"Person ID\" AS personId FROM analytics.\"Hog Data\".\"raw-events\") " + + "source(renamedPerson)")); + } + + @Test + public void testRejectsDerivedOutputWithoutAnInferableName() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + String hogql = "SELECT * FROM (SELECT event + personId FROM events) source"; + + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile(envelope(hogql, OptionalLong.of(7)), Optional.of(context))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(exception.getLocation()).contains(new Location(1, hogql.indexOf("event + personId") + 1)); + assertThat(exception).hasMessageContaining("Cannot infer HogQL CTE output name"); + } + + @Test + public void testInfersSetOperationOutputsAndPrunesBranchesByPosition() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + HogQlCompilationResult cte = compiler.compile( + envelope( + "WITH source AS (" + + "SELECT event AS chosen, personId AS unused FROM events " + + "UNION ALL " + + "SELECT personId AS rightChosen, event AS rightUnused FROM events) " + + "SELECT chosen FROM source", + OptionalLong.of(7)), + Optional.of(context)); + HogQlCompilationResult derived = compiler.compile( + envelope( + "SELECT COLUMNS('value') FROM (" + + "SELECT event AS value FROM events " + + "INTERSECT " + + "SELECT personId AS other FROM events) source", + OptionalLong.of(7)), + Optional.of(context)); + + assertThat(cte.statement()).isEqualTo(sqlParser.createStatement( + "WITH source AS (" + + "SELECT event_name AS chosen FROM analytics.\"Hog Data\".\"raw-events\" " + + "UNION ALL " + + "SELECT \"Person ID\" AS rightChosen FROM analytics.\"Hog Data\".\"raw-events\") " + + "SELECT \"chosen\" AS chosen FROM source")); + assertThat(derived.statement()).isEqualTo(sqlParser.createStatement( + "SELECT \"value\" AS \"value\" FROM (" + + "SELECT event_name AS value FROM analytics.\"Hog Data\".\"raw-events\" " + + "INTERSECT " + + "SELECT \"Person ID\" AS other FROM analytics.\"Hog Data\".\"raw-events\") source")); + } + + @Test + public void testRejectsUnsafeSetOperationOutputInference() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + TrinoException incompatibleArity = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile( + envelope( + "WITH source AS (SELECT event FROM events EXCEPT SELECT event, personId FROM events) SELECT event FROM source", + OptionalLong.of(7)), + Optional.of(context))); + TrinoException unnamed = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile( + envelope( + "WITH source AS (SELECT event FROM events UNION ALL SELECT event + personId FROM events) SELECT event FROM source", + OptionalLong.of(7)), + Optional.of(context))); + TrinoException ambiguous = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile( + envelope( + "WITH source AS (SELECT event AS value, personId AS value FROM events UNION ALL " + + "SELECT event AS firstValue, personId AS secondValue FROM events) SELECT value FROM source", + OptionalLong.of(7)), + Optional.of(context))); + TrinoException unsafeAliases = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile( + envelope( + "WITH source(onlyValue) AS (SELECT event AS value, personId AS other FROM events UNION ALL " + + "SELECT event AS rightValue, personId AS rightOther FROM events) SELECT onlyValue FROM source", + OptionalLong.of(7)), + Optional.of(context))); + + assertThat(incompatibleArity.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(incompatibleArity).hasMessageContaining("set operation branches have incompatible output arity"); + assertThat(unnamed.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(unnamed).hasMessageContaining("Cannot infer HogQL CTE output name"); + assertThat(ambiguous.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(ambiguous).hasMessageContaining("output names must be unique"); + assertThat(unsafeAliases.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(unsafeAliases).hasMessageContaining("column alias count does not match its output count"); + } + + @Test + public void testUnknownLogicalFieldFailsAtOriginalLocation() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile(envelope("SELECT missing FROM events", OptionalLong.empty()), Optional.of(context))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(exception.getLocation()).contains(new Location(1, 8)); + assertThat(exception).hasMessage("line 1:8: Unknown HogQL field: missing"); + } + + @Test + public void testResolvesAliasedLogicalTablesAndJoinCriteria() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + HogQlCompilationResult result = compiler.compile( + envelope("SELECT e.event, p.name FROM events e JOIN persons p ON e.personId = p.personId", OptionalLong.of(7)), + Optional.of(context)); + + assertThat(result.catalogGeneration()).hasValue(7); + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT e.event_name AS event, p.full_name AS name " + + "FROM analytics.\"Hog Data\".\"raw-events\" e " + + "JOIN analytics.\"Hog Data\".\"raw-persons\" p ON e.\"Person ID\" = p.person_id")); + } + + @Test + public void testResolvesCorrelatedLogicalFieldsInExpressionSubqueries() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + HogQlCompilationResult inSubquery = compiler.compile( + envelope( + "SELECT e.event FROM events e WHERE e.personId IN " + + "(SELECT p.personId FROM persons p WHERE p.personId = e.personId)", + OptionalLong.of(7)), + Optional.of(context)); + HogQlCompilationResult scalarSubquery = compiler.compile( + envelope( + "SELECT e.event, (SELECT p.name FROM persons p WHERE p.personId = e.personId) FROM events e", + OptionalLong.of(7)), + Optional.of(context)); + + assertThat(inSubquery.statement()).isEqualTo(sqlParser.createStatement( + "SELECT e.event_name AS event FROM analytics.\"Hog Data\".\"raw-events\" e " + + "WHERE e.\"Person ID\" IN (" + + "SELECT p.person_id AS personId FROM analytics.\"Hog Data\".\"raw-persons\" p " + + "WHERE p.person_id = e.\"Person ID\")")); + assertThat(scalarSubquery.statement()).isEqualTo(sqlParser.createStatement( + "SELECT e.event_name AS event, (" + + "SELECT p.full_name AS name FROM analytics.\"Hog Data\".\"raw-persons\" p " + + "WHERE p.person_id = e.\"Person ID\") FROM analytics.\"Hog Data\".\"raw-events\" e")); + } + + @Test + public void testInnerLogicalQualifierShadowsCorrelatedOuterQualifier() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + + HogQlCompilationResult result = compiler.compile( + envelope("SELECT (SELECT e.name FROM persons e) FROM events e", OptionalLong.of(7)), + Optional.of(context)); + + assertThat(result.statement()).isEqualTo(sqlParser.createStatement( + "SELECT (SELECT e.full_name AS name FROM analytics.\"Hog Data\".\"raw-persons\" e) " + + "FROM analytics.\"Hog Data\".\"raw-events\" e")); + } + + @Test + public void testRejectsUnknownQualifierInCorrelatedLogicalSubquery() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + String hogql = "SELECT e.event FROM events e WHERE e.personId IN " + + "(SELECT p.personId FROM persons p WHERE p.personId = missing.personId)"; + + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile(envelope(hogql, OptionalLong.of(7)), Optional.of(context))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(exception.getLocation()).contains(new Location(1, hogql.indexOf("missing.personId") + 1)); + assertThat(exception).hasMessageContaining("Unknown HogQL field: personId"); + } + + @Test + public void testRejectsLogicalUsingWhenPhysicalColumnNamesDiffer() + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + String hogql = "SELECT e.event FROM events e JOIN persons p USING (personId)"; + + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile(envelope(hogql, OptionalLong.empty()), Optional.of(context))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(exception.getLocation()).contains(new Location(1, hogql.lastIndexOf("personId") + 1)); + assertThat(exception).hasMessageContaining("HogQL USING field maps to different physical columns: personId"); + } + + private static QuerySpecification querySpecification(HogQlCompilationResult result) + { + return (QuerySpecification) ((Query) result.statement()).getQueryBody(); + } + + private void assertResolutionFailure(HogQlSemanticCatalogContext context, String hogql, String errorToken, String message) + { + TrinoException exception = catchThrowableOfType( + TrinoException.class, + () -> compiler.compile(envelope(hogql, OptionalLong.empty()), Optional.of(context))); + + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_RESOLUTION_ERROR.toErrorCode()); + assertThat(exception.getLocation()).contains(new Location(1, hogql.indexOf(errorToken) + 1)); + assertThat(exception).hasMessage("line %s:%s: %s", 1, hogql.indexOf(errorToken) + 1, message); + } + + private static HogQlCompileEnvelope envelope(String query, OptionalLong generation) + { + return new HogQlCompileEnvelope( + query, + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + Map.of(), + Map.of(), + Map.of(), + Map.of(), + generation); + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlTemplateStringCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlTemplateStringCompiler.java new file mode 100644 index 000000000000..9f8df8e8ff1b --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlTemplateStringCompiler.java @@ -0,0 +1,35 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.sql.SqlFormatter; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TestHogQlTemplateStringCompiler +{ + @Test + public void testLowersTemplateStringToStockTrinoAst() + { + String sql = SqlFormatter.formatSql(new HogQlCompiler().compile( + "SELECT f'{year}-{month}-{day}' FROM calendar")); + + assertThat(sql) + .contains("concat(") + .contains("CAST(year AS varchar)") + .contains("CAST(month AS varchar)") + .contains("CAST(day AS varchar)"); + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlWindowCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlWindowCompiler.java new file mode 100644 index 000000000000..b38113040bcd --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlWindowCompiler.java @@ -0,0 +1,121 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.hogql.compiler.HogQlTypedValue.StringValue; +import io.trino.sql.SqlFormatter; +import io.trino.sql.parser.SqlParser; +import io.trino.sql.tree.Parameter; +import io.trino.sql.tree.Statement; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.Deque; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TestHogQlWindowCompiler +{ + private final HogQlCompiler compiler = new HogQlCompiler(); + private final SqlParser sqlParser = new SqlParser(); + + @ParameterizedTest + @ValueSource(strings = { + "SELECT row_number() OVER ()", + "SELECT sum(value) FILTER (WHERE enabled) OVER (PARTITION BY team_id ORDER BY timestamp DESC NULLS LAST)", + "SELECT avg(value) OVER (ORDER BY timestamp ROWS BETWEEN 2 PRECEDING AND CURRENT ROW)", + "SELECT first_value(value) OVER (ORDER BY timestamp RANGE UNBOUNDED PRECEDING)", + "SELECT row_number() OVER recent FROM events WINDOW recent AS (PARTITION BY team_id ORDER BY timestamp)", + "SELECT sum(value) OVER first_window, avg(value) OVER second_window FROM events " + + "WINDOW first_window AS (ROWS CURRENT ROW), second_window AS (RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)", + }) + public void testLowersWindowsToEquivalentStockTrinoAst(String hogql) + { + Statement statement = compiler.compile(hogql); + + assertThat(statement).isEqualTo(sqlParser.createStatement(hogql)); + assertThat(sqlParser.createStatement(SqlFormatter.formatSql(statement))).isEqualTo(statement); + } + + @ParameterizedTest + @ValueSource(strings = { + "ROWS UNBOUNDED PRECEDING", + "ROWS 2 PRECEDING", + "ROWS CURRENT ROW", + "ROWS 2 FOLLOWING", + "ROWS UNBOUNDED FOLLOWING", + "RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW", + "RANGE BETWEEN 2 PRECEDING AND 3 FOLLOWING", + }) + public void testLowersEveryCanonicalFrameBound(String frame) + { + String hogql = "SELECT sum(value) OVER (ORDER BY timestamp " + frame + ")"; + + assertThat(compiler.compile(hogql)).isEqualTo(sqlParser.createStatement(hogql)); + } + + @Test + public void testLowersCanonicalWindowNullTreatmentOrder() + { + assertThat(compiler.compile("SELECT first_value(value) OVER (ORDER BY timestamp) IGNORE NULLS")) + .isEqualTo(sqlParser.createStatement("SELECT first_value(value) IGNORE NULLS OVER (ORDER BY timestamp)")); + } + + @Test + public void testBindsPlaceholdersAcrossWindowComponents() + { + String hogql = "SELECT sum({value}) FILTER (WHERE {filter}) OVER " + + "(PARTITION BY {partition} ORDER BY {sort} ROWS BETWEEN {lower} PRECEDING AND {upper} FOLLOWING)"; + Map bindings = Map.of( + "value", typedValue("value"), + "filter", typedValue("filter"), + "partition", typedValue("partition"), + "sort", typedValue("sort"), + "lower", typedValue("lower"), + "upper", typedValue("upper")); + + HogQlCompilationResult result = compiler.compile(hogql, bindings); + + assertThat(result.parameterNames()).containsExactly("value", "filter", "partition", "sort", "lower", "upper"); + assertThat(parameters(result.statement())).extracting(Parameter::getId).containsExactly(0, 1, 2, 3, 4, 5); + } + + private static HogQlTypedValue typedValue(String value) + { + return new HogQlTypedValue("varchar", new StringValue(value)); + } + + private static List parameters(Statement statement) + { + List parameters = new ArrayList<>(); + Deque nodes = new ArrayDeque<>(); + nodes.add(statement); + while (!nodes.isEmpty()) { + io.trino.sql.tree.Node node = nodes.removeFirst(); + if (node instanceof Parameter parameter) { + parameters.add(parameter); + } + nodes.addAll(node.getChildren()); + } + return parameters.stream() + .sorted(Comparator.comparingInt(Parameter::getId)) + .toList(); + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlXCompiler.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlXCompiler.java new file mode 100644 index 000000000000..019ccef32cb0 --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/TestHogQlXCompiler.java @@ -0,0 +1,34 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler; + +import io.trino.sql.parser.SqlParser; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TestHogQlXCompiler +{ + private final SqlParser sqlParser = new SqlParser(); + + @Test + public void testLowersNestedTagToStockTrinoRows() + { + assertThat(new HogQlCompiler().compile( + "SELECT {event}Bold! FROM events")) + .isEqualTo(sqlParser.createStatement( + "SELECT ROW('__hx_tag', 'a', 'href', 'https://example.com', 'target', true, " + + "'children', ROW(event, ROW('__hx_tag', 'strong', 'children', ROW('Bold!')))) FROM events")); + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestBoundedAsyncHogQlExchangeRateSnapshotCache.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestBoundedAsyncHogQlExchangeRateSnapshotCache.java new file mode 100644 index 000000000000..3c2dc2a94173 --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestBoundedAsyncHogQlExchangeRateSnapshotCache.java @@ -0,0 +1,220 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.compiler.catalog.BoundedAsyncHogQlExchangeRateSnapshotCache.SnapshotLoader; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateException.Failure; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshot.ExchangeRate; +import org.junit.jupiter.api.Test; + +import java.time.Duration; +import java.util.ArrayDeque; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.OptionalLong; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.atomic.AtomicLong; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestBoundedAsyncHogQlExchangeRateSnapshotCache +{ + private static final Duration REFRESH_AFTER = Duration.ofNanos(10); + private static final Duration EXPIRE_AFTER = Duration.ofNanos(20); + private static final Duration FAILURE_BACKOFF = Duration.ofNanos(100); + + @Test + public void testLatestAndExactReadsShareLoadsAndRemainGenerationPinned() + { + AtomicLong ticker = new AtomicLong(); + ControlledLoader loader = new ControlledLoader(); + BoundedAsyncHogQlExchangeRateSnapshotCache cache = cache(3, ticker, loader); + + CompletableFuture initialLoad = loader.expect(OptionalLong.empty()); + CompletionStage first = cache.prewarm(); + CompletionStage shared = cache.prewarm(); + assertThat(loader.loadCount(OptionalLong.empty())).isEqualTo(1); + initialLoad.complete(snapshot(1)); + assertThat(first.toCompletableFuture()).isCompletedWithValue(snapshot(1)); + assertThat(shared.toCompletableFuture()).isCompletedWithValue(snapshot(1)); + + assertThat(cache.currentSnapshot(OptionalLong.of(1))).contains(snapshot(1)); + assertThat(loader.loadCount(OptionalLong.of(1))).isZero(); + + CompletableFuture latestRefresh = loader.expect(OptionalLong.empty()); + ticker.set(REFRESH_AFTER.toNanos()); + assertThat(cache.currentSnapshot()).contains(snapshot(1)); + latestRefresh.complete(snapshot(2)); + + assertThat(cache.currentSnapshot()).contains(snapshot(2)); + assertThat(cache.currentSnapshot(OptionalLong.of(1))).contains(snapshot(1)); + } + + @Test + public void testRejectsGenerationRegressionAndChangedSameGeneration() + { + AtomicLong ticker = new AtomicLong(); + ControlledLoader loader = new ControlledLoader(); + BoundedAsyncHogQlExchangeRateSnapshotCache cache = cache(3, ticker, loader); + completePrewarm(cache, loader, OptionalLong.empty(), snapshot(2)); + + CompletableFuture regressedLoad = loader.expect(OptionalLong.empty()); + CompletionStage regressed = cache.prewarm(); + regressedLoad.complete(snapshot(1)); + assertFailure(regressed, Failure.GENERATION_MISMATCH); + assertThat(cache.currentSnapshot()).contains(snapshot(2)); + + CompletableFuture changedLoad = loader.expect(OptionalLong.empty()); + CompletionStage changed = cache.prewarm(); + changedLoad.complete(snapshot(2, "9100000000")); + assertFailure(changed, Failure.GENERATION_MISMATCH); + assertThat(cache.currentSnapshot()).contains(snapshot(2)); + } + + @Test + public void testExactGenerationEvictionCancelsLoadAndNeverFallsBackToLatest() + { + AtomicLong ticker = new AtomicLong(); + ControlledLoader loader = new ControlledLoader(); + BoundedAsyncHogQlExchangeRateSnapshotCache cache = cache(2, ticker, loader); + completePrewarm(cache, loader, OptionalLong.empty(), snapshot(1)); + + CompletableFuture generationSevenLoad = loader.expect(OptionalLong.of(7)); + CompletionStage generationSeven = cache.prewarm(OptionalLong.of(7)); + CompletableFuture generationEightLoad = loader.expect(OptionalLong.of(8)); + CompletionStage generationEight = cache.prewarm(OptionalLong.of(8)); + + assertThat(generationSevenLoad).isCancelled(); + assertThat(generationSeven.toCompletableFuture()).isCompletedExceptionally(); + assertThat(cache.currentSnapshot()).contains(snapshot(1)); + generationEightLoad.complete(snapshot(8)); + assertThat(generationEight.toCompletableFuture()).isCompletedWithValue(snapshot(8)); + + cache.invalidate(); + CompletableFuture latestLoad = loader.expect(OptionalLong.empty()); + assertThat(cache.currentSnapshot()).isEmpty(); + latestLoad.complete(snapshot(9)); + CompletableFuture missingExact = loader.expect(OptionalLong.of(7)); + assertThat(cache.currentSnapshot(OptionalLong.of(7))).isEmpty(); + missingExact.completeExceptionally(new IllegalStateException("metadata unavailable")); + assertThat(cache.currentSnapshot(OptionalLong.of(7))).isEmpty(); + assertThat(cache.currentSnapshot()).contains(snapshot(9)); + } + + @Test + public void testRefreshFailureServesLastGoodOnlyUntilExpiryAndBacksOff() + { + AtomicLong ticker = new AtomicLong(); + ControlledLoader loader = new ControlledLoader(); + BoundedAsyncHogQlExchangeRateSnapshotCache cache = cache(2, ticker, loader); + completePrewarm(cache, loader, OptionalLong.empty(), snapshot(1)); + + CompletableFuture failedLoad = loader.expect(OptionalLong.empty()); + ticker.set(REFRESH_AFTER.toNanos()); + assertThat(cache.currentSnapshot()).contains(snapshot(1)); + failedLoad.completeExceptionally(new IllegalStateException("metadata unavailable")); + + ticker.set(EXPIRE_AFTER.toNanos()); + assertThat(cache.currentSnapshot()).isEmpty(); + assertThat(loader.loadCount(OptionalLong.empty())).isEqualTo(2); + + CompletableFuture recoveredLoad = loader.expect(OptionalLong.empty()); + ticker.set(REFRESH_AFTER.plus(FAILURE_BACKOFF).toNanos()); + assertThat(cache.currentSnapshot()).isEmpty(); + recoveredLoad.complete(snapshot(2)); + assertThat(cache.currentSnapshot()).contains(snapshot(2)); + } + + private static BoundedAsyncHogQlExchangeRateSnapshotCache cache(int maximumEntries, AtomicLong ticker, SnapshotLoader loader) + { + return new BoundedAsyncHogQlExchangeRateSnapshotCache( + maximumEntries, + REFRESH_AFTER, + EXPIRE_AFTER, + FAILURE_BACKOFF, + ticker::get, + Runnable::run, + loader); + } + + private static void completePrewarm( + BoundedAsyncHogQlExchangeRateSnapshotCache cache, + ControlledLoader loader, + OptionalLong expectedGeneration, + HogQlExchangeRateSnapshot snapshot) + { + CompletableFuture load = loader.expect(expectedGeneration); + CompletionStage refresh = cache.prewarm(expectedGeneration); + load.complete(snapshot); + assertThat(refresh.toCompletableFuture()).isCompletedWithValue(snapshot); + } + + private static void assertFailure(CompletionStage refresh, Failure failure) + { + assertThatThrownBy(refresh.toCompletableFuture()::join) + .cause() + .isInstanceOfSatisfying(HogQlExchangeRateException.class, exception -> assertThat(exception.failure()).isEqualTo(failure)); + } + + private static HogQlExchangeRateSnapshot snapshot(long generation) + { + return snapshot(generation, "9049000000"); + } + + private static HogQlExchangeRateSnapshot snapshot(long generation, String eurRate) + { + return new HogQlExchangeRateSnapshot( + 1, + 1, + generation, + "USD", + 10, + List.of( + new ExchangeRate("EUR", "2024-01-01", eurRate), + new ExchangeRate("USD", "1970-01-01", "10000000000"))); + } + + private static final class ControlledLoader + implements SnapshotLoader + { + private final Map>> expected = new HashMap<>(); + private final Map loadCounts = new HashMap<>(); + + public CompletableFuture expect(OptionalLong generation) + { + CompletableFuture future = new CompletableFuture<>(); + expected.computeIfAbsent(generation, _ -> new ArrayDeque<>()).add(future); + return future; + } + + public int loadCount(OptionalLong generation) + { + return loadCounts.getOrDefault(generation, 0); + } + + @Override + public CompletionStage load(OptionalLong expectedGeneration) + { + loadCounts.merge(expectedGeneration, 1, Integer::sum); + ArrayDeque> queue = expected.get(expectedGeneration); + if (queue == null || queue.isEmpty()) { + throw new AssertionError("unexpected exchange-rate load: " + expectedGeneration); + } + return queue.remove(); + } + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestBoundedAsyncHogQlSemanticCatalogSnapshotCache.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestBoundedAsyncHogQlSemanticCatalogSnapshotCache.java new file mode 100644 index 000000000000..b992c2d3f3e6 --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestBoundedAsyncHogQlSemanticCatalogSnapshotCache.java @@ -0,0 +1,541 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.compiler.catalog.BoundedAsyncHogQlSemanticCatalogSnapshotCache.SnapshotLoader; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogException.Failure; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotLoader.LoadRequest; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinRequest; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; +import io.trino.hogql.parser.HogQlLanguageVersion; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayDeque; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestBoundedAsyncHogQlSemanticCatalogSnapshotCache +{ + private static final HogQlLanguageVersion LANGUAGE_VERSION = HogQlLanguageVersion.valueOf("1.0.0"); + private static final PhysicalIdentifier CATALOG = catalog("ducklake"); + private static final Duration REFRESH_AFTER = Duration.ofNanos(10); + private static final Duration EXPIRE_AFTER = Duration.ofNanos(20); + private static final Duration FAILURE_BACKOFF = Duration.ofNanos(100); + + @Test + public void testOutOfOrderRefreshCannotPublishAfterInvalidation() + { + AtomicLong ticker = new AtomicLong(); + ControlledLoader loader = new ControlledLoader(); + CompletableFuture oldLoad = loader.expect(CATALOG); + BoundedAsyncHogQlSemanticCatalogSnapshotCache cache = cache(2, ticker, loader); + + CompletionStage invalidatedRefresh = cache.prewarm(CATALOG); + cache.invalidate(CATALOG); + + CompletableFuture newLoad = loader.expect(CATALOG); + CompletionStage currentRefresh = cache.prewarm(CATALOG); + newLoad.complete(snapshot(CATALOG, 2)); + oldLoad.complete(snapshot(CATALOG, 1)); + + assertThat(currentRefresh.toCompletableFuture()).isCompletedWithValueMatching(snapshot -> snapshot.generation() == 2); + assertThat(invalidatedRefresh.toCompletableFuture()).isCompletedExceptionally(); + assertThat(cache.currentSnapshot(CATALOG)).get().extracting(HogQlSemanticCatalogSnapshot::generation).isEqualTo(2L); + } + + @Test + public void testRejectsInvalidRefreshWithoutReplacingLastKnownGood() + { + AtomicLong ticker = new AtomicLong(); + ControlledLoader loader = new ControlledLoader(); + BoundedAsyncHogQlSemanticCatalogSnapshotCache cache = cache(2, ticker, loader); + completePrewarm(cache, loader, snapshot(CATALOG, 2)); + + CompletableFuture olderLoad = loader.expect(CATALOG); + CompletionStage olderRefresh = cache.prewarm(CATALOG); + olderLoad.complete(snapshot(CATALOG, 1)); + + assertRefreshFailure(olderRefresh, Failure.GENERATION_MISMATCH); + assertThat(cache.currentSnapshot(CATALOG)).get().extracting(HogQlSemanticCatalogSnapshot::generation).isEqualTo(2L); + + PhysicalIdentifier otherCatalog = catalog("other"); + CompletableFuture mismatchedLoad = loader.expect(CATALOG); + CompletionStage mismatchedRefresh = cache.prewarm(CATALOG); + mismatchedLoad.complete(snapshot(otherCatalog, 3)); + + assertRefreshFailure(mismatchedRefresh, Failure.CATALOG_MISMATCH); + assertThat(cache.currentSnapshot(CATALOG)).get().extracting(HogQlSemanticCatalogSnapshot::generation).isEqualTo(2L); + + CompletableFuture mutatedLoad = loader.expect(CATALOG); + CompletionStage mutatedRefresh = cache.prewarm(CATALOG); + mutatedLoad.complete(snapshot(CATALOG, 2, "changed_id")); + + assertRefreshFailure(mutatedRefresh, Failure.GENERATION_MISMATCH); + assertThat(cache.currentSnapshot(CATALOG)).get().extracting(snapshot -> snapshot.logicalTables().getFirst().fields().getFirst().name()).isEqualTo("id"); + } + + @Test + public void testServesStaleSnapshotWhileRefreshingAndFailsClosedAfterExpiry() + { + AtomicLong ticker = new AtomicLong(); + ControlledLoader loader = new ControlledLoader(); + BoundedAsyncHogQlSemanticCatalogSnapshotCache cache = cache(2, ticker, loader); + completePrewarm(cache, loader, snapshot(CATALOG, 1)); + + ticker.set(REFRESH_AFTER.toNanos() - 1); + assertThat(cache.currentSnapshot(CATALOG)).get().extracting(HogQlSemanticCatalogSnapshot::generation).isEqualTo(1L); + assertThat(loader.loadCount(CATALOG)).isEqualTo(1); + + CompletableFuture refresh = loader.expect(CATALOG); + ticker.set(REFRESH_AFTER.toNanos()); + assertThat(cache.currentSnapshot(CATALOG)).get().extracting(HogQlSemanticCatalogSnapshot::generation).isEqualTo(1L); + assertThat(loader.loadCount(CATALOG)).isEqualTo(2); + + ticker.set(EXPIRE_AFTER.toNanos()); + assertThat(cache.currentSnapshot(CATALOG)).isEmpty(); + assertThat(loader.loadCount(CATALOG)).isEqualTo(2); + + refresh.complete(snapshot(CATALOG, 2)); + assertThat(cache.currentSnapshot(CATALOG)).get().extracting(HogQlSemanticCatalogSnapshot::generation).isEqualTo(2L); + } + + @Test + public void testRefreshAgeSurvivesTickerWraparound() + { + AtomicLong ticker = new AtomicLong(Long.MAX_VALUE - 5); + ControlledLoader loader = new ControlledLoader(); + BoundedAsyncHogQlSemanticCatalogSnapshotCache cache = cache(2, ticker, loader); + completePrewarm(cache, loader, snapshot(CATALOG, 1)); + + loader.expect(CATALOG); + ticker.set(Long.MIN_VALUE + 4); + assertThat(cache.currentSnapshot(CATALOG)).isPresent(); + assertThat(loader.loadCount(CATALOG)).isEqualTo(2); + } + + @Test + public void testRefreshFailurePreservesSnapshotOnlyUntilExpiryAndBacksOff() + { + AtomicLong ticker = new AtomicLong(); + ControlledLoader loader = new ControlledLoader(); + BoundedAsyncHogQlSemanticCatalogSnapshotCache cache = cache(2, ticker, loader); + completePrewarm(cache, loader, snapshot(CATALOG, 1)); + + CompletableFuture failedLoad = loader.expect(CATALOG); + ticker.set(REFRESH_AFTER.toNanos()); + assertThat(cache.currentSnapshot(CATALOG)).isPresent(); + failedLoad.completeExceptionally(new IllegalStateException("metadata unavailable")); + + ticker.set(EXPIRE_AFTER.toNanos() - 1); + assertThat(cache.currentSnapshot(CATALOG)).isPresent(); + assertThat(loader.loadCount(CATALOG)).isEqualTo(2); + + ticker.set(EXPIRE_AFTER.toNanos()); + assertThat(cache.currentSnapshot(CATALOG)).isEmpty(); + assertThat(loader.loadCount(CATALOG)).isEqualTo(2); + + CompletableFuture recoveredLoad = loader.expect(CATALOG); + ticker.set(REFRESH_AFTER.plus(FAILURE_BACKOFF).toNanos()); + assertThat(cache.currentSnapshot(CATALOG)).isEmpty(); + assertThat(loader.loadCount(CATALOG)).isEqualTo(3); + recoveredLoad.complete(snapshot(CATALOG, 2)); + assertThat(cache.currentSnapshot(CATALOG)).get().extracting(HogQlSemanticCatalogSnapshot::generation).isEqualTo(2L); + } + + @Test + public void testCatalogEntriesAreIsolatedAndLeastRecentlyUsedEntryIsEvicted() + { + AtomicLong ticker = new AtomicLong(); + ControlledLoader loader = new ControlledLoader(); + BoundedAsyncHogQlSemanticCatalogSnapshotCache cache = cache(2, ticker, loader); + PhysicalIdentifier alpha = catalog("alpha"); + PhysicalIdentifier beta = catalog("beta"); + PhysicalIdentifier gamma = catalog("gamma"); + + completePrewarm(cache, loader, snapshot(alpha, 1)); + completePrewarm(cache, loader, snapshot(beta, 2)); + assertThat(cache.currentSnapshot(alpha)).get().extracting(HogQlSemanticCatalogSnapshot::generation).isEqualTo(1L); + completePrewarm(cache, loader, snapshot(gamma, 3)); + + assertThat(cache.currentSnapshot(alpha)).isPresent(); + assertThat(cache.currentSnapshot(gamma)).isPresent(); + loader.expect(beta); + assertThat(cache.currentSnapshot(beta)).isEmpty(); + assertThat(loader.loadCount(beta)).isEqualTo(2); + } + + @Test + public void testConcurrentRequestsShareOneRefresh() + throws Exception + { + AtomicLong ticker = new AtomicLong(); + ControlledLoader loader = new ControlledLoader(); + CompletableFuture load = loader.expect(CATALOG); + BoundedAsyncHogQlSemanticCatalogSnapshotCache cache = cache(2, ticker, loader); + int workers = 4; + CountDownLatch ready = new CountDownLatch(workers); + CountDownLatch start = new CountDownLatch(1); + + try (ExecutorService callers = Executors.newFixedThreadPool(workers)) { + List>> requests = java.util.stream.IntStream.range(0, workers) + .mapToObj(_ -> CompletableFuture.supplyAsync(() -> { + ready.countDown(); + await(start); + return cache.prewarm(CATALOG); + }, callers)) + .toList(); + assertThat(ready.await(10, SECONDS)).isTrue(); + start.countDown(); + List> refreshes = requests.stream() + .map(CompletableFuture::join) + .toList(); + + assertThat(loader.loadCount(CATALOG)).isEqualTo(1); + assertThat(refreshes).allMatch(refresh -> !refresh.toCompletableFuture().isDone()); + + load.complete(snapshot(CATALOG, 1)); + assertThat(refreshes).allSatisfy(refresh -> assertThat(refresh.toCompletableFuture()) + .isCompletedWithValueMatching(snapshot -> snapshot.generation() == 1)); + } + } + + @Test + public void testPinnedSnapshotRemainsStableAcrossRefreshPublication() + { + AtomicLong ticker = new AtomicLong(); + ControlledLoader loader = new ControlledLoader(); + BoundedAsyncHogQlSemanticCatalogSnapshotCache cache = cache(2, ticker, loader); + completePrewarm(cache, loader, snapshot(CATALOG, 1)); + HogQlSemanticCatalogSnapshotProvider provider = HogQlSemanticCatalogSnapshotProvider.fromCache(cache); + + CompletableFuture load = loader.expect(CATALOG); + ticker.set(REFRESH_AFTER.toNanos()); + PinnedSnapshot pinned = provider.pin(new PinRequest(CATALOG, LANGUAGE_VERSION, OptionalLong.empty())); + load.complete(snapshot(CATALOG, 2)); + + assertThat(pinned.generation()).isEqualTo(1); + assertThat(pinned.snapshot().generation()).isEqualTo(1); + assertThat(provider.pin(new PinRequest(CATALOG, LANGUAGE_VERSION, OptionalLong.of(2))).generation()).isEqualTo(2); + } + + @Test + public void testColdExactGenerationLoadsPinnedEntry() + { + AtomicLong ticker = new AtomicLong(); + ControlledLoader loader = new ControlledLoader(); + CompletableFuture exactLoad = loader.expect(CATALOG, OptionalLong.of(7)); + BoundedAsyncHogQlSemanticCatalogSnapshotCache cache = cache(3, ticker, loader); + + assertThat(cache.currentSnapshot(CATALOG, OptionalLong.of(7))).isEmpty(); + assertThat(loader.loadCount(CATALOG, OptionalLong.of(7))).isEqualTo(1); + assertThat(loader.loadCount(CATALOG)).isZero(); + + exactLoad.complete(snapshot(CATALOG, 7)); + assertThat(cache.currentSnapshot(CATALOG, OptionalLong.of(7))) + .get() + .extracting(HogQlSemanticCatalogSnapshot::generation) + .isEqualTo(7L); + } + + @Test + public void testLatestCanAdvancePastCachedHistoricalGeneration() + { + AtomicLong ticker = new AtomicLong(); + ControlledLoader loader = new ControlledLoader(); + BoundedAsyncHogQlSemanticCatalogSnapshotCache cache = cache(3, ticker, loader); + completePrewarm(cache, loader, snapshot(CATALOG, 1)); + + assertThat(cache.currentSnapshot(CATALOG, OptionalLong.of(1))) + .get() + .extracting(HogQlSemanticCatalogSnapshot::generation) + .isEqualTo(1L); + assertThat(loader.loadCount(CATALOG, OptionalLong.of(1))).isZero(); + + CompletableFuture latestRefresh = loader.expect(CATALOG); + ticker.set(REFRESH_AFTER.toNanos()); + assertThat(cache.currentSnapshot(CATALOG)).isPresent(); + latestRefresh.complete(snapshot(CATALOG, 2)); + + assertThat(cache.currentSnapshot(CATALOG)) + .get() + .extracting(HogQlSemanticCatalogSnapshot::generation) + .isEqualTo(2L); + assertThat(cache.currentSnapshot(CATALOG, OptionalLong.of(1))) + .get() + .extracting(HogQlSemanticCatalogSnapshot::generation) + .isEqualTo(1L); + } + + @Test + public void testExactGenerationChurnPreservesLatestAndCancelsEvictedLoad() + { + AtomicLong ticker = new AtomicLong(); + ControlledLoader loader = new ControlledLoader(); + BoundedAsyncHogQlSemanticCatalogSnapshotCache cache = cache(2, ticker, loader); + completePrewarm(cache, loader, snapshot(CATALOG, 1)); + + CompletableFuture generationSevenLoad = loader.expect(CATALOG, OptionalLong.of(7)); + CompletionStage generationSeven = cache.prewarm(CATALOG, OptionalLong.of(7)); + + CompletableFuture generationEightLoad = loader.expect(CATALOG, OptionalLong.of(8)); + CompletionStage generationEight = cache.prewarm(CATALOG, OptionalLong.of(8)); + + assertThat(generationSevenLoad).isCancelled(); + assertThat(generationSeven.toCompletableFuture()).isCompletedExceptionally(); + assertThat(cache.currentSnapshot(CATALOG)) + .get() + .extracting(HogQlSemanticCatalogSnapshot::generation) + .isEqualTo(1L); + + generationEightLoad.complete(snapshot(CATALOG, 8)); + assertThat(generationEight.toCompletableFuture()).isCompletedWithValueMatching(snapshot -> snapshot.generation() == 8); + } + + @Test + public void testExactGenerationOutageFailsClosedWithoutLatestFallback() + { + AtomicLong ticker = new AtomicLong(); + ControlledLoader loader = new ControlledLoader(); + BoundedAsyncHogQlSemanticCatalogSnapshotCache cache = cache(3, ticker, loader); + + CompletableFuture exactLoad = loader.expect(CATALOG, OptionalLong.of(7)); + assertThat(cache.currentSnapshot(CATALOG, OptionalLong.of(7))).isEmpty(); + exactLoad.complete(snapshot(CATALOG, 7)); + + ticker.set(11); + completePrewarm(cache, loader, snapshot(CATALOG, 8)); + + CompletableFuture failedReload = loader.expect(CATALOG, OptionalLong.of(7)); + ticker.set(EXPIRE_AFTER.toNanos()); + assertThat(cache.currentSnapshot(CATALOG, OptionalLong.of(7))).isEmpty(); + failedReload.completeExceptionally(new IllegalStateException("metadata unavailable")); + + assertThat(cache.currentSnapshot(CATALOG, OptionalLong.of(7))).isEmpty(); + assertThat(loader.loadCount(CATALOG, OptionalLong.of(7))).isEqualTo(2); + assertThat(cache.currentSnapshot(CATALOG)) + .get() + .extracting(HogQlSemanticCatalogSnapshot::generation) + .isEqualTo(8L); + } + + @Test + public void testOrdinaryAndDelimitedCatalogIdentifiersHaveIsolatedEntries() + { + AtomicLong ticker = new AtomicLong(); + ControlledLoader loader = new ControlledLoader(); + BoundedAsyncHogQlSemanticCatalogSnapshotCache cache = cache(2, ticker, loader); + PhysicalIdentifier ordinary = new PhysicalIdentifier("sales", false); + PhysicalIdentifier delimited = new PhysicalIdentifier("sales", true); + + completePrewarm(cache, loader, snapshot(ordinary, 1)); + completePrewarm(cache, loader, snapshot(delimited, 2)); + + assertThat(cache.currentSnapshot(ordinary)).get().extracting(HogQlSemanticCatalogSnapshot::generation).isEqualTo(1L); + assertThat(cache.currentSnapshot(delimited)).get().extracting(HogQlSemanticCatalogSnapshot::generation).isEqualTo(2L); + cache.invalidate(ordinary); + assertThat(cache.currentSnapshot(delimited)).get().extracting(HogQlSemanticCatalogSnapshot::generation).isEqualTo(2L); + } + + @Test + public void testMalformedSchemaV2RefreshRetainsLastKnownGoodSnapshot() + { + AtomicLong ticker = new AtomicLong(); + ArrayDeque> payloads = new ArrayDeque<>(); + HogQlSemanticCatalogSnapshotLoader jsonLoader = HogQlSemanticCatalogSnapshotLoader.fromJsonTransport( + _ -> payloads.remove(), + new HogQlSemanticCatalogSnapshotJsonDecoder()); + BoundedAsyncHogQlSemanticCatalogSnapshotCache cache = cache( + 2, + ticker, + (catalog, expectedGeneration) -> jsonLoader.load(expectedGeneration.isPresent() + ? LoadRequest.pinned(catalog, LANGUAGE_VERSION, expectedGeneration.orElseThrow()) + : LoadRequest.latest(catalog, LANGUAGE_VERSION))); + + CompletableFuture initialPayload = new CompletableFuture<>(); + payloads.add(initialPayload); + CompletionStage initialRefresh = cache.prewarm(CATALOG); + initialPayload.complete(snapshotJson(1).getBytes(StandardCharsets.UTF_8)); + assertThat(initialRefresh.toCompletableFuture()).isCompletedWithValueMatching(snapshot -> snapshot.generation() == 1); + + CompletableFuture malformedPayload = new CompletableFuture<>(); + payloads.add(malformedPayload); + ticker.set(REFRESH_AFTER.toNanos()); + assertThat(cache.currentSnapshot(CATALOG)).get().extracting(HogQlSemanticCatalogSnapshot::generation).isEqualTo(1L); + CompletionStage malformedRefresh = cache.prewarm(CATALOG); + malformedPayload.complete(snapshotJson(2) + .replace("\"logicalTables\": []", "\"logicalTables\": {}") + .getBytes(StandardCharsets.UTF_8)); + + assertThatThrownBy(malformedRefresh.toCompletableFuture()::join) + .cause() + .isInstanceOf(HogQlSemanticCatalogSnapshotJsonDecoder.DecodeException.class); + assertThat(cache.currentSnapshot(CATALOG)).get().extracting(HogQlSemanticCatalogSnapshot::generation).isEqualTo(1L); + } + + private static BoundedAsyncHogQlSemanticCatalogSnapshotCache cache(int maximumEntries, AtomicLong ticker, SnapshotLoader loader) + { + return new BoundedAsyncHogQlSemanticCatalogSnapshotCache( + maximumEntries, + REFRESH_AFTER, + EXPIRE_AFTER, + FAILURE_BACKOFF, + ticker::get, + Runnable::run, + loader); + } + + private static void completePrewarm( + BoundedAsyncHogQlSemanticCatalogSnapshotCache cache, + ControlledLoader loader, + HogQlSemanticCatalogSnapshot snapshot) + { + CompletableFuture load = loader.expect(snapshot.catalog()); + CompletionStage refresh = cache.prewarm(snapshot.catalog()); + load.complete(snapshot); + assertThat(refresh.toCompletableFuture()).isCompletedWithValue(snapshot); + } + + private static void assertRefreshFailure(CompletionStage refresh, Failure failure) + { + assertThatThrownBy(refresh.toCompletableFuture()::join) + .cause() + .isInstanceOfSatisfying(HogQlSemanticCatalogException.class, exception -> assertThat(exception.failure()).isEqualTo(failure)); + } + + private static void await(CountDownLatch latch) + { + try { + assertThat(latch.await(10, SECONDS)).isTrue(); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + + private static PhysicalIdentifier catalog(String name) + { + return new PhysicalIdentifier(name, false); + } + + private static HogQlSemanticCatalogSnapshot snapshot(PhysicalIdentifier catalog, long generation) + { + return snapshot(catalog, generation, "id"); + } + + private static HogQlSemanticCatalogSnapshot snapshot(PhysicalIdentifier catalog, long generation, String fieldName) + { + return new HogQlSemanticCatalogSnapshot( + 2, + LANGUAGE_VERSION, + catalog, + generation, + List.of(new LogicalTableDefinition( + "events", + new PhysicalQualifiedName(catalog, catalog("default"), catalog("events")), + List.of(new LogicalFieldDefinition( + fieldName, + catalog(fieldName), + "varchar", + LogicalType.STRING, + false, + true)), + List.of(), + List.of()))); + } + + private static String snapshotJson(long generation) + { + return """ + { + "protocolVersion": 1, + "schemaVersion": 2, + "languageVersion": "1.0.0", + "catalog": {"value": "ducklake", "delimited": false}, + "generation": %s, + "logicalTables": [], + "expressionFields": [], + "virtualTables": [], + "savedQueries": [], + "materializedViews": [], + "functions": [], + "modifierDefaults": [] + } + """.formatted(generation); + } + + private static final class ControlledLoader + implements SnapshotLoader + { + private final Map>> loads = new HashMap<>(); + private final Map loadCounts = new HashMap<>(); + + public CompletableFuture expect(PhysicalIdentifier catalog) + { + return expect(catalog, OptionalLong.empty()); + } + + public CompletableFuture expect(PhysicalIdentifier catalog, OptionalLong expectedGeneration) + { + CompletableFuture load = new CompletableFuture<>(); + loads.computeIfAbsent(new LoadKey(catalog, expectedGeneration), _ -> new ArrayDeque<>()).add(load); + return load; + } + + @Override + public CompletionStage load(PhysicalIdentifier catalog, OptionalLong expectedGeneration) + { + LoadKey key = new LoadKey(catalog, expectedGeneration); + loadCounts.computeIfAbsent(key, _ -> new AtomicInteger()).incrementAndGet(); + return Optional.ofNullable(loads.get(key)) + .map(ArrayDeque::poll) + .orElseThrow(() -> new IllegalStateException("unexpected load for catalog " + catalog.value() + " and generation " + expectedGeneration)); + } + + public int loadCount(PhysicalIdentifier catalog) + { + return loadCount(catalog, OptionalLong.empty()); + } + + public int loadCount(PhysicalIdentifier catalog, OptionalLong expectedGeneration) + { + return Optional.ofNullable(loadCounts.get(new LoadKey(catalog, expectedGeneration))) + .map(AtomicInteger::get) + .orElse(0); + } + } + + private record LoadKey(PhysicalIdentifier catalog, OptionalLong expectedGeneration) {} +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlExchangeRateSnapshotJsonDecoder.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlExchangeRateSnapshotJsonDecoder.java new file mode 100644 index 000000000000..a64dc486acf1 --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlExchangeRateSnapshotJsonDecoder.java @@ -0,0 +1,106 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotJsonDecoder.DecodeException; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotJsonDecoder.DecodeFailure; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotLoader.LoadRequest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.nio.charset.StandardCharsets; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlExchangeRateSnapshotJsonDecoder +{ + private final HogQlExchangeRateSnapshotJsonDecoder decoder = new HogQlExchangeRateSnapshotJsonDecoder(); + + @Test + public void testDecodesCanonicalLatestAndPinnedSnapshots() + { + HogQlExchangeRateSnapshot latest = decoder.decode(snapshotJson(7).getBytes(StandardCharsets.UTF_8), LoadRequest.latest()); + HogQlExchangeRateSnapshot pinned = decoder.decode(snapshotJson(7).getBytes(StandardCharsets.UTF_8), LoadRequest.pinned(7)); + + assertThat(latest).isEqualTo(pinned); + assertThat(latest.generation()).isEqualTo(7); + assertThat(latest.rates()).containsExactly( + new HogQlExchangeRateSnapshot.ExchangeRate("EUR", "2024-01-01", "9049000000"), + new HogQlExchangeRateSnapshot.ExchangeRate("USD", "1970-01-01", "10000000000")); + assertThatThrownBy(() -> latest.rates().clear()).isInstanceOf(UnsupportedOperationException.class); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("invalidDocuments") + public void testRejectsNoncanonicalDocuments(String name, String document, LoadRequest request, DecodeFailure failure) + { + assertThatThrownBy(() -> decoder.decode(document.getBytes(StandardCharsets.UTF_8), request)) + .isInstanceOfSatisfying(DecodeException.class, exception -> { + assertThat(exception.failure()).isEqualTo(failure); + assertThat(exception.getMessage()).hasSizeLessThan(128); + }); + } + + @Test + public void testRejectsPayloadOver32MiB() + { + byte[] payload = new byte[HogQlExchangeRateSnapshotJsonDecoder.MAXIMUM_PAYLOAD_BYTES + 1]; + + assertThatThrownBy(() -> decoder.decode(payload, LoadRequest.latest())) + .isInstanceOfSatisfying(DecodeException.class, exception -> assertThat(exception.failure()).isEqualTo(DecodeFailure.LIMIT_EXCEEDED)); + } + + private static Stream invalidDocuments() + { + String valid = snapshotJson(7); + return Stream.of( + Arguments.of("unknown field", valid.replace("\"generation\": 7", "\"generation\": 7, \"unknown\": true"), LoadRequest.latest(), DecodeFailure.INVALID_PAYLOAD), + Arguments.of("missing field", valid.replace("\"decimalScale\": 10,", ""), LoadRequest.latest(), DecodeFailure.INVALID_PAYLOAD), + Arguments.of("duplicate field", valid.replace("\"generation\": 7", "\"generation\": 7, \"generation\": 7"), LoadRequest.latest(), DecodeFailure.INVALID_PAYLOAD), + Arguments.of("trailing document", valid + "{}", LoadRequest.latest(), DecodeFailure.INVALID_PAYLOAD), + Arguments.of("null document", "null", LoadRequest.latest(), DecodeFailure.INVALID_PAYLOAD), + Arguments.of("unsupported protocol", valid.replace("\"protocolVersion\": 1", "\"protocolVersion\": 2"), LoadRequest.latest(), DecodeFailure.UNSUPPORTED_PROTOCOL), + Arguments.of("unsupported schema", valid.replace("\"schemaVersion\": 1", "\"schemaVersion\": 2"), LoadRequest.latest(), DecodeFailure.UNSUPPORTED_SCHEMA), + Arguments.of("generation mismatch", valid, LoadRequest.pinned(8), DecodeFailure.GENERATION_MISMATCH), + Arguments.of("nonpositive generation", valid.replace("\"generation\": 7", "\"generation\": 0"), LoadRequest.latest(), DecodeFailure.INVALID_PAYLOAD), + Arguments.of("wrong base", valid.replace("\"baseCurrency\": \"USD\"", "\"baseCurrency\": \"EUR\""), LoadRequest.latest(), DecodeFailure.INVALID_PAYLOAD), + Arguments.of("wrong scale", valid.replace("\"decimalScale\": 10", "\"decimalScale\": 9"), LoadRequest.latest(), DecodeFailure.INVALID_PAYLOAD), + Arguments.of("lowercase currency", valid.replace("\"EUR\"", "\"eur\""), LoadRequest.latest(), DecodeFailure.INVALID_PAYLOAD), + Arguments.of("invalid date", valid.replace("2024-01-01", "2024-02-30"), LoadRequest.latest(), DecodeFailure.INVALID_PAYLOAD), + Arguments.of("leading zero", valid.replace("9049000000", "09049000000"), LoadRequest.latest(), DecodeFailure.INVALID_PAYLOAD), + Arguments.of("base rate differs", valid.replace("10000000000", "9999999999"), LoadRequest.latest(), DecodeFailure.INVALID_PAYLOAD), + Arguments.of("unsorted rates", valid.replace("\"EUR\", \"effectiveDate\": \"2024-01-01\", \"unscaledRate\": \"9049000000\"", "\"ZZZ\", \"effectiveDate\": \"2024-01-01\", \"unscaledRate\": \"9049000000\""), LoadRequest.latest(), DecodeFailure.INVALID_PAYLOAD)); + } + + private static String snapshotJson(long generation) + { + return """ + { + "protocolVersion": 1, + "schemaVersion": 1, + "generation": %s, + "baseCurrency": "USD", + "decimalScale": 10, + "rates": [ + {"currency": "EUR", "effectiveDate": "2024-01-01", "unscaledRate": "9049000000"}, + {"currency": "USD", "effectiveDate": "1970-01-01", "unscaledRate": "10000000000"} + ] + } + """.formatted(generation); + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlExchangeRateSnapshotLoader.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlExchangeRateSnapshotLoader.java new file mode 100644 index 000000000000..3e1b5f90825d --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlExchangeRateSnapshotLoader.java @@ -0,0 +1,61 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotLoader.JsonTransport; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotLoader.LoadRequest; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlExchangeRateSnapshotLoader +{ + @Test + public void testPreservesLatestAndPinnedSemanticsThroughJsonTransport() + { + List requests = new ArrayList<>(); + JsonTransport transport = request -> { + requests.add(request); + long generation = request.expectedGeneration().orElse(9); + return CompletableFuture.completedFuture(snapshotJson(generation).getBytes(StandardCharsets.UTF_8)); + }; + HogQlExchangeRateSnapshotLoader loader = HogQlExchangeRateSnapshotLoader.fromJsonTransport( + transport, + new HogQlExchangeRateSnapshotJsonDecoder()); + + assertThat(loader.load(LoadRequest.latest()).toCompletableFuture().join().generation()).isEqualTo(9); + assertThat(loader.load(LoadRequest.pinned(7)).toCompletableFuture().join().generation()).isEqualTo(7); + assertThat(requests).containsExactly(LoadRequest.latest(), LoadRequest.pinned(7)); + } + + @Test + public void testRejectsInvalidPinnedGenerationBeforeTransport() + { + assertThatThrownBy(() -> LoadRequest.pinned(0)).isInstanceOf(IllegalArgumentException.class); + } + + private static String snapshotJson(long generation) + { + return """ + {"protocolVersion":1,"schemaVersion":1,"generation":%s,"baseCurrency":"USD","decimalScale":10, + "rates":[{"currency":"USD","effectiveDate":"1970-01-01","unscaledRate":"10000000000"}]} + """.formatted(generation); + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlExchangeRateSnapshotProvider.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlExchangeRateSnapshotProvider.java new file mode 100644 index 000000000000..126b2ee7af9a --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlExchangeRateSnapshotProvider.java @@ -0,0 +1,69 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.compiler.catalog.HogQlExchangeRateException.Failure; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshot.ExchangeRate; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.atomic.AtomicReference; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlExchangeRateSnapshotProvider +{ + @Test + public void testPinsOneExactGeneration() + { + AtomicReference requestedGeneration = new AtomicReference<>(); + HogQlExchangeRateSnapshotCache cache = expectedGeneration -> { + requestedGeneration.set(expectedGeneration); + return Optional.of(snapshot(7)); + }; + HogQlExchangeRateSnapshotProvider provider = HogQlExchangeRateSnapshotProvider.fromCache(cache); + + HogQlExchangeRateSnapshotProvider.PinnedSnapshot pinned = provider.pin(OptionalLong.of(7)); + + assertThat(requestedGeneration).hasValue(OptionalLong.of(7)); + assertThat(pinned.generation()).isEqualTo(7); + assertThat(pinned.snapshot()).isEqualTo(snapshot(7)); + } + + @Test + public void testFailsClosedForUnavailableAndMismatchedGeneration() + { + HogQlExchangeRateSnapshotProvider unavailable = HogQlExchangeRateSnapshotProvider.fromCache(_ -> Optional.empty()); + assertThatThrownBy(() -> unavailable.pin(OptionalLong.empty())) + .isInstanceOfSatisfying(HogQlExchangeRateException.class, exception -> assertThat(exception.failure()).isEqualTo(Failure.UNAVAILABLE)); + + HogQlExchangeRateSnapshotProvider mismatched = HogQlExchangeRateSnapshotProvider.fromCache(_ -> Optional.of(snapshot(2))); + assertThatThrownBy(() -> mismatched.pin(OptionalLong.of(1))) + .isInstanceOfSatisfying(HogQlExchangeRateException.class, exception -> assertThat(exception.failure()).isEqualTo(Failure.GENERATION_MISMATCH)); + } + + private static HogQlExchangeRateSnapshot snapshot(long generation) + { + return new HogQlExchangeRateSnapshot( + 1, + 1, + generation, + "USD", + 10, + List.of(new ExchangeRate("USD", "1970-01-01", "10000000000"))); + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java new file mode 100644 index 000000000000..c9c370a4cb19 --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshot.java @@ -0,0 +1,718 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ActionReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ArgumentReferenceRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.CastRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.CohortReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionArgument; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCallRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionCapabilityDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionImplementation; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionRewrite; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionSignature; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.JoinKey; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LazyProjectionDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LazyTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralEncoding; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ModifierBehavior; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.OperatorRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PredicateRepresentation; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyLookupRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyStorage; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationMembershipRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationMembershipRepresentation; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipCardinality; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipJoinSide; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ScopedFieldReferenceRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticModifierDefault; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticOperator; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.TypedLiteral; +import io.trino.hogql.parser.HogQlLanguageVersion; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlSemanticCatalogSnapshot +{ + private static final HogQlLanguageVersion LANGUAGE_VERSION = HogQlLanguageVersion.valueOf("1.0.0"); + private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier("ducklake", false); + + @Test + public void testDoesNotExposePublisherMutability() + { + List fields = new ArrayList<>(List.of(field("id"))); + List properties = new ArrayList<>(List.of(property("properties", "id"))); + List joinKeys = new ArrayList<>(List.of(new JoinKey("id", "id"))); + List relationships = new ArrayList<>(List.of( + new RelationshipDefinition("person", "persons", RelationshipCardinality.MANY_TO_ONE, joinKeys))); + List tables = new ArrayList<>(List.of( + table("events", fields, properties, relationships), + table("persons"))); + HogQlSemanticCatalogSnapshot snapshot = snapshot(tables); + + fields.add(field("event")); + properties.clear(); + joinKeys.clear(); + relationships.clear(); + tables.clear(); + + assertThat(snapshot.logicalTables()).hasSize(2); + assertThat(snapshot.logicalTable("events")).get().satisfies(table -> { + assertThat(table.fields()).hasSize(1); + assertThat(table.properties()).hasSize(1); + assertThat(table.relationships()).singleElement().satisfies(relationship -> assertThat(relationship.joinKeys()).hasSize(1)); + }); + assertThatThrownBy(() -> snapshot.logicalTables().add(table("persons"))) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> snapshot.logicalTable("events").orElseThrow().fields().add(field("event"))) + .isInstanceOf(UnsupportedOperationException.class); + assertThatThrownBy(() -> snapshot.logicalTable("events").orElseThrow().relationships().getFirst().joinKeys().clear()) + .isInstanceOf(UnsupportedOperationException.class); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("invalidSnapshots") + public void testRejectsInvalidDefinitions(String name, List tables, String message) + { + assertThatThrownBy(() -> snapshot(tables)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining(message); + } + + @Test + public void testAcceptsBidirectionalLazyRelationships() + { + HogQlSemanticCatalogSnapshot snapshot = snapshot(List.of( + table("events", List.of(field("id")), List.of(), List.of(relationship("person", "persons", "id", "id"))), + table("persons", List.of(field("id")), List.of(), List.of(relationship("events", "events", "id", "id"))))); + + assertThat(snapshot.logicalTable("events")).get() + .extracting(LogicalTableDefinition::relationships) + .satisfies(relationships -> assertThat(relationships).singleElement().extracting(RelationshipDefinition::targetTable).isEqualTo("persons")); + assertThat(snapshot.logicalTable("persons")).get() + .extracting(LogicalTableDefinition::relationships) + .satisfies(relationships -> assertThat(relationships).singleElement().extracting(RelationshipDefinition::targetTable).isEqualTo("events")); + } + + @Test + public void testSemanticDefinitionsAreDeeplyImmutable() + { + List arguments = new ArrayList<>(List.of(literal())); + FunctionCallRecipe recipe = new FunctionCallRecipe("identity", arguments); + List expressionFields = new ArrayList<>(List.of( + new ExpressionFieldDefinition("events", "derived", "bigint", LogicalType.INTEGER, false, true, recipe))); + List signatures = new ArrayList<>(List.of(new FunctionSignature(List.of("bigint"), "bigint", false))); + List functions = new ArrayList<>(List.of( + new FunctionCapabilityDefinition( + "identity", + FunctionKind.SCALAR, + FunctionImplementation.STOCK, + List.of(new PhysicalIdentifier("identity", false)), + signatures, + true, + false, + false, + false, + false))); + + HogQlSemanticCatalogSnapshot snapshot = semanticSnapshot(expressionFields, functions); + arguments.clear(); + expressionFields.clear(); + signatures.clear(); + functions.clear(); + + assertThat(snapshot.expressionFields()).singleElement().extracting(ExpressionFieldDefinition::recipe).isEqualTo(recipe); + assertThat(recipe.arguments()).singleElement().isEqualTo(literal()); + assertThat(snapshot.functions()).singleElement().extracting(FunctionCapabilityDefinition::signatures).satisfies(values -> assertThat(values).hasSize(1)); + assertThatThrownBy(() -> recipe.arguments().clear()).isInstanceOf(UnsupportedOperationException.class); + } + + @Test + public void testValidatesFunctionRewriteContract() + { + FunctionCapabilityDefinition isNull = rewriteFunction("isNull", FunctionKind.SCALAR, List.of(), Optional.of(FunctionRewrite.IS_NULL), List.of(new FunctionSignature(List.of("varchar"), "boolean", false))); + FunctionCapabilityDefinition isNotNull = rewriteFunction("isNotNull", FunctionKind.SCALAR, List.of(), Optional.of(FunctionRewrite.IS_NOT_NULL), List.of(new FunctionSignature(List.of("varchar"), "boolean", false))); + FunctionCapabilityDefinition countIf = rewriteFunction("countIf", FunctionKind.AGGREGATE, List.of(), Optional.of(FunctionRewrite.COUNT_IF), List.of(new FunctionSignature(List.of("boolean"), "bigint", false))); + FunctionCapabilityDefinition multiIf = rewriteFunction("multiIf", FunctionKind.SCALAR, List.of(), Optional.of(FunctionRewrite.MULTI_IF), List.of(new FunctionSignature(List.of("boolean", "any", "boolean", "any"), "any", true))); + + HogQlSemanticCatalogSnapshot snapshot = semanticSnapshot(List.of(), List.of(isNull, isNotNull, countIf, multiIf)); + + assertThat(snapshot.functions()).extracting(FunctionCapabilityDefinition::rewrite) + .containsExactly( + Optional.of(FunctionRewrite.IS_NULL), + Optional.of(FunctionRewrite.IS_NOT_NULL), + Optional.of(FunctionRewrite.COUNT_IF), + Optional.of(FunctionRewrite.MULTI_IF)); + assertThatThrownBy(() -> semanticSnapshot(List.of(), List.of( + rewriteFunction("missing", FunctionKind.SCALAR, List.of(), Optional.empty(), List.of(new FunctionSignature(List.of("varchar"), "boolean", false)))))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("must declare a rewrite"); + assertThatThrownBy(() -> semanticSnapshot(List.of(), List.of( + rewriteFunction("named", FunctionKind.SCALAR, List.of(new PhysicalIdentifier("named", false)), Optional.of(FunctionRewrite.IS_NULL), List.of(new FunctionSignature(List.of("varchar"), "boolean", false)))))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("cannot name a Trino function"); + assertThatThrownBy(() -> semanticSnapshot(List.of(), List.of( + new FunctionCapabilityDefinition( + "stock", + FunctionKind.SCALAR, + FunctionImplementation.STOCK, + List.of(new PhysicalIdentifier("stock", false)), + Optional.of(FunctionRewrite.IS_NULL), + List.of(new FunctionSignature(List.of("varchar"), "boolean", false)), + true, + false, + false, + false, + false)))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("cannot declare a rewrite"); + assertThatThrownBy(() -> semanticSnapshot(List.of(), List.of( + rewriteFunction("aggregate", FunctionKind.AGGREGATE, List.of(), Optional.of(FunctionRewrite.IS_NULL), List.of(new FunctionSignature(List.of("varchar"), "boolean", false)))))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("must be scalar"); + assertThatThrownBy(() -> semanticSnapshot(List.of(), List.of( + rewriteFunction("binary", FunctionKind.SCALAR, List.of(), Optional.of(FunctionRewrite.IS_NULL), List.of(new FunctionSignature(List.of("varchar", "varchar"), "boolean", false)))))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("invalid signature"); + assertThatThrownBy(() -> semanticSnapshot(List.of(), List.of( + rewriteFunction("scalarCountIf", FunctionKind.SCALAR, List.of(), Optional.of(FunctionRewrite.COUNT_IF), List.of(new FunctionSignature(List.of("boolean"), "bigint", false)))))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("kind must be aggregate"); + assertThatThrownBy(() -> semanticSnapshot(List.of(), List.of( + rewriteFunction("fixedMultiIf", FunctionKind.SCALAR, List.of(), Optional.of(FunctionRewrite.MULTI_IF), List.of(new FunctionSignature(List.of("boolean", "any", "any"), "any", false)))))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("invalid signature"); + assertInvalidRewrite(rewriteFunction(false, false, false, false, false, "boolean"), "must be deterministic"); + assertInvalidRewrite(rewriteFunction(true, true, false, false, false, "boolean"), "cannot support DISTINCT"); + assertInvalidRewrite(rewriteFunction(true, false, true, false, false, "boolean"), "cannot support ORDER BY"); + assertInvalidRewrite(rewriteFunction(true, false, false, true, false, "boolean"), "cannot support FILTER"); + assertInvalidRewrite(rewriteFunction(true, false, false, false, true, "boolean"), "cannot support window invocation"); + assertInvalidRewrite(rewriteFunction(true, false, false, false, false, "varchar"), "must return boolean"); + } + + @Test + public void testRejectsOverqualifiedModifierSessionProperty() + { + assertThatThrownBy(() -> semanticSnapshotWithModifiers(List.of(new SemanticModifierDefault( + "sampling", + ModifierBehavior.TRINO_SESSION_PROPERTY, + new TypedLiteral("boolean", LiteralEncoding.BOOLEAN, "false"), + List.of( + new PhysicalIdentifier("catalog", false), + new PhysicalIdentifier("schema", false), + new PhysicalIdentifier("property", false)))))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("invalid session property name"); + } + + @Test + public void testLogicalSemanticRecipesAreDeeplyImmutable() + { + List lookupArguments = new ArrayList<>(List.of( + new ArgumentReferenceRecipe(ExpressionArgument.PROPERTY_SOURCE), + new ArgumentReferenceRecipe(ExpressionArgument.PROPERTY_KEY))); + PropertyDefinition property = propertyWithLookup(new OperatorRecipe(SemanticOperator.SUBSCRIPT, lookupArguments)); + List relationshipPath = new ArrayList<>(List.of("self")); + List projections = new ArrayList<>(List.of(new LazyProjectionDefinition( + "browser", + "varchar", + LogicalType.STRING, + true, + true, + propertyLookup("events", "properties")))); + List lazyTables = new ArrayList<>(List.of(new LazyTableDefinition("events", "profile", relationshipPath, projections))); + List actions = new ArrayList<>(List.of(new ActionReference( + "paid", + "action-1", + "events", + new PredicateRepresentation(propertyLookup("events", "properties"))))); + List cohorts = new ArrayList<>(List.of(new CohortReference( + "active", + "cohort-1", + "events", + new RelationMembershipRepresentation(new RelationMembershipRecipe( + new RelationReference(RelationKind.LOGICAL_TABLE, "events"), + "properties", + "properties"))))); + + HogQlSemanticCatalogSnapshot snapshot = logicalSemanticSnapshot(property, relationshipPathPredicate(), lazyTables, actions, cohorts); + lookupArguments.clear(); + relationshipPath.clear(); + projections.clear(); + lazyTables.clear(); + actions.clear(); + cohorts.clear(); + + assertThat(snapshot.logicalTables().getFirst().properties().getFirst().lookupRecipe()).isPresent(); + assertThat(snapshot.logicalTables().getFirst().relationships().getFirst().joinPredicate()).isPresent(); + assertThat(snapshot.lazyTables()).singleElement().satisfies(lazy -> { + assertThat(lazy.relationshipPath()).containsExactly("self"); + assertThat(lazy.projections()).hasSize(1); + }); + assertThat(snapshot.actions()).hasSize(1); + assertThat(snapshot.cohorts()).hasSize(1); + assertThatThrownBy(() -> snapshot.lazyTables().getFirst().relationshipPath().clear()) + .isInstanceOf(UnsupportedOperationException.class); + } + + @Test + public void testRejectsInvalidLogicalSemanticScopes() + { + assertThatThrownBy(() -> logicalSemanticSnapshot( + propertyWithLookup(new ArgumentReferenceRecipe(ExpressionArgument.PROPERTY_SOURCE)), + relationshipPathPredicate(), + List.of(), + List.of(), + List.of())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("source and key arguments"); + + assertThatThrownBy(() -> logicalSemanticSnapshot( + propertyWithLookup(validLookupRecipe()), + Optional.empty(), + List.of(), + List.of(new ActionReference( + "invalid", + "action-1", + "events", + new PredicateRepresentation(new ScopedFieldReferenceRecipe(RelationshipJoinSide.SOURCE, "properties")))), + List.of())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("scoped field reference"); + + assertThatThrownBy(() -> logicalSemanticSnapshot( + propertyWithLookup(validLookupRecipe()), + relationshipPathPredicate(), + List.of(new LazyTableDefinition( + "events", + "profile", + List.of("missing"), + List.of(new LazyProjectionDefinition("browser", "varchar", LogicalType.STRING, true, true, propertyLookup("events", "properties"))))), + List.of(), + List.of())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("unknown relationship"); + + assertThatThrownBy(() -> logicalSemanticSnapshot( + propertyWithLookup(validLookupRecipe()), + relationshipPathPredicate(), + List.of(), + List.of(), + List.of(new CohortReference( + "invalid", + "cohort-1", + "events", + new RelationMembershipRepresentation(new RelationMembershipRecipe( + new RelationReference(RelationKind.LOGICAL_TABLE, "events"), + "properties", + "missing")))))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("unknown target field"); + } + + @Test + public void testEnforcesExpressionRecipeDepthAndNodeLimits() + { + ExpressionRecipe deepRecipe = literal(); + for (int depth = 1; depth < 65; depth++) { + deepRecipe = new CastRecipe(deepRecipe, "bigint"); + } + ExpressionRecipe finalDeepRecipe = deepRecipe; + assertThatThrownBy(() -> semanticSnapshot( + List.of(new ExpressionFieldDefinition("events", "derived", "bigint", LogicalType.INTEGER, false, true, finalDeepRecipe)), + List.of())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("depth limit"); + + List arguments = new ArrayList<>(); + for (int node = 0; node < 4_096; node++) { + arguments.add(literal()); + } + assertThatThrownBy(() -> semanticSnapshot( + List.of(new ExpressionFieldDefinition("events", "derived", "bigint", LogicalType.INTEGER, false, true, new FunctionCallRecipe("identity", arguments))), + List.of(variadicFunction("identity")))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("node limit"); + } + + @ParameterizedTest(name = "{0} with {1} arguments") + @MethodSource("invalidOperatorArities") + public void testRejectsInvalidOperatorArity(SemanticOperator operator, int argumentCount) + { + List arguments = new ArrayList<>(); + for (int index = 0; index < argumentCount; index++) { + arguments.add(literal()); + } + + assertThatThrownBy(() -> semanticSnapshot( + List.of(new ExpressionFieldDefinition( + "events", + "derived", + "bigint", + LogicalType.INTEGER, + false, + true, + new OperatorRecipe(operator, arguments))), + List.of())) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("operator arity"); + } + + @Test + public void testRejectsUnsupportedRecipeFunctionArity() + { + assertThatThrownBy(() -> semanticSnapshot( + List.of(new ExpressionFieldDefinition( + "events", + "derived", + "bigint", + LogicalType.INTEGER, + false, + true, + new FunctionCallRecipe("identity", List.of()))), + List.of(function("identity")))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("unsupported argument count"); + } + + private static Stream invalidOperatorArities() + { + return Stream.concat( + Stream.of(SemanticOperator.values()) + .filter(operator -> switch (operator) { + case NOT, NEGATE, IS_NULL, IS_NOT_NULL -> false; + default -> true; + }) + .map(operator -> Arguments.of(operator, 1)), + Stream.of(SemanticOperator.NOT, SemanticOperator.NEGATE, SemanticOperator.IS_NULL, SemanticOperator.IS_NOT_NULL) + .map(operator -> Arguments.of(operator, 2))); + } + + private static Stream invalidSnapshots() + { + LogicalTableDefinition events = table("events"); + LogicalTableDefinition persons = table("persons"); + return Stream.of( + Arguments.of("duplicate tables", List.of(events, events), "duplicate logical table"), + Arguments.of( + "duplicate members", + List.of(table("events", List.of(field("id"), field("ID")), List.of(), List.of())), + "duplicate logical member"), + Arguments.of( + "property collides with another field", + List.of(table("events", List.of(field("id"), field("properties")), List.of(property("id", "properties")), List.of())), + "duplicate logical member"), + Arguments.of( + "missing property field", + List.of(table("events", List.of(field("id")), List.of(property("properties", "missing")), List.of())), + "unknown source field"), + Arguments.of( + "missing relationship table", + List.of(table("events", List.of(field("id")), List.of(), List.of(relationship("person", "persons", "id", "id")))), + "unknown target table"), + Arguments.of( + "missing relationship source field", + List.of( + table("events", List.of(field("id")), List.of(), List.of(relationship("person", "persons", "missing", "id"))), + persons), + "unknown source field"), + Arguments.of( + "missing relationship target field", + List.of( + table("events", List.of(field("id")), List.of(), List.of(relationship("person", "persons", "id", "missing"))), + persons), + "unknown target field")); + } + + @ParameterizedTest + @ValueSource(strings = { + "events; DROP TABLE events", + "events\nSELECT 1", + "events/* injected */", + "events-- injected", + }) + public void testRejectsRawExecutablePhysicalDefinitions(String definition) + { + for (boolean delimited : List.of(false, true)) { + assertThatThrownBy(() -> new PhysicalIdentifier(definition, delimited)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("physical identifier"); + } + } + + private static HogQlSemanticCatalogSnapshot snapshot(List tables) + { + return new HogQlSemanticCatalogSnapshot(2, LANGUAGE_VERSION, CATALOG, 7, tables); + } + + private static HogQlSemanticCatalogSnapshot semanticSnapshot( + List expressionFields, + List functions) + { + return new HogQlSemanticCatalogSnapshot( + 1, + 2, + LANGUAGE_VERSION, + CATALOG, + 7, + List.of(table("events")), + expressionFields, + List.of(), + List.of(), + List.of(), + functions, + List.of()); + } + + private static HogQlSemanticCatalogSnapshot semanticSnapshotWithModifiers(List modifiers) + { + return new HogQlSemanticCatalogSnapshot( + 1, + 2, + LANGUAGE_VERSION, + CATALOG, + 7, + List.of(table("events")), + List.of(), + List.of(), + List.of(), + List.of(), + List.of(), + modifiers); + } + + private static LiteralRecipe literal() + { + return new LiteralRecipe(new TypedLiteral("bigint", LiteralEncoding.INTEGER, "1")); + } + + private static FunctionCapabilityDefinition function(String name) + { + return new FunctionCapabilityDefinition( + name, + FunctionKind.SCALAR, + FunctionImplementation.STOCK, + List.of(new PhysicalIdentifier(name, false)), + List.of(new FunctionSignature(List.of("bigint"), "bigint", false)), + true, + false, + false, + false, + false); + } + + private static FunctionCapabilityDefinition variadicFunction(String name) + { + return new FunctionCapabilityDefinition( + name, + FunctionKind.SCALAR, + FunctionImplementation.STOCK, + List.of(new PhysicalIdentifier(name, false)), + List.of(new FunctionSignature(List.of("bigint"), "bigint", true)), + true, + false, + false, + false, + false); + } + + private static FunctionCapabilityDefinition rewriteFunction( + String name, + FunctionKind kind, + List trinoName, + Optional rewrite, + List signatures) + { + return new FunctionCapabilityDefinition( + name, + kind, + FunctionImplementation.REWRITE, + trinoName, + rewrite, + signatures, + true, + false, + false, + false, + false); + } + + private static FunctionCapabilityDefinition rewriteFunction( + boolean deterministic, + boolean supportsDistinct, + boolean supportsOrderBy, + boolean supportsFilter, + boolean supportsWindow, + String returnType) + { + return new FunctionCapabilityDefinition( + "rewrite", + FunctionKind.SCALAR, + FunctionImplementation.REWRITE, + List.of(), + Optional.of(FunctionRewrite.IS_NULL), + List.of(new FunctionSignature(List.of("varchar"), returnType, false)), + deterministic, + supportsDistinct, + supportsOrderBy, + supportsFilter, + supportsWindow); + } + + private static void assertInvalidRewrite(FunctionCapabilityDefinition function, String message) + { + assertThatThrownBy(() -> semanticSnapshot(List.of(), List.of(function))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining(message); + } + + private static HogQlSemanticCatalogSnapshot logicalSemanticSnapshot( + PropertyDefinition property, + Optional joinPredicate, + List lazyTables, + List actions, + List cohorts) + { + return new HogQlSemanticCatalogSnapshot( + 1, + 2, + LANGUAGE_VERSION, + CATALOG, + 7, + List.of(table( + "events", + List.of(field("properties")), + List.of(property), + List.of(new RelationshipDefinition( + "self", + "events", + RelationshipCardinality.MANY_TO_ONE, + List.of(new JoinKey("properties", "properties")), + joinPredicate)))), + List.of(), + List.of(), + List.of(), + List.of(), + List.of(), + List.of(), + lazyTables, + actions, + cohorts); + } + + private static PropertyDefinition propertyWithLookup(ExpressionRecipe recipe) + { + return new PropertyDefinition( + "properties", + "properties", + PropertyStorage.JSON_OBJECT, + LogicalType.STRING, + true, + Optional.of("varchar"), + Optional.of("varchar"), + Optional.of(recipe)); + } + + private static OperatorRecipe validLookupRecipe() + { + return new OperatorRecipe( + SemanticOperator.SUBSCRIPT, + List.of( + new ArgumentReferenceRecipe(ExpressionArgument.PROPERTY_SOURCE), + new ArgumentReferenceRecipe(ExpressionArgument.PROPERTY_KEY))); + } + + private static Optional relationshipPathPredicate() + { + return Optional.of(new OperatorRecipe( + SemanticOperator.EQUAL, + List.of( + new ScopedFieldReferenceRecipe(RelationshipJoinSide.SOURCE, "properties"), + new ScopedFieldReferenceRecipe(RelationshipJoinSide.TARGET, "properties")))); + } + + private static PropertyLookupRecipe propertyLookup(String table, String property) + { + return new PropertyLookupRecipe(table, property, new LiteralRecipe(new TypedLiteral("varchar", LiteralEncoding.STRING, "browser"))); + } + + private static LogicalTableDefinition table(String name) + { + return table(name, List.of(field("id")), List.of(), List.of()); + } + + private static LogicalTableDefinition table( + String name, + List fields, + List properties, + List relationships) + { + return new LogicalTableDefinition( + name, + new PhysicalQualifiedName(CATALOG, new PhysicalIdentifier("default", false), new PhysicalIdentifier(name, false)), + fields, + properties, + relationships); + } + + private static LogicalFieldDefinition field(String name) + { + return new LogicalFieldDefinition( + name, + new PhysicalIdentifier(name, false), + "varchar", + LogicalType.STRING, + true, + true); + } + + private static PropertyDefinition property(String name, String sourceField) + { + return new PropertyDefinition(name, sourceField, PropertyStorage.JSON_OBJECT, LogicalType.JSON, true); + } + + private static RelationshipDefinition relationship(String name, String targetTable, String sourceField, String targetField) + { + return new RelationshipDefinition( + name, + targetTable, + RelationshipCardinality.MANY_TO_ONE, + List.of(new JoinKey(sourceField, targetField))); + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotJsonDecoder.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotJsonDecoder.java new file mode 100644 index 000000000000..8b289ea58b2a --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotJsonDecoder.java @@ -0,0 +1,592 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FunctionRewrite; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ModifierBehavior; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyStorage; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationKind; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipCardinality; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotJsonDecoder.DecodeFailure; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotJsonDecoder.Limits; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotLoader.LoadRequest; +import io.trino.hogql.parser.HogQlLanguageVersion; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlSemanticCatalogSnapshotJsonDecoder +{ + private static final HogQlLanguageVersion LANGUAGE_VERSION = HogQlLanguageVersion.valueOf("1.0.0"); + private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier("ducklake", false); + private static final String SECRET = "secret-catalog-payload-value"; + + @Test + public void testDecodesCompleteSnapshot() + { + HogQlSemanticCatalogSnapshot snapshot = new HogQlSemanticCatalogSnapshotJsonDecoder() + .decode(bytes(validSnapshotJson()), LoadRequest.latest(CATALOG, LANGUAGE_VERSION)); + + assertThat(snapshot.schemaVersion()).isEqualTo(2); + assertThat(snapshot.languageVersion()).isEqualTo(LANGUAGE_VERSION); + assertThat(snapshot.catalog()).isEqualTo(CATALOG); + assertThat(snapshot.generation()).isEqualTo(7); + assertThat(snapshot.logicalTables()).singleElement().satisfies(table -> { + assertThat(table.name()).isEqualTo("events"); + assertThat(table.physicalTable().catalog()).isEqualTo(CATALOG); + assertThat(table.physicalTable().schema()).isEqualTo(new PhysicalIdentifier("Analytics", true)); + assertThat(table.fields()).singleElement().satisfies(field -> { + assertThat(field.name()).isEqualTo("properties"); + assertThat(field.trinoTypeSignature()).isEqualTo("json"); + assertThat(field.logicalType()).isEqualTo(LogicalType.JSON); + assertThat(field.nullable()).isTrue(); + assertThat(field.starVisible()).isFalse(); + }); + assertThat(table.properties()).singleElement().satisfies(property -> { + assertThat(property.storage()).isEqualTo(PropertyStorage.JSON_OBJECT); + assertThat(property.logicalType()).isEqualTo(LogicalType.STRING); + }); + assertThat(table.relationships()).singleElement().satisfies(relationship -> { + assertThat(relationship.cardinality()).isEqualTo(RelationshipCardinality.MANY_TO_ONE); + assertThat(relationship.joinKeys()).singleElement().satisfies(joinKey -> { + assertThat(joinKey.sourceField()).isEqualTo("properties"); + assertThat(joinKey.targetField()).isEqualTo("properties"); + }); + }); + }); + assertThat(snapshot.expressionFields()).hasSize(2); + assertThat(snapshot.virtualTables()).singleElement().satisfies(table -> { + assertThat(table.name()).isEqualTo("visible_events"); + assertThat(table.source().kind()).isEqualTo(RelationKind.LOGICAL_TABLE); + assertThat(table.projections()).hasSize(2); + }); + assertThat(snapshot.savedQueries()).singleElement().satisfies(savedQuery -> { + assertThat(savedQuery.queryId()).isEqualTo("query-7"); + assertThat(savedQuery.target().kind()).isEqualTo(RelationKind.VIRTUAL_TABLE); + assertThat(savedQuery.fields()).hasSize(2); + }); + assertThat(snapshot.materializedViews()).singleElement().satisfies(view -> { + assertThat(view.name()).isEqualTo("daily_events"); + assertThat(view.fields()).singleElement().extracting(HogQlSemanticCatalogSnapshot.ReferencedField::name).isEqualTo("day"); + }); + assertThat(snapshot.functions()).singleElement().satisfies(function -> { + assertThat(function.name()).isEqualTo("length"); + assertThat(function.rewrite()).isEmpty(); + assertThat(function.signatures()).singleElement().satisfies(signature -> { + assertThat(signature.argumentTypes()).containsExactly("json"); + assertThat(signature.returnType()).isEqualTo("bigint"); + }); + }); + assertThat(snapshot.modifierDefaults()).singleElement().satisfies(modifier -> { + assertThat(modifier.behavior()).isEqualTo(ModifierBehavior.TRINO_SESSION_PROPERTY); + assertThat(modifier.sessionProperty()).extracting(PhysicalIdentifier::value).containsExactly("hogql", "sampling"); + }); + assertThat(snapshot.logicalTables().getFirst().properties().getFirst()).satisfies(property -> { + assertThat(property.keyTypeSignature()).contains("varchar"); + assertThat(property.valueTypeSignature()).contains("varchar"); + assertThat(property.lookupRecipe()).isPresent(); + }); + assertThat(snapshot.logicalTables().getFirst().relationships().getFirst().joinPredicate()).isPresent(); + assertThat(snapshot.lazyTables()).singleElement().satisfies(table -> { + assertThat(table.relationshipPath()).containsExactly("self"); + assertThat(table.projections()).singleElement().extracting(HogQlSemanticCatalogSnapshot.LazyProjectionDefinition::name).isEqualTo("browser"); + }); + assertThat(snapshot.actions()).singleElement().extracting(HogQlSemanticCatalogSnapshot.ActionReference::actionId).isEqualTo("action-7"); + assertThat(snapshot.cohorts()).singleElement().extracting(HogQlSemanticCatalogSnapshot.CohortReference::cohortId).isEqualTo("cohort-7"); + } + + @Test + public void testDecodesClosedFunctionRewrite() + throws Exception + { + HogQlSemanticCatalogSnapshot snapshot = new HogQlSemanticCatalogSnapshotJsonDecoder() + .decode(bytes(functionPayload("REWRITE", Optional.of("IS_NULL"), true, false)), LoadRequest.latest(CATALOG, LANGUAGE_VERSION)); + + assertThat(snapshot.functions()).singleElement().satisfies(function -> { + assertThat(function.implementation()).isEqualTo(HogQlSemanticCatalogSnapshot.FunctionImplementation.REWRITE); + assertThat(function.rewrite()).contains(FunctionRewrite.IS_NULL); + assertThat(function.trinoName()).isEmpty(); + }); + } + + @Test + public void testRejectsInvalidFunctionRewritePayloads() + throws Exception + { + List payloads = List.of( + functionPayload("REWRITE", Optional.empty(), true, false), + functionPayload("REWRITE", Optional.of("IS_NULL"), false, false), + functionPayload("STOCK", Optional.of("IS_NULL"), false, false), + functionPayload("UDF", Optional.of("IS_NOT_NULL"), false, false), + functionPayload("REWRITE", Optional.of(SECRET), true, false), + functionPayload("REWRITE", Optional.empty(), true, true), + functionPayloadWithBoolean("deterministic", false), + functionPayloadWithBoolean("supportsDistinct", true), + functionPayloadWithBoolean("supportsOrderBy", true), + functionPayloadWithBoolean("supportsFilter", true), + functionPayloadWithBoolean("supportsWindow", true), + functionPayloadWithReturnType("varchar")); + + payloads.forEach(payload -> assertDecodeFailure( + new HogQlSemanticCatalogSnapshotJsonDecoder(), + payload, + LoadRequest.latest(CATALOG, LANGUAGE_VERSION), + DecodeFailure.INVALID_PAYLOAD)); + } + + @Test + public void testDecodesSnapshotFromOlderSchemaVersionTwoPublisher() + throws Exception + { + ObjectNode payload = (ObjectNode) new ObjectMapper().readTree(validSnapshotJson()); + payload.remove(List.of("lazyTables", "actions", "cohorts")); + ObjectNode table = (ObjectNode) ((ArrayNode) payload.get("logicalTables")).get(0); + ObjectNode property = (ObjectNode) ((ArrayNode) table.get("properties")).get(0); + property.remove(List.of("keyTypeSignature", "valueTypeSignature", "lookupRecipe")); + ObjectNode relationship = (ObjectNode) ((ArrayNode) table.get("relationships")).get(0); + relationship.remove("joinPredicate"); + + HogQlSemanticCatalogSnapshot snapshot = new HogQlSemanticCatalogSnapshotJsonDecoder() + .decode(bytes(payload.toString()), LoadRequest.latest(CATALOG, LANGUAGE_VERSION)); + + assertThat(snapshot.lazyTables()).isEmpty(); + assertThat(snapshot.actions()).isEmpty(); + assertThat(snapshot.cohorts()).isEmpty(); + assertThat(snapshot.logicalTables().getFirst().properties().getFirst().lookupRecipe()).isEmpty(); + assertThat(snapshot.logicalTables().getFirst().relationships().getFirst().joinPredicate()).isEmpty(); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("compatibilityFailures") + public void testRejectsIncompatibleSnapshots(String name, String payload, LoadRequest request, DecodeFailure failure) + { + assertDecodeFailure(new HogQlSemanticCatalogSnapshotJsonDecoder(), payload, request, failure); + } + + private static Stream compatibilityFailures() + { + return Stream.of( + Arguments.of( + "protocol", + validSnapshotJson().replace("\"protocolVersion\": 1", "\"protocolVersion\": 2"), + LoadRequest.latest(CATALOG, LANGUAGE_VERSION), + DecodeFailure.UNSUPPORTED_PROTOCOL), + Arguments.of( + "schema", + validSnapshotJson().replace("\"schemaVersion\": 2", "\"schemaVersion\": 1"), + LoadRequest.latest(CATALOG, LANGUAGE_VERSION), + DecodeFailure.UNSUPPORTED_SCHEMA), + Arguments.of( + "language", + validSnapshotJson().replace("\"languageVersion\": \"1.0.0\"", "\"languageVersion\": \"2.0.0\""), + LoadRequest.latest(CATALOG, LANGUAGE_VERSION), + DecodeFailure.LANGUAGE_VERSION_MISMATCH), + Arguments.of( + "catalog", + validSnapshotJson().replace("\"value\": \"ducklake\"", "\"value\": \"other\""), + LoadRequest.latest(CATALOG, LANGUAGE_VERSION), + DecodeFailure.CATALOG_MISMATCH), + Arguments.of( + "pinned generation", + validSnapshotJson(), + LoadRequest.pinned(CATALOG, LANGUAGE_VERSION, 8), + DecodeFailure.GENERATION_MISMATCH), + Arguments.of( + "nonpositive generation", + validSnapshotJson().replace("\"generation\": 7", "\"generation\": 0"), + LoadRequest.latest(CATALOG, LANGUAGE_VERSION), + DecodeFailure.GENERATION_MISMATCH)); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("strictFailures") + public void testRejectsMalformedOrExtendedPayloads(String name, String payload) + { + assertDecodeFailure( + new HogQlSemanticCatalogSnapshotJsonDecoder(), + payload, + LoadRequest.latest(CATALOG, LANGUAGE_VERSION), + DecodeFailure.INVALID_PAYLOAD); + } + + private static Stream strictFailures() + { + return Stream.of( + Arguments.of("unknown root field", validSnapshotJson().replace("\"logicalTables\":", "\"unknown\": true, \"logicalTables\":")), + Arguments.of("unknown nested field", validSnapshotJson().replace("\"starVisible\": false", "\"starVisible\": false, \"unknown\": true")), + Arguments.of("unknown recipe field", validSnapshotJson().replace("\"fieldReference\": {", "\"unknown\": true, \"fieldReference\": {")), + Arguments.of("unknown recipe kind", validSnapshotJson().replace("\"kind\": \"FUNCTION_CALL\"", "\"kind\": \"" + SECRET + "\"")), + Arguments.of("mismatched recipe payload", validSnapshotJson().replace("\"functionCall\": {", "\"literal\": {")), + Arguments.of("partial property lookup metadata", validSnapshotJson().replace("\"valueTypeSignature\": \"varchar\",\n", "")), + Arguments.of("mismatched entity representation", validSnapshotJson().replace("\"kind\": \"PREDICATE\"", "\"kind\": \"RELATION\"")), + Arguments.of( + "unknown function reference", + validSnapshotJson().replaceFirst("\"name\": \"length\",", "\"name\": \"missing\",")), + Arguments.of("unknown virtual source", validSnapshotJson().replace("\"kind\": \"LOGICAL_TABLE\", \"name\": \"events\"", "\"kind\": \"LOGICAL_TABLE\", \"name\": \"missing\"")), + Arguments.of("saved query target missing field", validSnapshotJson().replace("\"name\": \"browser_length\", \"trinoTypeSignature\": \"bigint\"", "\"name\": \"missing\", \"trinoTypeSignature\": \"bigint\"")), + Arguments.of("invalid literal encoding", validSnapshotJson().replace("\"encoding\": \"INTEGER\", \"value\": \"1\"", "\"encoding\": \"INTEGER\", \"value\": \"" + SECRET + "\"")), + Arguments.of( + "invalid modifier session property", + validSnapshotJson().replace( + "\"sessionProperty\": [", + "\"unknown\": true,\n \"sessionProperty\": [")), + Arguments.of( + "overqualified modifier session property", + validSnapshotJson().replace( + "{\"value\": \"hogql\", \"delimited\": false},", + "{\"value\": \"catalog\", \"delimited\": false},\n {\"value\": \"schema\", \"delimited\": false},")), + Arguments.of("duplicate field", validSnapshotJson().replace("\"protocolVersion\": 1", "\"protocolVersion\": 1, \"protocolVersion\": 1")), + Arguments.of("missing field", validSnapshotJson().replace("\"generation\": 7,", "")), + Arguments.of("wrong field type", validSnapshotJson().replace("\"nullable\": true", "\"nullable\": \"true\"")), + Arguments.of("invalid enum", validSnapshotJson().replace("\"logicalType\": \"JSON\"", "\"logicalType\": \"" + SECRET + "\"")), + Arguments.of("trailing content", validSnapshotJson() + " true"), + Arguments.of("malformed JSON", "{\"protocolVersion\":" + SECRET)); + } + + @Test + public void testEnforcesPayloadLimit() + { + HogQlSemanticCatalogSnapshotJsonDecoder decoder = new HogQlSemanticCatalogSnapshotJsonDecoder( + new Limits(bytes(validSnapshotJson()).length - 1, 64, 100)); + + assertDecodeFailure(decoder, validSnapshotJson(), LoadRequest.latest(CATALOG, LANGUAGE_VERSION), DecodeFailure.LIMIT_EXCEEDED); + } + + @Test + public void testEnforcesDepthLimit() + { + String payload = validSnapshotJson().replace( + "\"logicalTables\":", + "\"unknown\": [[[[[true]]]]], \"logicalTables\":"); + HogQlSemanticCatalogSnapshotJsonDecoder decoder = new HogQlSemanticCatalogSnapshotJsonDecoder(new Limits(1_000_000, 4, 100)); + + assertDecodeFailure(decoder, payload, LoadRequest.latest(CATALOG, LANGUAGE_VERSION), DecodeFailure.LIMIT_EXCEEDED); + } + + @Test + public void testEnforcesCumulativeCollectionLimit() + { + HogQlSemanticCatalogSnapshotJsonDecoder decoder = new HogQlSemanticCatalogSnapshotJsonDecoder(new Limits(1_000_000, 64, 1)); + + assertDecodeFailure(decoder, validSnapshotJson(), LoadRequest.latest(CATALOG, LANGUAGE_VERSION), DecodeFailure.LIMIT_EXCEEDED); + } + + private static void assertDecodeFailure( + HogQlSemanticCatalogSnapshotJsonDecoder decoder, + String payload, + LoadRequest request, + DecodeFailure expectedFailure) + { + assertThatThrownBy(() -> decoder.decode(bytes(payload), request)) + .isInstanceOfSatisfying(HogQlSemanticCatalogSnapshotJsonDecoder.DecodeException.class, exception -> { + assertThat(exception.failure()).isEqualTo(expectedFailure); + assertThat(exception).hasMessageNotContaining(SECRET); + assertThat(exception.getMessage()).hasSizeLessThan(128); + assertThat(exception.getCause()).isNull(); + }); + } + + private static byte[] bytes(String value) + { + return value.getBytes(StandardCharsets.UTF_8); + } + + private static String functionPayload(String implementation, Optional rewrite, boolean emptyTrinoName, boolean nullRewrite) + throws Exception + { + ObjectNode payload = (ObjectNode) new ObjectMapper().readTree(validSnapshotJson()); + ObjectNode function = (ObjectNode) ((ArrayNode) payload.get("functions")).get(0); + function.put("implementation", implementation); + rewrite.ifPresentOrElse(value -> function.put("rewrite", value), () -> { + if (nullRewrite) { + function.putNull("rewrite"); + } + else { + function.remove("rewrite"); + } + }); + if (emptyTrinoName) { + ((ArrayNode) function.get("trinoName")).removeAll(); + } + if (implementation.equals("REWRITE")) { + ObjectNode signature = (ObjectNode) ((ArrayNode) function.get("signatures")).get(0); + signature.put("returnType", "boolean"); + } + return payload.toString(); + } + + private static String functionPayloadWithBoolean(String field, boolean value) + throws Exception + { + ObjectNode payload = (ObjectNode) new ObjectMapper().readTree(functionPayload("REWRITE", Optional.of("IS_NULL"), true, false)); + ObjectNode function = (ObjectNode) ((ArrayNode) payload.get("functions")).get(0); + function.put(field, value); + return payload.toString(); + } + + private static String functionPayloadWithReturnType(String returnType) + throws Exception + { + ObjectNode payload = (ObjectNode) new ObjectMapper().readTree(functionPayload("REWRITE", Optional.of("IS_NULL"), true, false)); + ObjectNode function = (ObjectNode) ((ArrayNode) payload.get("functions")).get(0); + ObjectNode signature = (ObjectNode) ((ArrayNode) function.get("signatures")).get(0); + signature.put("returnType", returnType); + return payload.toString(); + } + + private static String validSnapshotJson() + { + return """ + { + "protocolVersion": 1, + "schemaVersion": 2, + "languageVersion": "1.0.0", + "catalog": {"value": "ducklake", "delimited": false}, + "generation": 7, + "logicalTables": [ + { + "name": "events", + "physicalTable": { + "catalog": {"value": "ducklake", "delimited": false}, + "schema": {"value": "Analytics", "delimited": true}, + "table": {"value": "events", "delimited": false} + }, + "fields": [ + { + "name": "properties", + "physicalColumn": {"value": "properties", "delimited": false}, + "trinoTypeSignature": "json", + "logicalType": "JSON", + "nullable": true, + "starVisible": false + } + ], + "properties": [ + { + "name": "properties", + "sourceField": "properties", + "storage": "JSON_OBJECT", + "logicalType": "STRING", + "nullable": true, + "keyTypeSignature": "varchar", + "valueTypeSignature": "varchar", + "lookupRecipe": { + "kind": "OPERATOR", + "operator": { + "operator": "SUBSCRIPT", + "arguments": [ + {"kind": "ARGUMENT_REFERENCE", "argumentReference": {"argument": "PROPERTY_SOURCE"}}, + {"kind": "ARGUMENT_REFERENCE", "argumentReference": {"argument": "PROPERTY_KEY"}} + ] + } + } + } + ], + "relationships": [ + { + "name": "self", + "targetTable": "events", + "cardinality": "MANY_TO_ONE", + "joinKeys": [{"sourceField": "properties", "targetField": "properties"}], + "joinPredicate": { + "kind": "OPERATOR", + "operator": { + "operator": "EQUAL", + "arguments": [ + {"kind": "SCOPED_FIELD_REFERENCE", "scopedFieldReference": {"side": "SOURCE", "field": "properties"}}, + {"kind": "SCOPED_FIELD_REFERENCE", "scopedFieldReference": {"side": "TARGET", "field": "properties"}} + ] + } + } + } + ] + } + ], + "expressionFields": [ + { + "table": "events", + "name": "browser_length", + "trinoTypeSignature": "bigint", + "logicalType": "INTEGER", + "nullable": true, + "starVisible": false, + "recipe": { + "kind": "FUNCTION_CALL", + "functionCall": { + "name": "length", + "arguments": [ + {"kind": "FIELD_REFERENCE", "fieldReference": {"table": "events", "field": "properties"}} + ] + } + } + }, + { + "table": "events", + "name": "adjusted_count", + "trinoTypeSignature": "bigint", + "logicalType": "INTEGER", + "nullable": false, + "starVisible": false, + "recipe": { + "kind": "CAST", + "cast": { + "targetTypeSignature": "bigint", + "expression": { + "kind": "OPERATOR", + "operator": { + "operator": "ADD", + "arguments": [ + {"kind": "LITERAL", "literal": {"typeSignature": "bigint", "encoding": "INTEGER", "value": "1"}}, + {"kind": "LITERAL", "literal": {"typeSignature": "bigint", "encoding": "INTEGER", "value": "2"}} + ] + } + } + } + } + } + ], + "virtualTables": [ + { + "name": "visible_events", + "source": {"kind": "LOGICAL_TABLE", "name": "events"}, + "projections": [ + {"name": "properties", "sourceField": "properties", "starVisible": true}, + {"name": "browser_length", "sourceField": "browser_length", "starVisible": false} + ] + } + ], + "savedQueries": [ + { + "name": "saved_events", + "queryId": "query-7", + "target": {"kind": "VIRTUAL_TABLE", "name": "visible_events"}, + "fields": [ + {"name": "properties", "trinoTypeSignature": "json", "logicalType": "JSON", "nullable": true, "starVisible": true}, + {"name": "browser_length", "trinoTypeSignature": "bigint", "logicalType": "INTEGER", "nullable": true, "starVisible": false} + ] + } + ], + "materializedViews": [ + { + "name": "daily_events", + "physicalView": { + "catalog": {"value": "ducklake", "delimited": false}, + "schema": {"value": "Analytics", "delimited": true}, + "table": {"value": "daily_events", "delimited": false} + }, + "fields": [ + {"name": "day", "trinoTypeSignature": "date", "logicalType": "DATE", "nullable": false, "starVisible": true} + ] + } + ], + "functions": [ + { + "name": "length", + "kind": "SCALAR", + "implementation": "STOCK", + "trinoName": [{"value": "length", "delimited": false}], + "signatures": [{"argumentTypes": ["json"], "returnType": "bigint", "variadic": false}], + "deterministic": true, + "supportsDistinct": false, + "supportsOrderBy": false, + "supportsFilter": false, + "supportsWindow": false + } + ], + "modifierDefaults": [ + { + "name": "sampling", + "behavior": "TRINO_SESSION_PROPERTY", + "defaultValue": {"typeSignature": "bigint", "encoding": "INTEGER", "value": "1"}, + "sessionProperty": [ + {"value": "hogql", "delimited": false}, + {"value": "sampling", "delimited": false} + ] + } + ], + "lazyTables": [ + { + "table": "events", + "name": "profile", + "relationshipPath": ["self"], + "projections": [ + { + "name": "browser", + "trinoTypeSignature": "varchar", + "logicalType": "STRING", + "nullable": true, + "starVisible": true, + "recipe": { + "kind": "PROPERTY_LOOKUP", + "propertyLookup": { + "table": "events", + "property": "properties", + "key": {"kind": "LITERAL", "literal": {"typeSignature": "varchar", "encoding": "STRING", "value": "browser"}} + } + } + } + ] + } + ], + "actions": [ + { + "name": "paid_event", + "actionId": "action-7", + "table": "events", + "representation": { + "kind": "PREDICATE", + "predicate": { + "kind": "PROPERTY_LOOKUP", + "propertyLookup": { + "table": "events", + "property": "properties", + "key": {"kind": "LITERAL", "literal": {"typeSignature": "varchar", "encoding": "STRING", "value": "browser"}} + } + } + } + } + ], + "cohorts": [ + { + "name": "active_people", + "cohortId": "cohort-7", + "table": "events", + "representation": { + "kind": "RELATION", + "relation": { + "relation": {"kind": "MATERIALIZED_VIEW", "name": "daily_events"}, + "sourceField": "properties", + "targetField": "day" + } + } + } + ] + } + """; + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotLoader.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotLoader.java new file mode 100644 index 000000000000..04d2df65a820 --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotLoader.java @@ -0,0 +1,84 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotLoader.JsonTransport; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotLoader.LoadRequest; +import io.trino.hogql.parser.HogQlLanguageVersion; +import org.junit.jupiter.api.Test; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlSemanticCatalogSnapshotLoader +{ + private static final HogQlLanguageVersion LANGUAGE_VERSION = HogQlLanguageVersion.valueOf("1.0.0"); + private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier("ducklake", false); + + @Test + public void testPreservesLatestAndPinnedReadSemanticsThroughJsonTransport() + { + List requests = new ArrayList<>(); + JsonTransport transport = request -> { + requests.add(request); + return CompletableFuture.completedFuture(snapshotJson(request.expectedGeneration().orElse(9)).getBytes(StandardCharsets.UTF_8)); + }; + HogQlSemanticCatalogSnapshotLoader loader = HogQlSemanticCatalogSnapshotLoader.fromJsonTransport( + transport, + new HogQlSemanticCatalogSnapshotJsonDecoder()); + + HogQlSemanticCatalogSnapshot latest = loader.load(LoadRequest.latest(CATALOG, LANGUAGE_VERSION)).toCompletableFuture().join(); + HogQlSemanticCatalogSnapshot pinned = loader.load(LoadRequest.pinned(CATALOG, LANGUAGE_VERSION, 7)).toCompletableFuture().join(); + + assertThat(latest.generation()).isEqualTo(9); + assertThat(pinned.generation()).isEqualTo(7); + assertThat(requests).containsExactly( + LoadRequest.latest(CATALOG, LANGUAGE_VERSION), + LoadRequest.pinned(CATALOG, LANGUAGE_VERSION, 7)); + } + + @Test + public void testRejectsInvalidPinnedGenerationBeforeTransport() + { + assertThatThrownBy(() -> LoadRequest.pinned(CATALOG, LANGUAGE_VERSION, 0)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageNotContaining("ducklake"); + } + + private static String snapshotJson(long generation) + { + return """ + { + "protocolVersion": 1, + "schemaVersion": 2, + "languageVersion": "1.0.0", + "catalog": {"value": "ducklake", "delimited": false}, + "generation": %s, + "logicalTables": [], + "expressionFields": [], + "virtualTables": [], + "savedQueries": [], + "materializedViews": [], + "functions": [], + "modifierDefaults": [] + } + """.formatted(generation); + } +} diff --git a/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotProvider.java b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotProvider.java new file mode 100644 index 000000000000..2769cc7cc03a --- /dev/null +++ b/core/trino-hogql-compiler/src/test/java/io/trino/hogql/compiler/catalog/TestHogQlSemanticCatalogSnapshotProvider.java @@ -0,0 +1,153 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.compiler.catalog; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogException.Failure; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinRequest; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; +import io.trino.hogql.parser.HogQlLanguageVersion; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.List; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Stream; + +import static io.trino.spi.ErrorType.EXTERNAL; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlSemanticCatalogSnapshotProvider +{ + private static final HogQlLanguageVersion LANGUAGE_VERSION = HogQlLanguageVersion.valueOf("1.0.0"); + private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier("ducklake", false); + + @Test + public void testPinsOneSnapshotWithoutCrossGenerationReads() + { + AtomicInteger reads = new AtomicInteger(); + HogQlSemanticCatalogSnapshotCache cache = _ -> Optional.of(snapshot(reads.incrementAndGet())); + HogQlSemanticCatalogSnapshotProvider provider = HogQlSemanticCatalogSnapshotProvider.fromCache(cache); + + PinnedSnapshot pinned = provider.pin(new PinRequest(CATALOG, LANGUAGE_VERSION, OptionalLong.empty())); + + assertThat(reads).hasValue(1); + assertThat(pinned.generation()).isEqualTo(1); + assertThat(pinned.snapshot().generation()).isEqualTo(1); + assertThat(pinned.logicalTable("events")).isPresent(); + assertThat(reads).hasValue(1); + } + + @Test + public void testExpectedGenerationIsPassedToExactCacheLookup() + { + AtomicReference requestedGeneration = new AtomicReference<>(); + HogQlSemanticCatalogSnapshotCache cache = new HogQlSemanticCatalogSnapshotCache() + { + @Override + public Optional currentSnapshot(PhysicalIdentifier catalog) + { + throw new AssertionError("expected exact cache lookup"); + } + + @Override + public Optional currentSnapshot(PhysicalIdentifier catalog, OptionalLong expectedGeneration) + { + requestedGeneration.set(expectedGeneration); + return Optional.of(snapshot(7)); + } + }; + HogQlSemanticCatalogSnapshotProvider provider = HogQlSemanticCatalogSnapshotProvider.fromCache(cache); + + assertThat(provider.pin(new PinRequest(CATALOG, LANGUAGE_VERSION, OptionalLong.of(7))).generation()).isEqualTo(7); + assertThat(requestedGeneration).hasValue(OptionalLong.of(7)); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("failClosedCases") + public void testFailsClosed(String name, HogQlSemanticCatalogSnapshotCache cache, PinRequest request, Failure failure) + { + HogQlSemanticCatalogSnapshotProvider provider = HogQlSemanticCatalogSnapshotProvider.fromCache(cache); + + assertThatThrownBy(() -> provider.pin(request)) + .isInstanceOfSatisfying(HogQlSemanticCatalogException.class, exception -> { + assertThat(exception.failure()).isEqualTo(failure); + assertThat(exception.getErrorCode().getName()).isEqualTo( + failure == Failure.UNAVAILABLE ? "HOGQL_CATALOG_NOT_READY" : "HOGQL_CATALOG_GENERATION_MISMATCH"); + assertThat(exception.getErrorCode().getType()).isEqualTo(EXTERNAL); + }); + } + + private static Stream failClosedCases() + { + PhysicalIdentifier otherCatalog = new PhysicalIdentifier("other", false); + return Stream.of( + Arguments.of( + "unavailable", + (HogQlSemanticCatalogSnapshotCache) _ -> Optional.empty(), + new PinRequest(CATALOG, LANGUAGE_VERSION, OptionalLong.empty()), + Failure.UNAVAILABLE), + Arguments.of( + "cache returned another catalog", + (HogQlSemanticCatalogSnapshotCache) _ -> Optional.of(snapshot(otherCatalog, LANGUAGE_VERSION, 1)), + new PinRequest(CATALOG, LANGUAGE_VERSION, OptionalLong.empty()), + Failure.CATALOG_MISMATCH), + Arguments.of( + "language mismatch", + (HogQlSemanticCatalogSnapshotCache) _ -> Optional.of(snapshot(CATALOG, HogQlLanguageVersion.valueOf("2.0.0"), 1)), + new PinRequest(CATALOG, LANGUAGE_VERSION, OptionalLong.empty()), + Failure.LANGUAGE_VERSION_MISMATCH), + Arguments.of( + "generation mismatch", + (HogQlSemanticCatalogSnapshotCache) _ -> Optional.of(snapshot(2)), + new PinRequest(CATALOG, LANGUAGE_VERSION, OptionalLong.of(1)), + Failure.GENERATION_MISMATCH)); + } + + private static HogQlSemanticCatalogSnapshot snapshot(long generation) + { + return snapshot(CATALOG, LANGUAGE_VERSION, generation); + } + + private static HogQlSemanticCatalogSnapshot snapshot(PhysicalIdentifier catalog, HogQlLanguageVersion languageVersion, long generation) + { + return new HogQlSemanticCatalogSnapshot( + 2, + languageVersion, + catalog, + generation, + List.of(new LogicalTableDefinition( + "events", + new PhysicalQualifiedName(catalog, new PhysicalIdentifier("default", false), new PhysicalIdentifier("events", false)), + List.of(new LogicalFieldDefinition( + "id", + new PhysicalIdentifier("id", false), + "varchar", + LogicalType.STRING, + false, + true)), + List.of(), + List.of()))); + } +} diff --git a/core/trino-hogql-parser/pom.xml b/core/trino-hogql-parser/pom.xml new file mode 100644 index 000000000000..496956ef0ec8 --- /dev/null +++ b/core/trino-hogql-parser/pom.xml @@ -0,0 +1,71 @@ + + + 4.0.0 + + + io.trino + trino-root + 484-SNAPSHOT + ../../pom.xml + + + trino-hogql-parser + ${project.artifactId} + Trino - HogQL parser + + + + com.fasterxml.jackson.core + jackson-annotations + + + + io.airlift + json + + + + org.antlr + antlr4-runtime + + + + org.antlr + antlr4 + test + + + + org.assertj + assertj-core + test + + + + org.junit.jupiter + junit-jupiter-api + test + + + + org.junit.jupiter + junit-jupiter-engine + test + + + + org.junit.jupiter + junit-jupiter-params + test + + + + + + + org.antlr + antlr4-maven-plugin + + + + diff --git a/core/trino-hogql-parser/src/main/antlr4/io/trino/hogql/parser/antlr/HogQlBase.g4 b/core/trino-hogql-parser/src/main/antlr4/io/trino/hogql/parser/antlr/HogQlBase.g4 new file mode 100644 index 000000000000..43f3658f43c7 --- /dev/null +++ b/core/trino-hogql-parser/src/main/antlr4/io/trino/hogql/parser/antlr/HogQlBase.g4 @@ -0,0 +1,101 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +grammar HogQlBase; + +options { caseInsensitive = true; } + +singleStatement + : query SEMICOLON? EOF + ; + +query + : SELECT projection (COMMA projection)* (FROM qualifiedName)? + ; + +projection + : ASTERISK + | expression + ; + +expression + : literal + | qualifiedName + ; + +literal + : INTEGER_VALUE + | STRING + | TRUE + | FALSE + | NULL + ; + +qualifiedName + : identifier (DOT identifier)* + ; + +identifier + : IDENTIFIER + | QUOTED_IDENTIFIER + | BACKQUOTED_IDENTIFIER + ; + +SELECT: 'SELECT'; +FROM: 'FROM'; +TRUE: 'TRUE'; +FALSE: 'FALSE'; +NULL: 'NULL'; + +ASTERISK: '*'; +COMMA: ','; +DOT: '.'; +SEMICOLON: ';'; + +STRING + : '\'' (~['\\] | '\'\'')* '\'' + ; + +INTEGER_VALUE + : '0' + | [1-9] DIGIT* + ; + +IDENTIFIER + : (LETTER | '_') (LETTER | DIGIT | '_' | '$')* + ; + +QUOTED_IDENTIFIER + : '"' (~'"' | '""')+ '"' + ; + +BACKQUOTED_IDENTIFIER + : '`' (~'`' | '``')+ '`' + ; + +fragment DIGIT + : [0-9] + ; + +fragment LETTER + : [A-Z] + ; + +WHITESPACE + : [ \r\n\t]+ -> channel(HIDDEN) + ; + +UNRECOGNIZED + : . + ; diff --git a/core/trino-hogql-parser/src/main/antlr4/io/trino/hogql/parser/canonical/HogQLLexer.g4 b/core/trino-hogql-parser/src/main/antlr4/io/trino/hogql/parser/canonical/HogQLLexer.g4 new file mode 100644 index 000000000000..1dd9302c0341 --- /dev/null +++ b/core/trino-hogql-parser/src/main/antlr4/io/trino/hogql/parser/canonical/HogQLLexer.g4 @@ -0,0 +1,450 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +lexer grammar HogQLLexer; + +@members { + +private static boolean isAsciiAlpha(int character) { + return character >= 0 && character < 128 && + ((character >= 'a' && character <= 'z') || (character >= 'A' && character <= 'Z')); +} + +private static boolean isAsciiAlphanumeric(int character) { + return isAsciiAlpha(character) || (character >= '0' && character <= '9'); +} + +private static boolean isAsciiWhitespace(int character) { + return character == ' ' || character == '\t' || character == '\n' || + character == '\u000B' || character == '\f' || character == '\r'; +} + +private int skipWhitespaceAndComments(int index) { + while (true) { + int character = _input.LA(index); + if (isAsciiWhitespace(character)) { + index++; + continue; + } + + if (character == '/' && _input.LA(index + 1) == '/') { + index += 2; + } + else if (character == '-' && _input.LA(index + 1) == '-') { + index += 2; + } + else if (character == '#') { + index++; + } + else { + return index; + } + + while (true) { + character = _input.LA(index); + if (character <= 0 || character == '\n' || character == '\r') { + break; + } + index++; + } + } +} + +private boolean isOpeningTag() { + int firstCharacter = _input.LA(1); + if (!isAsciiAlpha(firstCharacter) && firstCharacter != '_') { + return false; + } + + int index = 2; + while (true) { + int character = _input.LA(index); + if (isAsciiAlphanumeric(character) || character == '_' || character == '-') { + index++; + } + else { + break; + } + } + + int character = _input.LA(index); + if (character == '>' || character == '/') { + return true; + } + + if (isAsciiWhitespace(character)) { + index = skipWhitespaceAndComments(index + 1); + character = _input.LA(index); + return isAsciiAlphanumeric(character) || character == '_' || character == '>' || character == '/'; + } + + return false; +} + +} + +// NB! We cat either HogQLLexter.cpp.g4 or HogQLLexter.python.g4 when generating the grammar. + +// NOTE: don't forget to add new keywords to the parser rule "keyword"! + +// Keywords + +ALL: A L L; +AND: A N D; +ANTI: A N T I; +ANY: A N Y; +ARRAY: A R R A Y; +AS: A S; +ASCENDING: A S C | A S C E N D I N G; +ASOF: A S O F; +BETWEEN: B E T W E E N; +BOTH: B O T H; +BY: B Y; +CASE: C A S E; +CAST: C A S T; +CATCH: C A T C H; +COHORT: C O H O R T; +COLLATE: C O L L A T E; +COLUMNS: C O L U M N S; +CROSS: C R O S S; +CUBE: C U B E; +CURRENT: C U R R E N T; +DATE: D A T E; +DAY: D A Y; +DESC: D E S C; +DESCENDING: D E S C E N D I N G; +DISTINCT: D I S T I N C T; +ELSE: E L S E; +END: E N D; +EXCEPT: E X C E P T; +EXCLUDE: E X C L U D E; +EXTRACT: E X T R A C T; +FINAL: F I N A L; +FILL: F I L L; +FILTER: F I L T E R; +FINALLY: F I N A L L Y; +FIRST: F I R S T; +FN: F N; +FOLLOWING: F O L L O W I N G; +FOR: F O R; +FROM: F R O M; +FULL: F U L L; +FUN: F U N; +GROUP: G R O U P; +GROUPING: G R O U P I N G; +HAVING: H A V I N G; +HOUR: H O U R; +ID: I D; +IF: I F; +ILIKE: I L I K E; +IGNORE: I G N O R E; +INCLUDE: I N C L U D E; +IN: I N; +INF: I N F | I N F I N I T Y; +INNER: I N N E R; +INTERSECT: I N T E R S E C T; +INTERPOLATE: I N T E R P O L A T E; +INTERVAL: I N T E R V A L; +IS: I S; +JOIN: J O I N; +KEY: K E Y; +LAMBDA: L A M B D A; +LAST: L A S T; +LEADING: L E A D I N G; +LEFT: L E F T; +LET: L E T; +LIKE: L I K E; +LIMIT: L I M I T; +MATERIALIZED: M A T E R I A L I Z E D; +MINUTE: M I N U T E; +MONTH: M O N T H; +NAME: N A M E; +NATURAL: N A T U R A L; +NAN_SQL: N A N; // conflicts with macro NAN +NOT: N O T; +NULL_SQL: N U L L; // conflicts with macro NULL +NULLS: N U L L S; +OFFSET: O F F S E T; +ON: O N; +OR: O R; +ORDER: O R D E R; +OUTER: O U T E R; +OVER: O V E R; +PARTITION: P A R T I T I O N; +PIVOT: P I V O T; +POSITIONAL: P O S I T I O N A L; +PRECEDING: P R E C E D I N G; +PREWHERE: P R E W H E R E; +QUALIFY: Q U A L I F Y; +QUARTER: Q U A R T E R; +RANGE: R A N G E; +RECURSIVE: R E C U R S I V E; +REPLACE: R E P L A C E; +RETURN: R E T U R N; +RIGHT: R I G H T; +ROLLUP: R O L L U P; +ROW: R O W; +ROWS: R O W S; +SAMPLE: S A M P L E; +SECOND: S E C O N D; +SELECT: S E L E C T; +SEMI: S E M I; +SETS: S E T S; +SETTINGS: S E T T I N G S; +STEP: S T E P; +SUBSTRING: S U B S T R I N G; +THEN: T H E N; +THROW: T H R O W; +TIES: T I E S; +TIMESTAMP: T I M E S T A M P; +TIME: T I M E; +LOCAL: L O C A L; +ZONE: Z O N E; +TO: T O; +TOP: T O P; +TOTALS: T O T A L S; +TRAILING: T R A I L I N G; +TRIM: T R I M; +TRUNCATE: T R U N C A T E; +TRY: T R Y; +TRY_CAST: T R Y '_' C A S T; +UNBOUNDED: U N B O U N D E D; +UNION: U N I O N; +UNPIVOT: U N P I V O T; +USING: U S I N G; +VALUES: V A L U E S; +WEEK: W E E K; +WHEN: W H E N; +WHERE: W H E R E; +WHILE: W H I L E; +WINDOW: W I N D O W; +WITH: W I T H; +WITHIN: W I T H I N; +YEAR: Y E A R | Y Y Y Y; + +// Tokens + +// copied from clickhouse_driver/util/escape.py +ESCAPE_CHAR_COMMON + : BACKSLASH B + | BACKSLASH F + | BACKSLASH R + | BACKSLASH N + | BACKSLASH T + | BACKSLASH '0' + | BACKSLASH A + | BACKSLASH V + | BACKSLASH BACKSLASH + | BACKSLASH X HEX_DIGIT HEX_DIGIT; + +IDENTIFIER + : (LETTER | UNDERSCORE | DOLLAR) (LETTER | UNDERSCORE | DEC_DIGIT | DOLLAR)* + ; +QUOTED_IDENTIFIER + : BACKQUOTE ( ~([\\`]) | ESCAPE_CHAR_COMMON | BACKSLASH QUOTE_SINGLE | (BACKQUOTE BACKQUOTE) )* BACKQUOTE + | QUOTE_DOUBLE ( ~([\\"]) | ESCAPE_CHAR_COMMON | BACKSLASH QUOTE_DOUBLE | (QUOTE_DOUBLE QUOTE_DOUBLE) )* QUOTE_DOUBLE + ; +FLOATING_LITERAL + // Hex-float exponent: strict C99 `p`/`P` only — `e`/`E` stays a hex digit, so `0x1e5` is 485, not a float. + : HEXADECIMAL_LITERAL DOT HEX_DIGIT* P (PLUS | DASH)? DEC_DIGIT+ + | HEXADECIMAL_LITERAL P (PLUS | DASH)? DEC_DIGIT+ + | DECIMAL_LITERAL DOT DEC_DIGIT* E (PLUS | DASH)? DEC_DIGIT+ + | DOT DECIMAL_LITERAL E (PLUS | DASH)? DEC_DIGIT+ + | DECIMAL_LITERAL E (PLUS | DASH)? DEC_DIGIT+ + ; +// Binary literals (`0b1010`). Declared first so it wins the length-tie against MALFORMED_BINARY_LITERAL. +BINARY_LITERAL: '0' B BIN_DIGIT+; +OCTAL_LITERAL: '0' OCT_DIGIT+; +DECIMAL_LITERAL: DEC_DIGIT+; +HEXADECIMAL_LITERAL: '0' X HEX_DIGIT+; +// Postgres-16 `0o` octal — unsupported; lexed as a real token so the visitor can reject it clearly. +OCTAL_PREFIX_LITERAL: '0' [oO] DEC_DIGIT+; +// Malformed binary (`0b22`) BINARY_LITERAL didn't consume — caught so it can't re-tokenise as `0` + IDENTIFIER. +MALFORMED_BINARY_LITERAL: '0' [bB] DEC_DIGIT+; + +// It's important that quote-symbol is a single character. +STRING_LITERAL: QUOTE_SINGLE ( ~([\\']) | ESCAPE_CHAR_COMMON | BACKSLASH QUOTE_SINGLE | (QUOTE_SINGLE QUOTE_SINGLE) )* QUOTE_SINGLE; + + +// Alphabet and allowed symbols + +fragment A: [aA]; +fragment B: [bB]; +fragment C: [cC]; +fragment D: [dD]; +fragment E: [eE]; +fragment F: [fF]; +fragment G: [gG]; +fragment H: [hH]; +fragment I: [iI]; +fragment J: [jJ]; +fragment K: [kK]; +fragment L: [lL]; +fragment M: [mM]; +fragment N: [nN]; +fragment O: [oO]; +fragment P: [pP]; +fragment Q: [qQ]; +fragment R: [rR]; +fragment S: [sS]; +fragment T: [tT]; +fragment U: [uU]; +fragment V: [vV]; +fragment W: [wW]; +fragment X: [xX]; +fragment Y: [yY]; +fragment Z: [zZ]; + +fragment LETTER: [a-zA-Z]; +fragment BIN_DIGIT: [01]; +fragment OCT_DIGIT: [0-7]; +fragment DEC_DIGIT: [0-9]; +fragment HEX_DIGIT: [0-9a-fA-F]; + +ARROW: '->'; +ASTERISK: '*'; +BACKQUOTE: '`'; +BACKSLASH: '\\'; +DOUBLECOLON: '::'; +COLONEQUALS: ':='; +COLON: ':'; +COMMA: ','; +CONCAT: '||'; +DASH: '-'; +DOLLAR: '$'; +DOT: '.'; +EQ_DOUBLE: '=='; +EQ_SINGLE: '='; +GT_EQ: '>='; +GT: '>'; +HASH: '#'; +IREGEX_SINGLE: '~*'; +IREGEX_DOUBLE: '=~*'; +LBRACE: '{' -> pushMode(DEFAULT_MODE); +LBRACKET: '['; +LPAREN: '('; +NULL_SAFE_EQ: '<=>'; +LT_EQ: '<='; +TAG_LT_SLASH: ' type(LT_SLASH), pushMode(HOGQLX_TAG_CLOSE); +TAG_LT_OPEN: '<' {isOpeningTag()}? -> type(LT), pushMode(HOGQLX_TAG_OPEN); +LT: '<'; +LT_SLASH: ''; +NOT_IREGEX: '!~*'; +NOT_REGEX: '!~'; +NULL_PROPERTY: '?.'; +NULLISH: '??'; +PERCENT: '%'; +PLUS: '+'; +QUERY: '?'; +QUOTE_DOUBLE: '"'; +QUOTE_SINGLE_TEMPLATE: 'f\'' -> pushMode(IN_TEMPLATE_STRING); // start of regular f'' template strings +QUOTE_SINGLE_TEMPLATE_FULL: 'F\'' -> pushMode(IN_FULL_TEMPLATE_STRING); // magic F' symbol used to parse "full text" templates +QUOTE_SINGLE: '\''; +REGEX_SINGLE: '~'; +REGEX_DOUBLE: '=~'; +RBRACE: '}' -> popMode; +RBRACKET: ']'; +RPAREN: ')'; +SEMICOLON: ';'; +SLASH: '/'; +SLASH_GT: '/>'; +UNDERSCORE: '_'; + +// Comments and whitespace +MULTI_LINE_COMMENT: '/*' .*? '*/' -> skip; +SINGLE_LINE_COMMENT: ('--' | '//') ~('\n'|'\r')* ('\n' | '\r' | EOF) -> skip; +// MySQL-style `#` comments. `#` is excluded so positional references (`#1`) keep +// working — a `#` comment whose text starts with a digit is the one MySQL-ism this rejects. +HASH_COMMENT: '#' (~[0-9\n\r] ~[\n\r]*)? ('\n' | '\r' | EOF) -> skip; +// whitespace is hidden and not skipped so that it's preserved in ANTLR errors like "no viable alternative" +// The class is the full Unicode `White_Space` set, not just ASCII: a +// NO-BREAK SPACE or other Unicode space (often pasted in from rich +// editors or docs) is genuine whitespace and must keep separating +// tokens. Recognising it here keeps such programs valid — otherwise it +// would fall through to UNEXPECTED_CHARACTER below and fail the whole +// parse. U+FEFF (BOM) is included too, so a file saved with a +// byte-order mark still parses. +WHITESPACE: [ \t\r\n\u000B\u000C\u0085\u00A0\u1680\u2000-\u200A\u2028\u2029\u202F\u205F\u3000\uFEFF] -> channel(HIDDEN); + +// Catch-all for any character no rule above matched. Without this the +// lexer raises a recoverable token-recognition error and DROPS the +// character — so stray input (a JavaScript `!`, `&&`, …) silently +// vanishes and the surrounding text parses as a different, valid-looking +// program. Emitting an explicit token instead means the parser has no +// rule for it and fails loudly with a SyntaxError. Listed last so it +// only ever fires as a true fallback (maximal munch keeps `!=`, `!~`, +// multi-character operators, comments, etc. intact). +UNEXPECTED_CHARACTER: . ; + +// ───────── f' TEMPLATE STRING MODE ───────── +mode IN_TEMPLATE_STRING; +STRING_TEXT: ((~([\\'{])) | ESCAPE_CHAR_COMMON | BACKSLASH QUOTE_SINGLE | (BACKSLASH LBRACE) | (QUOTE_SINGLE QUOTE_SINGLE))+; +STRING_ESCAPE_TRIGGER: LBRACE -> pushMode(DEFAULT_MODE); +STRING_QUOTE_SINGLE: QUOTE_SINGLE -> type(QUOTE_SINGLE), popMode; + +// ───────── F' FULL TEMPLATE STRING MODE ───────── +// a magic F' takes us to "full template strings" mode, where we don't need to escape single quotes and parse until EOF +// this can't be used within a normal columnExpr, but has to be parsed for separately +mode IN_FULL_TEMPLATE_STRING; +FULL_STRING_TEXT: ((~([{])) | ESCAPE_CHAR_COMMON | (BACKSLASH LBRACE))+; +FULL_STRING_ESCAPE_TRIGGER: LBRACE -> pushMode(DEFAULT_MODE); + +// ───────── HOGQLX TAG MODE for opening/self-closing tags ───────── +mode HOGQLX_TAG_OPEN; + +TAG_SELF_CLOSE_GT : '/>' -> type(SLASH_GT), popMode; // +TAG_OPEN_GT : '>' -> type(GT), popMode, pushMode(HOGQLX_TEXT); // + +// Skip comments between attributes — without these, the recoverable lexer error drops the delimiters and re-tokenises the body as phantom attributes. +TAG_MULTI_LINE_COMMENT : '/*' .*? '*/' -> skip; +TAG_SINGLE_LINE_COMMENT : ('--' | '//') ~('\n'|'\r')* ('\n' | '\r' | EOF) -> skip; + +// minimal token set; map everything back to the default token types +TAG_IDENT : [a-zA-Z_][a-zA-Z0-9_-]* -> type(IDENTIFIER); +TAG_EQ : '=' -> type(EQ_SINGLE); +TAG_STRING : STRING_LITERAL -> type(STRING_LITERAL); +TAG_WS : [ \t\r\n]+ -> channel(HIDDEN); +TAG_LBRACE : '{' -> type(LBRACE), pushMode(DEFAULT_MODE); +// Catch-all for unmatched bytes (e.g. `#`, `&`, `@`) so the parser fails loudly instead of silently re-tokenising the surrounding text. +TAG_UNEXPECTED : . -> type(UNEXPECTED_CHARACTER); + + +// ───────── HOGQLX TAG MODE for closing tags ───────── +mode HOGQLX_TAG_CLOSE; + +TAGC_GT : '>' -> type(GT), popMode; // *** no TEXT push *** +TAGC_MULTI_LINE_COMMENT : '/*' .*? '*/' -> skip; +TAGC_SINGLE_LINE_COMMENT : ('--' | '//') ~('\n'|'\r')* ('\n' | '\r' | EOF) -> skip; +TAGC_IDENT : [a-zA-Z_][a-zA-Z0-9_-]* -> type(IDENTIFIER); +TAGC_WS : [ \t\r\n]+ -> channel(HIDDEN); +TAGC_UNEXPECTED : . -> type(UNEXPECTED_CHARACTER); + + +// ───────── HOGQLX TEXT MODE ───────── +mode HOGQLX_TEXT; + +HOGQLX_TEXT_TEXT + : ~[<{]+ ; // everything except “{” or “<” + +HOGQLX_TEXT_LBRACE + : '{' -> type(LBRACE), pushMode(DEFAULT_MODE); + +HOGQLX_TEXT_LT_SLASH + : ' type(LT_SLASH), popMode, pushMode(HOGQLX_TAG_CLOSE); + +HOGQLX_TEXT_LT + : '<' -> type(LT), pushMode(HOGQLX_TAG_OPEN); + +HOGQLX_TEXT_WS + : [ \t\r\n]+ -> channel(HIDDEN); diff --git a/core/trino-hogql-parser/src/main/antlr4/io/trino/hogql/parser/canonical/HogQLParser.g4 b/core/trino-hogql-parser/src/main/antlr4/io/trino/hogql/parser/canonical/HogQLParser.g4 new file mode 100644 index 000000000000..a91e0a2d0f78 --- /dev/null +++ b/core/trino-hogql-parser/src/main/antlr4/io/trino/hogql/parser/canonical/HogQLParser.g4 @@ -0,0 +1,458 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +parser grammar HogQLParser; + +options { + tokenVocab = HogQLLexer; +} + + +program: declaration* EOF; + +declaration: varDecl | statement ; + +expression: columnExpr; + +varDecl: LET identifier ( COLONEQUALS expression )? ; +identifierList: nestedIdentifier (COMMA nestedIdentifier)* COMMA?; + +statement : returnStmt + | throwStmt + | tryCatchStmt + | ifStmt + | whileStmt + | forInStmt + | forStmt + | funcStmt + | block + | exprStmt + | emptyStmt + ; + +returnStmt : RETURN expression? SEMICOLON?; +throwStmt : THROW expression SEMICOLON?; +catchBlock : CATCH (LPAREN catchVar=identifier (COLON catchType=identifier)? RPAREN)? catchStmt=block; +tryCatchStmt : TRY tryStmt=block catchBlock* (FINALLY finallyStmt=block)?; +ifStmt : IF LPAREN expression RPAREN statement ( ELSE statement )? ; +whileStmt : WHILE LPAREN expression RPAREN statement SEMICOLON?; +forStmt : FOR LPAREN + (initializerVarDeclr=varDecl | initializerVarAssignment=varAssignment | initializerExpression=expression)? SEMICOLON + condition=expression? SEMICOLON + (incrementVarDeclr=varDecl | incrementVarAssignment=varAssignment | incrementExpression=expression)? + RPAREN statement SEMICOLON?; +forInStmt : FOR LPAREN LET identifier (COMMA identifier)? IN expression RPAREN statement SEMICOLON?; +funcStmt : (FN | FUN) identifier LPAREN identifierList? RPAREN block; +varAssignment : expression COLONEQUALS expression ; +// Assignment folded in as an optional suffix: one expression-leading alternative in +// `statement` means `declaration*` parses without unbounded lookahead. `varAssignment` is forStmt-only. +exprStmt : expression (COLONEQUALS expression)? SEMICOLON?; +emptyStmt : SEMICOLON ; +block : LBRACE declaration* RBRACE ; + +kvPair: expression ':' expression ; +kvPairList: kvPair (COMMA kvPair)* COMMA?; + + +// SELECT statement +select: (selectSetStmt | selectStmt | hogqlxTagElement) SEMICOLON? EOF; + +selectStmtWithParens: selectStmt | withClause LPAREN selectSetStmt RPAREN | LPAREN selectSetStmt RPAREN | placeholder; + +subsequentSelectSetClause: (EXCEPT ALL (BY NAME)? | EXCEPT (BY NAME)? | UNION ALL (BY NAME)? | UNION DISTINCT (BY NAME)? | UNION (BY NAME)? | INTERSECT ALL (BY NAME)? | INTERSECT DISTINCT (BY NAME)? | INTERSECT (BY NAME)?) selectStmtWithParens; +selectSetStmt: selectStmtWithParens (subsequentSelectSetClause)* orderByClause? limitAndOffsetClauseOptional?; +limitAndOffsetClauseOptional + : LIMIT columnExpr PERCENT? (COMMA columnExpr)? (WITH TIES)? + | LIMIT columnExpr PERCENT? (WITH TIES)? OFFSET columnExpr + | OFFSET columnExpr + ; + +selectStmt: + with=withClause? + SELECT DISTINCT? topClause? + columns=selectColumnExprListBeforeFrom + from=fromClause? + arrayJoinClause? + prewhereClause? + where=whereClause? + (USING? sampleClause)? + groupByClause? (WITH (CUBE | ROLLUP))? (WITH TOTALS)? + havingClause? + qualifyClause? + (USING sampleClause)? + windowClause? + orderByClause? + limitByClause? + (limitAndOffsetClause | offsetOnlyClause)? + settingsClause? + ; + +withClause: WITH RECURSIVE? withExprList; +topClause: TOP DECIMAL_LITERAL (WITH TIES)?; +fromClause: FROM joinExpr; +arrayJoinClause: (LEFT | INNER)? ARRAY JOIN columnExprList; +windowClause: WINDOW identifier AS LPAREN windowExpr RPAREN (COMMA identifier AS LPAREN windowExpr RPAREN)*; +prewhereClause: PREWHERE columnExpr; +whereClause: WHERE columnExpr; +groupByClause: GROUP BY ( + ALL + | (CUBE | ROLLUP) LPAREN columnExprList RPAREN + | GROUPING SETS LPAREN groupingSetList RPAREN + | columnExprList + ); +groupingSetList: groupingSet (COMMA groupingSet)*; +groupingSet: LPAREN columnExprList? RPAREN; +havingClause: HAVING columnExpr; +qualifyClause: QUALIFY columnExpr; +orderByClause: ORDER BY orderExprList interpolateClause?; +interpolateClause: INTERPOLATE (LPAREN interpolateExpr (COMMA interpolateExpr)* RPAREN)?; +projectionOrderByClause: ORDER BY columnExprList; +limitByClause: LIMIT limitExpr BY columnExprList; +limitAndOffsetClause + : LIMIT columnExpr PERCENT? (COMMA columnExpr)? (WITH TIES)? // compact OFFSET-optional form + | LIMIT columnExpr PERCENT? (WITH TIES)? OFFSET columnExpr // verbose OFFSET-included form with WITH TIES + ; +offsetOnlyClause: OFFSET columnExpr; +settingsClause: SETTINGS settingExprList; + +valuesClause: VALUES valuesRow (COMMA valuesRow)*; +valuesRow: LPAREN columnExpr (COMMA columnExpr)* RPAREN; + +joinExpr + : joinExpr NATURAL? joinOp? JOIN joinExpr joinConstraintClause? # JoinExprOp + | joinExpr POSITIONAL JOIN joinExpr joinConstraintClause? # JoinExprPositional + | joinExpr joinOpCross joinExpr # JoinExprCrossOp + | joinExpr PIVOT LPAREN columnExprList pivotColumnList (GROUP BY columnExprList)? RPAREN # JoinExprPivot + | joinExpr UNPIVOT (INCLUDE NULLS)? LPAREN unpivotColumnList RPAREN # JoinExprUnpivot + | tableExpr FINAL? sampleClause? # JoinExprTable + | LPAREN joinExpr RPAREN # JoinExprParens + ; +joinOp + : ((ALL | ANY | ASOF)? INNER | INNER (ALL | ANY | ASOF)? | (ALL | ANY | ASOF) | ANTI | SEMI | ASOF (ANTI | SEMI)) # JoinOpInner + | ( (SEMI | ALL | ANTI | ANY | ASOF)? (LEFT | RIGHT) OUTER? + | (LEFT | RIGHT) OUTER? (SEMI | ALL | ANTI | ANY | ASOF)? + | ASOF (ANTI | SEMI) (LEFT | RIGHT) OUTER? + ) # JoinOpLeftRight + | ((ALL | ANY | ASOF)? FULL OUTER? | FULL OUTER? (ALL | ANY | ASOF)?) # JoinOpFull + ; +joinOpCross + : CROSS JOIN + | COMMA + ; +joinConstraintClause + : ON columnExprList + | USING LPAREN columnExprList RPAREN + | USING columnExprList + ; + +sampleClause: SAMPLE ratioExpr PERCENT? (OFFSET ratioExpr)? (LPAREN identifier RPAREN)?; +limitExpr: columnExpr ((COMMA | OFFSET) columnExpr)?; +orderExprList: orderExpr (COMMA orderExpr)*; +orderExpr: columnExpr (ASCENDING | DESCENDING | DESC)? (NULLS (FIRST | LAST))? (COLLATE STRING_LITERAL)? withFillClause?; +withFillClause: WITH FILL (FROM columnExpr)? (TO columnExpr)? (STEP columnExpr)?; +interpolateExpr: columnExpr (AS columnExpr)?; +ratioExpr: placeholder | numberLiteral (SLASH numberLiteral)?; +settingExprList: settingExpr (COMMA settingExpr)*; +settingExpr: identifier EQ_SINGLE literal; + +windowExpr: winPartitionByClause? winOrderByClause? winFrameClause?; +winPartitionByClause: PARTITION BY columnExprList; +winOrderByClause: ORDER BY orderExprList; +withinGroupClause: WITHIN GROUP LPAREN orderByClause RPAREN; +winFrameClause: (ROWS | RANGE) winFrameExtend; +winFrameExtend + : winFrameBound # frameStart + | BETWEEN winFrameBound AND winFrameBound # frameBetween + ; +winFrameBound: (CURRENT ROW | UNBOUNDED PRECEDING | UNBOUNDED FOLLOWING | columnExpr PRECEDING | columnExpr FOLLOWING); +//rangeClause: RANGE LPAREN (MIN identifier MAX identifier | MAX identifier MIN identifier) RPAREN; + +// Columns +expr: columnExpr EOF; +columnTypeExpr + : columnTypeExpr LBRACKET DECIMAL_LITERAL? RBRACKET # ColumnTypeExprArray // INTEGER[], VARCHAR[3] + | identifier LPAREN identifier columnTypeExpr (COMMA identifier columnTypeExpr)* COMMA? RPAREN # ColumnTypeExprNested // Nested + | identifier LPAREN enumValue (COMMA enumValue)* COMMA? RPAREN # ColumnTypeExprEnum // Enum + | identifier LPAREN columnTypeExpr (COMMA columnTypeExpr)* COMMA? RPAREN # ColumnTypeExprComplex // Array, Tuple + | identifier LPAREN columnExprList? RPAREN # ColumnTypeExprParam // FixedString(N) + | identifier identifier+ # ColumnTypeExprCompound // TIME WITH TIME ZONE + | identifier # ColumnTypeExprSimple // UInt64 + ; +// Restricted type expr for :: casts — no parenthesized variants to avoid ambiguity with function calls +columnTypeCastExpr + : columnTypeCastIdentifier WITH LOCAL? TIME ZONE # ColumnTypeCastExprWithTimeZone + | columnTypeCastIdentifier # ColumnTypeCastExprSimple + ; +columnTypeCastIdentifier + : IDENTIFIER + | QUOTED_IDENTIFIER + | interval + | keywordForTypeCast + ; +keywordForTypeCast + : DATE + | TIME + | TIMESTAMP + | INTERVAL + ; +columnExprList: columnExpr (COMMA columnExpr)* COMMA?; +selectColumnExprListBeforeFrom + : selectColumnExpr (COMMA selectColumnExpr)* COMMA # SelectColumnExprListBeforeFromTrailingComma + | selectColumnExprList # SelectColumnExprListBeforeFromPlain + ; +selectColumnExprList: selectColumnExpr (COMMA selectColumnExpr)* COMMA?; +selectColumnExpr + : identifier COLON columnExpr # ColumnExprAliasBefore + | FROM implicitAlias # ColumnExprInvalidFromImplicitAlias + | columnExpr # ColumnExprSelectValue + | columnExpr implicitAlias # ColumnExprAliasImplicit + ; +// Two precedence layers. `columnExpr` is the outer boolean/ternary/alias tier (loosest +// binding). `columnExprValue` holds everything tighter — arithmetic, comparisons, NOT, +// BETWEEN, and the primary/leaf productions. BETWEEN lives in the value tier at the comparison +// level, so its tested expression and both bounds bind tighter than AND/OR/NOT (matching +// ClickHouse and SQL); this is what fixes `a BETWEEN low AND high AND rest` grouping as +// `(a BETWEEN low AND high) AND rest` rather than letting the bounds swallow the AND chain. +// Splitting into two rules is the only way to express this in ANTLR4: the interior operand of +// a left-recursive alternative always parses at precedence 0, so a flat rule cannot stop the +// bounds from consuming AND (the reason for ilezhankin's original TODO). NOT stays in +// `columnExprValue` (looser than every value operator, tighter than AND/OR) so it keeps sitting +// *after* `ColumnExprFunction` in the same rule — ANTLR then still prefers the function form for +// `not(args)` (a `not` function call) over the `NOT (args)` operator, matching the old grammar. +columnExpr + : columnExpr AND columnExpr # ColumnExprAnd + | columnExpr OR columnExpr # ColumnExprOr + | columnExpr QUERY columnExpr COLON columnExpr # ColumnExprTernaryOp + | columnExpr AS (identifier | STRING_LITERAL) # ColumnExprAlias + | columnExprValue # ColumnExprValuePassthrough + ; + +columnExprValue + : CASE caseExpr=columnExpr? (WHEN whenExpr=columnExpr THEN thenExpr=columnExpr)+ (ELSE elseExpr=columnExpr)? END # ColumnExprCase + | CAST LPAREN columnExpr AS columnTypeExpr RPAREN # ColumnExprCast + | TRY_CAST LPAREN columnExpr AS columnTypeExpr RPAREN # ColumnExprTryCast + | DATE STRING_LITERAL # ColumnExprDate +// | EXTRACT LPAREN interval FROM columnExpr RPAREN # ColumnExprExtract // Interferes with a function call + | INTERVAL columnExpr interval # ColumnExprInterval + | INTERVAL STRING_LITERAL # ColumnExprIntervalString + | SUBSTRING LPAREN columnExpr FROM columnExpr (FOR columnExpr)? RPAREN # ColumnExprSubstring + | TIMESTAMP STRING_LITERAL # ColumnExprTimestamp + | TRIM LPAREN (BOTH | LEADING | TRAILING) string FROM columnExpr RPAREN # ColumnExprTrim + | COLUMNS LPAREN STRING_LITERAL RPAREN # ColumnExprColumnsRegex + | COLUMNS LPAREN columnExprList RPAREN # ColumnExprColumnsList + | (COLUMNS LPAREN ASTERISK EXCLUDE LPAREN identifierList RPAREN REPLACE LPAREN columnsReplaceList RPAREN RPAREN + | LPAREN ASTERISK EXCLUDE LPAREN identifierList RPAREN REPLACE LPAREN columnsReplaceList RPAREN RPAREN + ) # ColumnExprColumnsExcludeReplace + | COLUMNS LPAREN ASTERISK EXCLUDE LPAREN identifierList RPAREN RPAREN # ColumnExprColumnsExclude + | (COLUMNS LPAREN ASTERISK REPLACE LPAREN columnsReplaceList RPAREN RPAREN + | LPAREN ASTERISK REPLACE LPAREN columnsReplaceList RPAREN RPAREN + ) # ColumnExprColumnsReplace + | COLUMNS LPAREN ASTERISK RPAREN # ColumnExprColumnsAll + | COLUMNS LPAREN identifier DOT ASTERISK EXCLUDE LPAREN identifierList RPAREN REPLACE LPAREN columnsReplaceList RPAREN RPAREN # ColumnExprColumnsQualifiedExcludeReplace + | COLUMNS LPAREN identifier DOT ASTERISK EXCLUDE LPAREN identifierList RPAREN RPAREN # ColumnExprColumnsQualifiedExclude + | COLUMNS LPAREN identifier DOT ASTERISK REPLACE LPAREN columnsReplaceList RPAREN RPAREN # ColumnExprColumnsQualifiedReplace + | COLUMNS LPAREN identifier DOT ASTERISK RPAREN # ColumnExprColumnsQualifiedAll + | ASTERISK COLUMNS LPAREN STRING_LITERAL RPAREN # ColumnExprSpreadColumnsRegex + | ASTERISK COLUMNS LPAREN columnExprList RPAREN # ColumnExprSpreadColumnsList + | identifier LPAREN columnExprs=columnExprList? RPAREN withinGroupClause # ColumnExprFunctionWithinGroup + | identifier (LPAREN columnExprs=columnExprList? RPAREN) (LPAREN DISTINCT? columnArgList=columnExprList? RPAREN)? (FILTER LPAREN WHERE filterExpr=columnExpr RPAREN)? OVER LPAREN windowExpr RPAREN # ColumnExprWinFunction + | identifier (LPAREN columnExprs=columnExprList? RPAREN) (LPAREN DISTINCT? columnArgList=columnExprList? RPAREN)? (FILTER LPAREN WHERE filterExpr=columnExpr RPAREN)? OVER identifier # ColumnExprWinFunctionTarget + | identifier (LPAREN columnExprs=columnExprList? RPAREN)? LPAREN DISTINCT? columnArgList=columnExprList? (ORDER BY orderExprList)? RPAREN (FILTER LPAREN WHERE filterExpr=columnExpr RPAREN)? # ColumnExprFunction + | columnExprValue LPAREN selectSetStmt RPAREN # ColumnExprCallSelect + | columnExprValue LPAREN columnExprList? RPAREN # ColumnExprCall + | hogqlxTagElement # ColumnExprTagElement + | templateString # ColumnExprTemplateString + | literal # ColumnExprLiteral + + // FIXME(ilezhankin): this part looks very ugly, maybe there is another way to express it + | columnExprValue LBRACKET columnExpr RBRACKET # ColumnExprArrayAccess + | columnExprValue LBRACKET columnExpr? COLON columnExpr? RBRACKET # ColumnExprArraySlice + | columnExprValue DOT DECIMAL_LITERAL # ColumnExprTupleAccess + | columnExprValue DOT identifier # ColumnExprPropertyAccess + | columnExprValue NULL_PROPERTY LBRACKET columnExpr RBRACKET # ColumnExprNullArrayAccess + | columnExprValue NULL_PROPERTY DECIMAL_LITERAL # ColumnExprNullTupleAccess + | columnExprValue NULL_PROPERTY identifier # ColumnExprNullPropertyAccess + | columnExprValue DOUBLECOLON columnTypeCastExpr # ColumnExprTypeCast + | DASH columnExprValue # ColumnExprNegate + | left=columnExprValue ( operator=ASTERISK // * + | operator=SLASH // / + | operator=PERCENT // % + ) right=columnExprValue # ColumnExprPrecedence1 + | left=columnExprValue ( operator=PLUS // + + | operator=DASH // - + | operator=CONCAT // || + ) right=columnExprValue # ColumnExprPrecedence2 + | left=columnExprValue ( operator=EQ_DOUBLE // = + | operator=EQ_SINGLE // == + | operator=NOT_EQ // != + | operator=LT_EQ // <= + | operator=LT // < + | operator=GT_EQ // >= + | operator=GT // > + | operator=NOT? IN COHORT? // in, not in; in cohort; not in cohort + | operator=NOT? (LIKE | ILIKE) // like, not like, ilike, not ilike + | operator=REGEX_SINGLE // ~ + | operator=REGEX_DOUBLE // =~ + | operator=NOT_REGEX // !~ + | operator=IREGEX_SINGLE // ~* + | operator=IREGEX_DOUBLE // =~* + | operator=NOT_IREGEX // !~* + ) right=columnExprValue # ColumnExprPrecedence3 + | columnExprValue IGNORE NULLS # ColumnExprIgnoreNulls + | columnExprValue IS NOT? NULL_SQL # ColumnExprIsNull + | columnExprValue IS NOT? DISTINCT FROM columnExprValue # ColumnExprIsDistinctFrom + | columnExprValue NULL_SAFE_EQ columnExprValue # ColumnExprNullSafeEq // MySQL `a <=> b` ≡ `a IS NOT DISTINCT FROM b` + | columnExprValue NULLISH columnExprValue # ColumnExprNullish + | columnExprValue NOT? BETWEEN columnExprValue AND columnExprValue # ColumnExprBetween + | NOT columnExprValue # ColumnExprNot + | (tableIdentifier DOT)? ASTERISK (EXCLUDE LPAREN identifierList RPAREN)? # ColumnExprAsterisk // single-column only + | LAMBDA identifier (COMMA identifier)* COMMA? COLON columnExpr # ColumnExprColonLambda + | LPAREN selectSetStmt RPAREN # ColumnExprSubquery // single-column only + | LPAREN columnExpr RPAREN # ColumnExprParens // single-column only + | LPAREN columnExprList RPAREN # ColumnExprTuple + | ARRAY? LBRACKET columnExprList? RBRACKET # ColumnExprArray + | LBRACE (kvPairList)? RBRACE # ColumnExprDict + | columnLambdaExpr # ColumnExprLambda + | identifier COLONEQUALS columnExpr # ColumnExprNamedArg + | HASH DECIMAL_LITERAL # ColumnExprPositional + | columnIdentifier # ColumnExprIdentifier + ; + +columnLambdaExpr: + ( LPAREN identifier (COMMA identifier)* COMMA? RPAREN + | identifier (COMMA identifier)* COMMA? + | LPAREN RPAREN + ) + ARROW (columnExpr | block) # ArrowLambda + | LAMBDA identifier (COMMA identifier)* COMMA? COLON columnExpr # ColonLambda + ; + +columnsReplaceList: columnsReplaceItem (COMMA columnsReplaceItem)*; +columnsReplaceItem: columnExpr AS identifier; + +hogqlxChildElement + : hogqlxTagElement + | hogqlxText + | LBRACE columnExpr RBRACE; + +hogqlxText : HOGQLX_TEXT_TEXT ; + +hogqlxTagElement + : LT identifier hogqlxTagAttribute* SLASH_GT # HogqlxTagElementClosed + | LT identifier hogqlxTagAttribute* GT hogqlxChildElement* LT_SLASH identifier GT # HogqlxTagElementNested + ; +hogqlxTagAttribute + : identifier EQ_SINGLE string + | identifier EQ_SINGLE LBRACE columnExpr RBRACE + | identifier + ; + +withExprList: withExpr (COMMA withExpr)* COMMA?; +withExpr + : identifier withExprColumnNameList? (USING KEY withExprColumnNameList)? AS (NOT? MATERIALIZED)? LPAREN selectSetStmt RPAREN # WithExprSubquery + // NOTE: asterisk and subquery goes before |columnExpr| so that we can mark them as multi-column expressions. + | columnExpr AS identifier # WithExprColumn + ; + +withExprColumnNameList: LPAREN identifier (COMMA identifier)* RPAREN; + + +// This is slightly different in HogQL compared to ClickHouse SQL +// HogQL allows unlimited ("*") nestedIdentifier-s "properties.b.a.a.w.a.s". +// We parse and convert "databaseIdentifier.tableIdentifier.columnIdentifier.nestedIdentifier.*" +// to just one ast.Field(chain=['a','b','columnIdentifier','on','and','on']). +columnIdentifier: placeholder | ((tableIdentifier DOT)? nestedIdentifier); +nestedIdentifier: identifier (DOT identifier)*; +tableExpr + : tableIdentifier # TableExprIdentifier + | tableFunctionExpr # TableExprFunction + | LPAREN selectSetStmt RPAREN # TableExprSubquery + | LPAREN valuesClause RPAREN # TableExprValues + | tableExpr PIVOT LPAREN columnExprList pivotColumnList (GROUP BY columnExprList)? RPAREN # TableExprPivot + | tableExpr UNPIVOT (INCLUDE NULLS)? LPAREN unpivotColumnList RPAREN # TableExprUnpivot + | tableExpr (alias | AS identifier) columnAliases? # TableExprAlias + | hogqlxTagElement # TableExprTag + | placeholder # TableExprPlaceholder + ; + +pivotColumnList: FOR pivotColumn+; +pivotColumn: columnExprTupleOrSingle IN LPAREN columnExprList RPAREN; +unpivotColumnList: unpivotColumn (COMMA unpivotColumn)* COMMA?; +unpivotColumn: columnExprTupleOrSingle FOR columnExprTupleOrSingle IN LPAREN columnExprList RPAREN (columnExprTupleOrSingle IN LPAREN columnExprList RPAREN)*; +columnExprTupleOrSingle: LPAREN columnExprList RPAREN | columnExpr; +columnAliases: LPAREN identifier (COMMA identifier)* RPAREN; +tableFunctionExpr: identifier LPAREN tableArgList? RPAREN; +tableIdentifier: (databaseIdentifier DOT)? nestedIdentifier; +tableArgList: columnExpr (COMMA columnExpr)* COMMA?; + +// Databases + +databaseIdentifier: identifier; + +// Basics + +floatingLiteral + : FLOATING_LITERAL + | DOT (DECIMAL_LITERAL | OCTAL_LITERAL) + | DECIMAL_LITERAL DOT (DECIMAL_LITERAL | OCTAL_LITERAL)? // can't move this to the lexer or it will break nested tuple access: t.1.2 + ; +numberLiteral: (PLUS | DASH)? (floatingLiteral | BINARY_LITERAL | OCTAL_LITERAL | OCTAL_PREFIX_LITERAL | DECIMAL_LITERAL | HEXADECIMAL_LITERAL | INF | NAN_SQL); +literal + : numberLiteral + | STRING_LITERAL + | NULL_SQL + ; +interval: SECOND | MINUTE | HOUR | DAY | WEEK | MONTH | QUARTER | YEAR; +keyword + // except NULL_SQL, INF, NAN_SQL + : ALL | AND | ANTI | ANY | ARRAY | AS | ASCENDING | ASOF | BETWEEN | BOTH | BY | CASE + | CAST | COHORT | COLLATE | COLUMNS | CROSS | CUBE | CURRENT | DATE | DESC | DESCENDING + | DISTINCT | ELSE | END | EXCLUDE | EXTRACT | FILL | FILTER | FINAL | FIRST + | FOR | FOLLOWING | FROM | FULL | GROUP | HAVING | ID | INTERPOLATE | IS + | GROUPING | IF | IGNORE | ILIKE | INCLUDE | IN | INNER | INTERVAL | JOIN | KEY + | LAMBDA | LAST | LEADING | LEFT | LIKE | LIMIT + | LOCAL | NAME | NATURAL | NOT | NULLS | OFFSET | ON | OR | ORDER | OUTER | OVER | PARTITION + | PIVOT | POSITIONAL | PRECEDING | PREWHERE | QUALIFY | RANGE | RECURSIVE | REPLACE | RETURN | RIGHT | ROLLUP | ROW + | ROWS | SAMPLE | SELECT | SEMI | SETS | SETTINGS | STEP | SUBSTRING + | THEN | TIES | TIME | TIMESTAMP | TOTALS | TRAILING | TRIM | TRUNCATE | TRY_CAST | TO | TOP + | UNBOUNDED | UNION | UNPIVOT | USING | VALUES | WHEN | WHERE | WINDOW | WITH + | ZONE + ; +keywordForAlias + : DATE | FIRST | ID | KEY + ; +keywordForImplicitAlias + : ASCENDING + | COHORT + | DATE + | DESCENDING + | FINAL + | ID + | RETURN + | TOP + | TOTALS + ; +alias: IDENTIFIER | QUOTED_IDENTIFIER | keywordForAlias; // |interval| can't be an alias, otherwise 'INTERVAL 1 SOMETHING' becomes ambiguous. +implicitAlias: IDENTIFIER | QUOTED_IDENTIFIER | keywordForImplicitAlias; +identifier: IDENTIFIER | QUOTED_IDENTIFIER | interval | keyword; +enumValue: string EQ_SINGLE numberLiteral; +placeholder: LBRACE columnExpr RBRACE; + +string: STRING_LITERAL | templateString; +templateString : QUOTE_SINGLE_TEMPLATE stringContents* QUOTE_SINGLE ; +stringContents : STRING_ESCAPE_TRIGGER columnExpr RBRACE | STRING_TEXT; + +// These are magic "full template strings", which are used to parse "full text field" templates without the surrounding SQL. +// We will need to add F' to the start of the string to change the lexer's mode. +fullTemplateString: QUOTE_SINGLE_TEMPLATE_FULL stringContentsFull* EOF ; +stringContentsFull : FULL_STRING_ESCAPE_TRIGGER columnExpr RBRACE | FULL_STRING_TEXT; diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlCompatibilityManifest.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlCompatibilityManifest.java new file mode 100644 index 000000000000..cd7cacd28601 --- /dev/null +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlCompatibilityManifest.java @@ -0,0 +1,323 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonValue; +import io.airlift.json.JsonCodec; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HashMap; +import java.util.HexFormat; +import java.util.List; +import java.util.Map; + +import static io.airlift.json.JsonCodec.jsonCodec; +import static java.util.Objects.requireNonNull; + +public record HogQlCompatibilityManifest( + int schemaVersion, + HogQlLanguageVersion languageVersion, + String grammarSha256, + String grammarAlternativeIdentityManifestSha256, + String grammarFeatureManifestSha256, + List sourceUnlabeledAlternativeRules, + List features) +{ + private static final String LANGUAGE_RESOURCE_ROOT = "/io/trino/hogql/parser/language/1.0.0/"; + private static final String CURRENT_MANIFEST_RESOURCE = LANGUAGE_RESOURCE_ROOT + "trino-compatibility.json"; + private static final JsonCodec MANIFEST_CODEC = jsonCodec(HogQlCompatibilityManifest.class); + private static final JsonCodec GRAMMAR_FEATURE_MANIFEST_CODEC = jsonCodec(PublishedGrammarFeatureManifest.class); + private static final HogQlCompatibilityManifest CURRENT = loadCurrent(); + + @JsonCreator + public HogQlCompatibilityManifest( + @JsonProperty("schemaVersion") int schemaVersion, + @JsonProperty("languageVersion") HogQlLanguageVersion languageVersion, + @JsonProperty("grammarSha256") String grammarSha256, + @JsonProperty("grammarAlternativeIdentityManifestSha256") String grammarAlternativeIdentityManifestSha256, + @JsonProperty("grammarFeatureManifestSha256") String grammarFeatureManifestSha256, + @JsonProperty("sourceUnlabeledAlternativeRules") List sourceUnlabeledAlternativeRules, + @JsonProperty("features") List features) + { + if (schemaVersion != 1) { + throw new IllegalArgumentException("unsupported HogQL compatibility manifest schema: " + schemaVersion); + } + this.schemaVersion = schemaVersion; + this.languageVersion = requireNonNull(languageVersion, "languageVersion is null"); + this.grammarSha256 = requireNonNull(grammarSha256, "grammarSha256 is null"); + this.grammarAlternativeIdentityManifestSha256 = requireNonNull(grammarAlternativeIdentityManifestSha256, "grammarAlternativeIdentityManifestSha256 is null"); + this.grammarFeatureManifestSha256 = requireNonNull(grammarFeatureManifestSha256, "grammarFeatureManifestSha256 is null"); + this.sourceUnlabeledAlternativeRules = List.copyOf(requireNonNull(sourceUnlabeledAlternativeRules, "sourceUnlabeledAlternativeRules is null")); + this.features = List.copyOf(requireNonNull(features, "features is null")); + } + + public static HogQlCompatibilityManifest current() + { + return CURRENT; + } + + private static HogQlCompatibilityManifest loadCurrent() + { + HogQlCompatibilityManifest manifest = MANIFEST_CODEC.fromJson(readResource(CURRENT_MANIFEST_RESOURCE)); + HogQlLanguageContract languageContract = HogQlLanguageContract.current(); + if (!manifest.languageVersion().equals(languageContract.languageVersion())) { + throw new IllegalStateException("HogQL compatibility manifest language version does not match the language contract"); + } + if (!manifest.grammarSha256().equals(languageContract.grammarSha256())) { + throw new IllegalStateException("HogQL compatibility manifest grammar hash does not match the language contract"); + } + if (!manifest.grammarFeatureManifestSha256().equals(languageContract.grammarFeatureManifest().sha256())) { + throw new IllegalStateException("HogQL compatibility manifest feature hash does not match the language contract"); + } + HogQlGrammarAlternativeIdentities alternativeIdentities = HogQlGrammarAlternativeIdentities.current(); + if (!manifest.grammarAlternativeIdentityManifestSha256().equals(HogQlGrammarAlternativeIdentities.currentSha256())) { + throw new IllegalStateException("HogQL compatibility manifest alternative identity hash does not match the published sidecar"); + } + + String featureManifestResource = LANGUAGE_RESOURCE_ROOT + languageContract.grammarFeatureManifest().path(); + byte[] featureManifestBytes = readResourceBytes(featureManifestResource); + if (!sha256(featureManifestBytes).equals(manifest.grammarFeatureManifestSha256())) { + throw new IllegalStateException("published HogQL grammar feature manifest checksum does not match the compatibility manifest"); + } + PublishedGrammarFeatureManifest published = GRAMMAR_FEATURE_MANIFEST_CODEC.fromJson(new String(featureManifestBytes, StandardCharsets.UTF_8)); + validateFeatures(manifest, published, alternativeIdentities); + return manifest; + } + + private static void validateFeatures( + HogQlCompatibilityManifest manifest, + PublishedGrammarFeatureManifest published, + HogQlGrammarAlternativeIdentities alternativeIdentities) + { + if (!published.languageVersion().equals(manifest.languageVersion()) || !published.grammarSha256().equals(manifest.grammarSha256())) { + throw new IllegalStateException("published HogQL grammar features do not match the compatibility manifest language"); + } + + Map publishedById = new HashMap<>(); + for (PublishedFeature feature : published.features()) { + if (publishedById.put(feature.id(), feature) != null) { + throw new IllegalStateException("duplicate published HogQL grammar feature: " + feature.id()); + } + } + + Map sourceUnlabeledByRule = new HashMap<>(); + for (SourceUnlabeledAlternativeRule sourceUnlabeled : published.validationErrors()) { + if (sourceUnlabeledByRule.put(sourceUnlabeled.rule(), sourceUnlabeled) != null) { + throw new IllegalStateException("duplicate source-unlabeled HogQL grammar alternative rule: " + sourceUnlabeled.rule()); + } + } + for (HogQlGrammarAlternativeIdentities.AlternativeRule rule : alternativeIdentities.rules()) { + SourceUnlabeledAlternativeRule sourceUnlabeled = sourceUnlabeledByRule.remove(rule.rule()); + if (sourceUnlabeled == null || sourceUnlabeled.alternativeCount() != rule.alternatives().size() || sourceUnlabeled.queryReachable() != rule.queryReachable()) { + throw new IllegalStateException("HogQL grammar alternative identities do not match the published rule: " + rule.rule()); + } + for (HogQlGrammarAlternativeIdentities.Alternative alternative : rule.alternatives()) { + PublishedFeature feature = new PublishedFeature(alternative.id(), "parserAlternative", rule.queryReachable()); + if (publishedById.put(feature.id(), feature) != null) { + throw new IllegalStateException("duplicate published HogQL grammar feature: " + feature.id()); + } + } + } + if (!sourceUnlabeledByRule.isEmpty()) { + throw new IllegalStateException("HogQL grammar alternative identities do not cover every source-unlabeled rule"); + } + + Map compatibilityById = new HashMap<>(); + for (Feature feature : manifest.features()) { + if (compatibilityById.put(feature.id(), feature) != null) { + throw new IllegalStateException("duplicate HogQL compatibility feature: " + feature.id()); + } + PublishedFeature source = publishedById.get(feature.id()); + if (source == null) { + throw new IllegalStateException("unknown HogQL compatibility feature: " + feature.id()); + } + if (!feature.kind().equals(source.kind()) || feature.queryReachable() != source.queryReachable()) { + throw new IllegalStateException("HogQL compatibility feature does not match its published definition: " + feature.id()); + } + if (feature.queryReachable() == (feature.status() == Status.NOT_QUERY_LANGUAGE)) { + throw new IllegalStateException("invalid HogQL query-language status for feature: " + feature.id()); + } + } + if (!compatibilityById.keySet().equals(publishedById.keySet())) { + throw new IllegalStateException("HogQL compatibility manifest does not account for every published grammar feature"); + } + if (!manifest.sourceUnlabeledAlternativeRules().equals(published.validationErrors())) { + throw new IllegalStateException("HogQL compatibility manifest does not acknowledge every source-unlabeled grammar alternative rule"); + } + } + + private static String readResource(String path) + { + return new String(readResourceBytes(path), StandardCharsets.UTF_8); + } + + private static byte[] readResourceBytes(String path) + { + try (InputStream input = HogQlCompatibilityManifest.class.getResourceAsStream(path)) { + if (input == null) { + throw new IllegalStateException("HogQL compatibility resource is missing: " + path); + } + return input.readAllBytes(); + } + catch (IOException e) { + throw new UncheckedIOException("failed to read HogQL compatibility resource", e); + } + } + + private static String sha256(byte[] content) + { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(content)); + } + catch (NoSuchAlgorithmException e) { + throw new AssertionError(e); + } + } + + public record Feature( + String id, + String kind, + boolean queryReachable, + Status status, + String parseHandler, + String loweringHandler, + List testCaseIds) + { + @JsonCreator + public Feature( + @JsonProperty("id") String id, + @JsonProperty("kind") String kind, + @JsonProperty("queryReachable") boolean queryReachable, + @JsonProperty("status") Status status, + @JsonProperty("parseHandler") String parseHandler, + @JsonProperty("loweringHandler") String loweringHandler, + @JsonProperty("testCaseIds") List testCaseIds) + { + this.id = requireNonNull(id, "id is null"); + this.kind = requireNonNull(kind, "kind is null"); + this.queryReachable = queryReachable; + this.status = requireNonNull(status, "status is null"); + this.parseHandler = requireNonNull(parseHandler, "parseHandler is null"); + this.loweringHandler = requireNonNull(loweringHandler, "loweringHandler is null"); + this.testCaseIds = List.copyOf(requireNonNull(testCaseIds, "testCaseIds is null")); + if (id.isBlank() || kind.isBlank() || parseHandler.isBlank() || loweringHandler.isBlank() || testCaseIds.isEmpty() || testCaseIds.stream().anyMatch(String::isBlank)) { + throw new IllegalArgumentException("HogQL compatibility feature fields must be non-empty: " + id); + } + if (status == Status.SUPPORTED && (parseHandler.equals("UnsupportedFeature") || loweringHandler.equals("UnsupportedFeature"))) { + throw new IllegalArgumentException("supported HogQL feature must have parse and lowering handlers: " + id); + } + if (status == Status.EXPLICIT_CURRENT_ERROR && !loweringHandler.equals("UnsupportedFeature")) { + throw new IllegalArgumentException("unsupported HogQL feature must use the explicit error handler: " + id); + } + if (status == Status.NOT_QUERY_LANGUAGE && (!parseHandler.equals("NotQueryLanguage") || !loweringHandler.equals("NotQueryLanguage"))) { + throw new IllegalArgumentException("non-query HogQL feature must not have query handlers: " + id); + } + } + } + + public enum Status + { + SUPPORTED("supported"), + EXPLICIT_CURRENT_ERROR("explicitCurrentError"), + NOT_QUERY_LANGUAGE("notQueryLanguage"); + + private final String value; + + Status(String value) + { + this.value = value; + } + + @JsonCreator + public static Status fromJson(String value) + { + for (Status status : values()) { + if (status.value.equals(value)) { + return status; + } + } + throw new IllegalArgumentException("unknown HogQL compatibility status: " + value); + } + + @JsonValue + public String toJson() + { + return value; + } + } + + public record SourceUnlabeledAlternativeRule( + int alternativeCount, + String code, + boolean queryReachable, + String rule) + { + @JsonCreator + public SourceUnlabeledAlternativeRule( + @JsonProperty("alternativeCount") int alternativeCount, + @JsonProperty("code") String code, + @JsonProperty("queryReachable") boolean queryReachable, + @JsonProperty("rule") String rule) + { + if (alternativeCount < 2) { + throw new IllegalArgumentException("source-unlabeled grammar rule must have multiple alternatives: " + rule); + } + this.alternativeCount = alternativeCount; + this.code = requireNonNull(code, "code is null"); + this.queryReachable = queryReachable; + this.rule = requireNonNull(rule, "rule is null"); + } + } + + public record PublishedGrammarFeatureManifest( + HogQlLanguageVersion languageVersion, + String grammarSha256, + List features, + List validationErrors) + { + @JsonCreator + public PublishedGrammarFeatureManifest( + @JsonProperty("languageVersion") HogQlLanguageVersion languageVersion, + @JsonProperty("grammarSha256") String grammarSha256, + @JsonProperty("features") List features, + @JsonProperty("validationErrors") List validationErrors) + { + this.languageVersion = requireNonNull(languageVersion, "languageVersion is null"); + this.grammarSha256 = requireNonNull(grammarSha256, "grammarSha256 is null"); + this.features = List.copyOf(requireNonNull(features, "features is null")); + this.validationErrors = List.copyOf(requireNonNull(validationErrors, "validationErrors is null")); + } + } + + public record PublishedFeature(String id, String kind, boolean queryReachable) + { + @JsonCreator + public PublishedFeature( + @JsonProperty("id") String id, + @JsonProperty("kind") String kind, + @JsonProperty("queryReachable") boolean queryReachable) + { + this.id = requireNonNull(id, "id is null"); + this.kind = requireNonNull(kind, "kind is null"); + this.queryReachable = queryReachable; + } + } +} diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlGrammarAlternativeIdentities.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlGrammarAlternativeIdentities.java new file mode 100644 index 000000000000..a3b5dd26117e --- /dev/null +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlGrammarAlternativeIdentities.java @@ -0,0 +1,165 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.airlift.json.JsonCodec; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HashSet; +import java.util.HexFormat; +import java.util.List; +import java.util.Set; + +import static io.airlift.json.JsonCodec.jsonCodec; +import static java.util.Objects.requireNonNull; + +public record HogQlGrammarAlternativeIdentities( + int schemaVersion, + HogQlLanguageVersion languageVersion, + String grammarSha256, + List rules) +{ + private static final String CURRENT_RESOURCE = "/io/trino/hogql/parser/language/1.0.0/grammar-alternative-identities.json"; + private static final JsonCodec CODEC = jsonCodec(HogQlGrammarAlternativeIdentities.class); + private static final LoadedIdentities CURRENT = loadCurrent(); + + @JsonCreator + public HogQlGrammarAlternativeIdentities( + @JsonProperty("schemaVersion") int schemaVersion, + @JsonProperty("languageVersion") HogQlLanguageVersion languageVersion, + @JsonProperty("grammarSha256") String grammarSha256, + @JsonProperty("rules") List rules) + { + if (schemaVersion != 1) { + throw new IllegalArgumentException("unsupported HogQL grammar alternative identity schema: " + schemaVersion); + } + this.schemaVersion = schemaVersion; + this.languageVersion = requireNonNull(languageVersion, "languageVersion is null"); + this.grammarSha256 = requireNonNull(grammarSha256, "grammarSha256 is null"); + this.rules = List.copyOf(requireNonNull(rules, "rules is null")); + + Set ruleNames = new HashSet<>(); + Set featureIds = new HashSet<>(); + for (AlternativeRule rule : this.rules) { + if (!ruleNames.add(rule.rule())) { + throw new IllegalArgumentException("duplicate HogQL grammar alternative identity rule: " + rule.rule()); + } + for (Alternative alternative : rule.alternatives()) { + if (!featureIds.add(alternative.id())) { + throw new IllegalArgumentException("duplicate HogQL grammar alternative identity: " + alternative.id()); + } + } + } + } + + public static HogQlGrammarAlternativeIdentities current() + { + return CURRENT.identities(); + } + + public static String currentSha256() + { + return CURRENT.sha256(); + } + + private static LoadedIdentities loadCurrent() + { + byte[] content = readResourceBytes(); + HogQlGrammarAlternativeIdentities identities = CODEC.fromJson(new String(content, StandardCharsets.UTF_8)); + HogQlLanguageContract languageContract = HogQlLanguageContract.current(); + if (!identities.languageVersion().equals(languageContract.languageVersion()) || !identities.grammarSha256().equals(languageContract.grammarSha256())) { + throw new IllegalStateException("HogQL grammar alternative identities do not match the language contract"); + } + return new LoadedIdentities(identities, sha256(content)); + } + + private static byte[] readResourceBytes() + { + try (InputStream input = HogQlGrammarAlternativeIdentities.class.getResourceAsStream(CURRENT_RESOURCE)) { + if (input == null) { + throw new IllegalStateException("HogQL grammar alternative identities are missing: " + CURRENT_RESOURCE); + } + return input.readAllBytes(); + } + catch (IOException e) { + throw new UncheckedIOException("failed to read HogQL grammar alternative identities", e); + } + } + + private static String sha256(byte[] content) + { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(content)); + } + catch (NoSuchAlgorithmException e) { + throw new AssertionError(e); + } + } + + private record LoadedIdentities(HogQlGrammarAlternativeIdentities identities, String sha256) {} + + public record AlternativeRule(String rule, boolean queryReachable, List alternatives) + { + @JsonCreator + public AlternativeRule( + @JsonProperty("rule") String rule, + @JsonProperty("queryReachable") boolean queryReachable, + @JsonProperty("alternatives") List alternatives) + { + this.rule = requireNonNull(rule, "rule is null"); + this.queryReachable = queryReachable; + this.alternatives = List.copyOf(requireNonNull(alternatives, "alternatives is null")); + if (rule.isBlank() || this.alternatives.size() < 2) { + throw new IllegalArgumentException("HogQL grammar alternative identity rule must have multiple alternatives: " + rule); + } + + Set fingerprints = new HashSet<>(); + for (Alternative alternative : this.alternatives) { + String prefix = "alternative:" + rule + ":"; + if (!alternative.id().startsWith(prefix)) { + throw new IllegalArgumentException("HogQL grammar alternative identity does not match its rule: " + alternative.id()); + } + String semanticId = alternative.id().substring(prefix.length()); + if (semanticId.isBlank() || semanticId.matches("(?:alt(?:ernative)?[-:]?)?\\d+")) { + throw new IllegalArgumentException("HogQL grammar alternative identity must not be positional: " + alternative.id()); + } + if (!fingerprints.add(alternative.structuralFingerprint())) { + throw new IllegalArgumentException("duplicate structural fingerprint for HogQL grammar rule: " + rule); + } + } + } + } + + public record Alternative(String id, String structuralFingerprint) + { + @JsonCreator + public Alternative( + @JsonProperty("id") String id, + @JsonProperty("structuralFingerprint") String structuralFingerprint) + { + this.id = requireNonNull(id, "id is null"); + this.structuralFingerprint = requireNonNull(structuralFingerprint, "structuralFingerprint is null"); + if (id.isBlank() || !structuralFingerprint.matches("[0-9a-f]{64}")) { + throw new IllegalArgumentException("invalid HogQL grammar alternative identity: " + id); + } + } + } +} diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlLanguageContract.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlLanguageContract.java new file mode 100644 index 000000000000..c653198c3b58 --- /dev/null +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlLanguageContract.java @@ -0,0 +1,146 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.airlift.json.JsonCodec; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; + +import static io.airlift.json.JsonCodec.jsonCodec; +import static java.util.Objects.requireNonNull; + +public record HogQlLanguageContract( + int schemaVersion, + HogQlLanguageVersion languageVersion, + String antlrVersion, + String canonicalParser, + List corpora, + Map entryPoints, + List files, + GrammarFeatureManifest grammarFeatureManifest, + String grammarSha256) +{ + private static final String CURRENT_CONTRACT_RESOURCE = "/io/trino/hogql/parser/language/1.0.0/language.json"; + private static final JsonCodec CONTRACT_CODEC = jsonCodec(HogQlLanguageContract.class); + private static final HogQlLanguageContract CURRENT = loadCurrent(); + + @JsonCreator + public HogQlLanguageContract( + @JsonProperty("schemaVersion") int schemaVersion, + @JsonProperty("languageVersion") HogQlLanguageVersion languageVersion, + @JsonProperty("antlrVersion") String antlrVersion, + @JsonProperty("canonicalParser") String canonicalParser, + @JsonProperty("corpora") List corpora, + @JsonProperty("entryPoints") Map entryPoints, + @JsonProperty("files") List files, + @JsonProperty("grammarFeatureManifest") GrammarFeatureManifest grammarFeatureManifest, + @JsonProperty("grammarSha256") String grammarSha256) + { + if (schemaVersion != 1) { + throw new IllegalArgumentException("unsupported HogQL language contract schema: " + schemaVersion); + } + this.schemaVersion = schemaVersion; + this.languageVersion = requireNonNull(languageVersion, "languageVersion is null"); + this.antlrVersion = requireNonNull(antlrVersion, "antlrVersion is null"); + this.canonicalParser = requireNonNull(canonicalParser, "canonicalParser is null"); + this.corpora = List.copyOf(requireNonNull(corpora, "corpora is null")); + this.entryPoints = Map.copyOf(requireNonNull(entryPoints, "entryPoints is null")); + this.files = List.copyOf(requireNonNull(files, "files is null")); + this.grammarFeatureManifest = requireNonNull(grammarFeatureManifest, "grammarFeatureManifest is null"); + this.grammarSha256 = requireNonNull(grammarSha256, "grammarSha256 is null"); + } + + public static HogQlLanguageContract current() + { + return CURRENT; + } + + private static HogQlLanguageContract loadCurrent() + { + try (InputStream input = HogQlLanguageContract.class.getResourceAsStream(CURRENT_CONTRACT_RESOURCE)) { + if (input == null) { + throw new IllegalStateException("HogQL language contract resource is missing: " + CURRENT_CONTRACT_RESOURCE); + } + return CONTRACT_CODEC.fromJson(new String(input.readAllBytes(), StandardCharsets.UTF_8)); + } + catch (IOException e) { + throw new UncheckedIOException("failed to read HogQL language contract", e); + } + } + + public record GrammarFile(String path, String sha256) + { + @JsonCreator + public GrammarFile( + @JsonProperty("path") String path, + @JsonProperty("sha256") String sha256) + { + this.path = requireNonNull(path, "path is null"); + this.sha256 = requireNonNull(sha256, "sha256 is null"); + } + } + + public record GrammarFeatureManifest(String path, int schemaVersion, String sha256) + { + @JsonCreator + public GrammarFeatureManifest( + @JsonProperty("path") String path, + @JsonProperty("schemaVersion") int schemaVersion, + @JsonProperty("sha256") String sha256) + { + this.path = requireNonNull(path, "path is null"); + if (schemaVersion != 1) { + throw new IllegalArgumentException("unsupported HogQL grammar feature schema: " + schemaVersion); + } + this.schemaVersion = schemaVersion; + this.sha256 = requireNonNull(sha256, "sha256 is null"); + } + } + + public record Corpus( + String manifestPath, + String manifestSha256, + String oraclePath, + String oracleSha256, + int schemaVersion, + String slice) + { + @JsonCreator + public Corpus( + @JsonProperty("manifestPath") String manifestPath, + @JsonProperty("manifestSha256") String manifestSha256, + @JsonProperty("oraclePath") String oraclePath, + @JsonProperty("oracleSha256") String oracleSha256, + @JsonProperty("schemaVersion") int schemaVersion, + @JsonProperty("slice") String slice) + { + this.manifestPath = requireNonNull(manifestPath, "manifestPath is null"); + this.manifestSha256 = requireNonNull(manifestSha256, "manifestSha256 is null"); + this.oraclePath = requireNonNull(oraclePath, "oraclePath is null"); + this.oracleSha256 = requireNonNull(oracleSha256, "oracleSha256 is null"); + if (schemaVersion != 1) { + throw new IllegalArgumentException("unsupported HogQL corpus schema: " + schemaVersion); + } + this.schemaVersion = schemaVersion; + this.slice = requireNonNull(slice, "slice is null"); + } + } +} diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlLanguageVersion.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlLanguageVersion.java new file mode 100644 index 000000000000..a6262fffe821 --- /dev/null +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlLanguageVersion.java @@ -0,0 +1,52 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +public record HogQlLanguageVersion(int major, int minor, int patch) +{ + private static final Pattern VERSION_PATTERN = Pattern.compile("(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)\\.(0|[1-9][0-9]*)"); + + public HogQlLanguageVersion + { + if (major < 0 || minor < 0 || patch < 0) { + throw new IllegalArgumentException("language version components must be non-negative"); + } + } + + @JsonCreator + public static HogQlLanguageVersion valueOf(String value) + { + Matcher matcher = VERSION_PATTERN.matcher(value); + if (!matcher.matches()) { + throw new IllegalArgumentException("invalid HogQL language version: " + value); + } + return new HogQlLanguageVersion( + Integer.parseInt(matcher.group(1)), + Integer.parseInt(matcher.group(2)), + Integer.parseInt(matcher.group(3))); + } + + @Override + @JsonValue + public String toString() + { + return "%s.%s.%s".formatted(major, minor, patch); + } +} diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java new file mode 100644 index 000000000000..71b2622d801a --- /dev/null +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParser.java @@ -0,0 +1,2482 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import io.trino.hogql.parser.canonical.HogQLLexer; +import io.trino.hogql.parser.canonical.HogQLParser; +import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.AliasedRelation; +import io.trino.hogql.parser.tree.HogQlQuery.ArrayExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BetweenExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator; +import io.trino.hogql.parser.tree.HogQlQuery.CaseExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CaseWhen; +import io.trino.hogql.parser.tree.HogQlQuery.CastExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CastTypeDialect; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsList; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsRegex; +import io.trino.hogql.parser.tree.HogQlQuery.CommonTableExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CommonTableReference; +import io.trino.hogql.parser.tree.HogQlQuery.Expression; +import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; +import io.trino.hogql.parser.tree.HogQlQuery.FrameBound; +import io.trino.hogql.parser.tree.HogQlQuery.FrameBoundType; +import io.trino.hogql.parser.tree.HogQlQuery.FrameType; +import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; +import io.trino.hogql.parser.tree.HogQlQuery.Identifier; +import io.trino.hogql.parser.tree.HogQlQuery.InCohortExpression; +import io.trino.hogql.parser.tree.HogQlQuery.InExpression; +import io.trino.hogql.parser.tree.HogQlQuery.InSubqueryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IntervalExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IntervalUnit; +import io.trino.hogql.parser.tree.HogQlQuery.IsNullExpression; +import io.trino.hogql.parser.tree.HogQlQuery.JoinCriteria; +import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; +import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; +import io.trino.hogql.parser.tree.HogQlQuery.JoinType; +import io.trino.hogql.parser.tree.HogQlQuery.JoinUsing; +import io.trino.hogql.parser.tree.HogQlQuery.LambdaExpression; +import io.trino.hogql.parser.tree.HogQlQuery.LimitBy; +import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; +import io.trino.hogql.parser.tree.HogQlQuery.NullPlacement; +import io.trino.hogql.parser.tree.HogQlQuery.NullTreatment; +import io.trino.hogql.parser.tree.HogQlQuery.PivotAggregation; +import io.trino.hogql.parser.tree.HogQlQuery.PivotRelation; +import io.trino.hogql.parser.tree.HogQlQuery.PivotValueGroup; +import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; +import io.trino.hogql.parser.tree.HogQlQuery.Projection; +import io.trino.hogql.parser.tree.HogQlQuery.Relation; +import io.trino.hogql.parser.tree.HogQlQuery.ScalarSubqueryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.SetOperation; +import io.trino.hogql.parser.tree.HogQlQuery.SetOperationType; +import io.trino.hogql.parser.tree.HogQlQuery.SortDirection; +import io.trino.hogql.parser.tree.HogQlQuery.SortItem; +import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; +import io.trino.hogql.parser.tree.HogQlQuery.Star; +import io.trino.hogql.parser.tree.HogQlQuery.StarReplacement; +import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; +import io.trino.hogql.parser.tree.HogQlQuery.SubscriptExpression; +import io.trino.hogql.parser.tree.HogQlQuery.TablePlaceholder; +import io.trino.hogql.parser.tree.HogQlQuery.TableReference; +import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; +import io.trino.hogql.parser.tree.HogQlQuery.UnaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.UnnestRelation; +import io.trino.hogql.parser.tree.HogQlQuery.ValuesRelation; +import io.trino.hogql.parser.tree.HogQlQuery.Window; +import io.trino.hogql.parser.tree.HogQlQuery.WindowDefinition; +import io.trino.hogql.parser.tree.HogQlQuery.WindowFrame; +import io.trino.hogql.parser.tree.HogQlQuery.WindowReference; +import io.trino.hogql.parser.tree.HogQlQuery.WindowSpecification; +import io.trino.hogql.parser.tree.HogQlSyntaxTree; +import org.antlr.v4.runtime.ANTLRErrorListener; +import org.antlr.v4.runtime.BailErrorStrategy; +import org.antlr.v4.runtime.BaseErrorListener; +import org.antlr.v4.runtime.CharStream; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.DefaultErrorStrategy; +import org.antlr.v4.runtime.ParserRuleContext; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.Recognizer; +import org.antlr.v4.runtime.Token; +import org.antlr.v4.runtime.TokenFactory; +import org.antlr.v4.runtime.TokenSource; +import org.antlr.v4.runtime.atn.PredictionMode; +import org.antlr.v4.runtime.misc.ParseCancellationException; +import org.antlr.v4.runtime.tree.ErrorNode; +import org.antlr.v4.runtime.tree.ParseTree; +import org.antlr.v4.runtime.tree.ParseTreeListener; +import org.antlr.v4.runtime.tree.TerminalNode; + +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.ADD; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.AND; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.CONCAT; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.DIVIDE; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.EQUAL; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.GREATER_THAN; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.GREATER_THAN_OR_EQUAL; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.ILIKE; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.LESS_THAN; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.LESS_THAN_OR_EQUAL; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.LIKE; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.MODULO; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.MULTIPLY; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.NOT_ILIKE; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.NOT_LIKE; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.NOT_EQUAL; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.OR; +import static io.trino.hogql.parser.tree.HogQlQuery.BinaryOperator.SUBTRACT; +import static io.trino.hogql.parser.tree.HogQlQuery.LiteralKind.INTEGER; +import static io.trino.hogql.parser.tree.HogQlQuery.LiteralKind.NULL; +import static io.trino.hogql.parser.tree.HogQlQuery.LiteralKind.STRING; +import static io.trino.hogql.parser.tree.HogQlQuery.UnaryOperator.NEGATE; +import static io.trino.hogql.parser.tree.HogQlQuery.UnaryOperator.NOT; +import static io.trino.hogql.parser.tree.HogQlQuery.UnaryOperator.POSITIVE; +import static java.lang.Character.digit; +import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.joining; + +public final class HogQlParser +{ + private static final HogQlLanguageVersion CURRENT_LANGUAGE_VERSION = HogQlLanguageContract.current().languageVersion(); + + private final HogQlParserLimits limits; + + private static final ANTLRErrorListener ERROR_LISTENER = new BaseErrorListener() + { + @Override + public void syntaxError(Recognizer recognizer, Object offendingSymbol, int line, int charPositionInLine, String message, RecognitionException cause) + { + throw new HogQlParsingException(message, cause, line, charPositionInLine + 1); + } + }; + + public HogQlParser() + { + this(HogQlParserLimits.defaults()); + } + + HogQlParser(HogQlParserLimits limits) + { + this.limits = requireNonNull(limits, "limits is null"); + } + + public HogQlQuery parseStatement(String hogql) + { + return parseStatement(hogql, CURRENT_LANGUAGE_VERSION); + } + + public HogQlQuery parseStatement(String hogql, HogQlLanguageVersion languageVersion) + { + try { + ParsedSyntax parsed = parse(hogql, languageVersion, EntryPoint.QUERY); + return new AstBuilder(parsed.source()).build((HogQLParser.SelectContext) parsed.tree()); + } + catch (StackOverflowError _) { + throw new HogQlParsingException("statement is too large", null, 1, 1); + } + } + + public HogQlSyntaxTree parseSyntax(String hogql) + { + return parseSyntax(hogql, CURRENT_LANGUAGE_VERSION); + } + + public HogQlSyntaxTree parseSyntax(String hogql, HogQlLanguageVersion languageVersion) + { + try { + ParsedSyntax parsed = parse(hogql, languageVersion, EntryPoint.QUERY); + return new SyntaxTreeBuilder(parsed.source()).build((HogQLParser.SelectContext) parsed.tree()); + } + catch (StackOverflowError _) { + throw new HogQlParsingException("statement is too large", null, 1, 1); + } + } + + public HogQlSyntaxTree parseExpressionSyntax(String hogql) + { + return parseExpressionSyntax(hogql, CURRENT_LANGUAGE_VERSION); + } + + public HogQlSyntaxTree parseExpressionSyntax(String hogql, HogQlLanguageVersion languageVersion) + { + try { + ParsedSyntax parsed = parse(hogql, languageVersion, EntryPoint.EXPRESSION); + return new SyntaxTreeBuilder(parsed.source()).build(parsed.tree()); + } + catch (StackOverflowError _) { + throw new HogQlParsingException("expression is too large", null, 1, 1); + } + } + + private ParsedSyntax parse(String hogql, HogQlLanguageVersion languageVersion, EntryPoint entryPoint) + { + requireNonNull(hogql, "hogql is null"); + requireNonNull(languageVersion, "languageVersion is null"); + if (!languageVersion.equals(CURRENT_LANGUAGE_VERSION)) { + throw new IllegalArgumentException("unsupported HogQL language version: " + languageVersion); + } + + HogQLLexer lexer = new HogQLLexer(CharStreams.fromString(hogql)); + CommonTokenStream tokenStream = new CommonTokenStream(new BoundedTokenSource(lexer, limits.maxTokens())); + HogQLParser parser = new HogQLParser(tokenStream); + + lexer.removeErrorListeners(); + lexer.addErrorListener(ERROR_LISTENER); + parser.removeErrorListeners(); + + ParserRuleContext tree; + try { + parser.getInterpreter().setPredictionMode(PredictionMode.SLL); + parser.setErrorHandler(new BailErrorStrategy()); + parser.addParseListener(new ParseBudgetListener(limits)); + tree = entryPoint.parse(parser); + } + catch (ParseCancellationException _) { + parser.reset(); + parser.getInterpreter().setPredictionMode(PredictionMode.LL); + parser.setErrorHandler(new DefaultErrorStrategy()); + parser.addErrorListener(ERROR_LISTENER); + parser.removeParseListeners(); + parser.addParseListener(new ParseBudgetListener(limits)); + tree = entryPoint.parse(parser); + } + if (parser.getCurrentToken().getType() != Token.EOF) { + Token trailing = parser.getCurrentToken(); + throw new HogQlParsingException("unexpected trailing input", null, trailing.getLine(), trailing.getCharPositionInLine() + 1); + } + return new ParsedSyntax(hogql, tree); + } + + private record ParsedSyntax(String source, ParserRuleContext tree) {} + + private enum EntryPoint + { + QUERY { + @Override + public ParserRuleContext parse(HogQLParser parser) + { + return parser.select(); + } + }, + EXPRESSION { + @Override + public ParserRuleContext parse(HogQLParser parser) + { + return parser.expression(); + } + }; + + public abstract ParserRuleContext parse(HogQLParser parser); + } + + private static final class BoundedTokenSource + implements TokenSource + { + private final TokenSource delegate; + private final int maxTokens; + private int tokenCount; + + private BoundedTokenSource(TokenSource delegate, int maxTokens) + { + this.delegate = requireNonNull(delegate, "delegate is null"); + this.maxTokens = maxTokens; + } + + @Override + public Token nextToken() + { + Token token = delegate.nextToken(); + if (token.getType() != Token.EOF && ++tokenCount > maxTokens) { + throw new HogQlParsingException("token limit exceeded", null, token.getLine(), token.getCharPositionInLine() + 1); + } + return token; + } + + @Override + public int getLine() + { + return delegate.getLine(); + } + + @Override + public int getCharPositionInLine() + { + return delegate.getCharPositionInLine(); + } + + @Override + public CharStream getInputStream() + { + return delegate.getInputStream(); + } + + @Override + public String getSourceName() + { + return delegate.getSourceName(); + } + + @Override + public void setTokenFactory(TokenFactory factory) + { + delegate.setTokenFactory(factory); + } + + @Override + public TokenFactory getTokenFactory() + { + return delegate.getTokenFactory(); + } + } + + private static final class ParseBudgetListener + implements ParseTreeListener + { + private final HogQlParserLimits limits; + private int depth; + private int nodes; + + private ParseBudgetListener(HogQlParserLimits limits) + { + this.limits = requireNonNull(limits, "limits is null"); + } + + @Override + public void visitTerminal(TerminalNode node) + { + addNode(node.getSymbol()); + } + + @Override + public void visitErrorNode(ErrorNode node) + { + addNode(node.getSymbol()); + } + + @Override + public void enterEveryRule(ParserRuleContext context) + { + depth++; + if (depth > limits.maxParseDepth()) { + throw limitExceeded("parse depth limit exceeded", context.getStart()); + } + addNode(context.getStart()); + } + + @Override + public void exitEveryRule(ParserRuleContext context) + { + depth--; + } + + private void addNode(Token token) + { + if (++nodes > limits.maxParseTreeNodes()) { + throw limitExceeded("parse tree node limit exceeded", token); + } + } + + private static HogQlParsingException limitExceeded(String message, Token token) + { + return new HogQlParsingException(message, null, token.getLine(), token.getCharPositionInLine() + 1); + } + } + + private static final class SyntaxTreeBuilder + { + private final SourcePositions sourcePositions; + + private SyntaxTreeBuilder(String source) + { + sourcePositions = new SourcePositions(source); + } + + public HogQlSyntaxTree build(HogQLParser.SelectContext context) + { + HogQlSyntaxTree.LanguageClass languageClass; + if (context.hogqlxTagElement() != null) { + languageClass = HogQlSyntaxTree.LanguageClass.HOGQLX; + } + else if (containsRule(context, HogQLParser.RULE_block)) { + languageClass = HogQlSyntaxTree.LanguageClass.PROCEDURAL; + } + else { + languageClass = HogQlSyntaxTree.LanguageClass.READ_ONLY_QUERY; + } + return new HogQlSyntaxTree(languageClass, buildNode(context)); + } + + public HogQlSyntaxTree build(ParserRuleContext context) + { + HogQlSyntaxTree.LanguageClass languageClass = containsRule(context, HogQLParser.RULE_block) ? + HogQlSyntaxTree.LanguageClass.PROCEDURAL : + HogQlSyntaxTree.LanguageClass.READ_ONLY_QUERY; + return new HogQlSyntaxTree(languageClass, buildNode(context)); + } + + private static boolean containsRule(ParserRuleContext context, int ruleIndex) + { + if (context.getRuleIndex() == ruleIndex) { + return true; + } + for (int index = 0; index < context.getChildCount(); index++) { + if (context.getChild(index) instanceof ParserRuleContext child && containsRule(child, ruleIndex)) { + return true; + } + } + return false; + } + + private HogQlSyntaxTree.Node buildNode(ParserRuleContext context) + { + String rule = HogQLParser.ruleNames[context.getRuleIndex()]; + String contextName = context.getClass().getSimpleName(); + String baseContextName = Character.toUpperCase(rule.charAt(0)) + rule.substring(1) + "Context"; + Optional alternative = contextName.equals(baseContextName) ? + Optional.empty() : + Optional.of(contextName.substring(0, contextName.length() - "Context".length())); + + List children = new ArrayList<>(); + for (int index = 0; index < context.getChildCount(); index++) { + ParseTree child = context.getChild(index); + if (child instanceof ParserRuleContext ruleChild) { + children.add(buildNode(ruleChild)); + } + else if (child instanceof TerminalNode terminal && terminal.getSymbol().getType() != Token.EOF) { + children.add(buildToken(terminal.getSymbol())); + } + } + return new HogQlSyntaxTree.Node(rule, alternative, children, sourceSpan(context)); + } + + private HogQlSyntaxTree.Token buildToken(Token token) + { + String type = HogQLLexer.VOCABULARY.getSymbolicName(token.getType()); + if (type == null) { + type = HogQLLexer.VOCABULARY.getDisplayName(token.getType()); + } + return new HogQlSyntaxTree.Token(type, token.getText(), sourceSpan(token, token)); + } + + private SourceSpan sourceSpan(ParserRuleContext context) + { + return sourceSpan(context.getStart(), context.getStop()); + } + + private SourceSpan sourceSpan(Token start, Token stop) + { + int startOffset = Math.max(0, start.getStartIndex()); + int endOffset = Math.max(startOffset, stop.getStopIndex() + 1); + return new SourceSpan( + startOffset, + endOffset, + sourcePositions.line(startOffset), + sourcePositions.column(startOffset), + sourcePositions.line(endOffset), + sourcePositions.column(endOffset)); + } + } + + private static final class AstBuilder + { + private final String source; + private final SourcePositions sourcePositions; + private final Deque> commonTableScopes = new ArrayDeque<>(); + private final Deque> prohibitedCommonTableScopes = new ArrayDeque<>(); + private final Deque> withExpressionScopes = new ArrayDeque<>(); + + private AstBuilder(String source) + { + this.source = requireNonNull(source, "source is null"); + sourcePositions = new SourcePositions(source); + } + + public HogQlQuery build(HogQLParser.SelectContext context) + { + HogQlQuery query; + if (context.selectStmt() != null) { + query = buildQuery(new SelectedQuery(context.selectStmt(), null, null)); + } + else if (context.selectSetStmt() != null) { + query = buildSetQuery(context.selectSetStmt()); + } + else { + throw unsupported(context, "query"); + } + validateUncorrelatedSubqueries(query); + return query; + } + + private HogQlQuery buildSetQuery(HogQLParser.SelectSetStmtContext context) + { + commonTableScopes.push(new LinkedHashSet<>()); + prohibitedCommonTableScopes.push(new LinkedHashSet<>()); + try { + QueryOperand first = buildSetOperand(context.selectStmtWithParens()); + if (context.subsequentSelectSetClause().isEmpty()) { + return attachSetLevelClauses(first.query(), context); + } + + List commonTables = List.of(); + if (!first.parenthesized() && !first.query().with().isEmpty()) { + commonTables = first.query().with(); + commonTables.stream() + .map(CommonTableExpression::name) + .map(HogQlParser.AstBuilder::canonicalName) + .forEach(commonTableScopes.getFirst()::add); + first = first.withQuery(copyQuery(List.of(), first.query().body(), first.query().orderBy(), first.query().limit(), first.query().offset(), first.query().span())); + } + + List operands = new ArrayList<>(); + List operators = new ArrayList<>(); + operands.add(first); + for (HogQLParser.SubsequentSelectSetClauseContext clause : context.subsequentSelectSetClause()) { + SetOperator operator = buildSetOperator(clause); + QueryOperand operand = buildSetOperand(clause.selectStmtWithParens()); + if (!operand.parenthesized() && !operand.query().with().isEmpty()) { + throw unsupported(clause.selectStmtWithParens(), "unparenthesized WITH set operand"); + } + operators.add(operator); + operands.add(operand); + } + + List orderBy = buildOrderBy(context.orderByClause()); + Pagination pagination = context.limitAndOffsetClauseOptional() == null + ? new Pagination(Optional.empty(), Optional.empty()) + : buildPagination(context.limitAndOffsetClauseOptional()); + QueryOperand last = operands.getLast(); + if (!last.parenthesized() && hasQueryClauses(last.query())) { + if (!orderBy.isEmpty() && !last.query().orderBy().isEmpty()) { + throw unsupported(context.orderByClause(), "multiple ORDER BY clauses"); + } + orderBy = orderBy.isEmpty() ? last.query().orderBy() : orderBy; + pagination = mergePagination( + new Pagination(last.query().limit(), last.query().offset()), + pagination, + context); + operands.set( + operands.size() - 1, + last.withQuery(copyQuery( + last.query().with(), + last.query().body(), + List.of(), + Optional.empty(), + Optional.empty(), + last.query().span()))); + } + QueryOperand operation = applySetOperationPrecedence(operands, operators); + return copyQuery( + commonTables, + operation.query().body(), + orderBy, + pagination.limit(), + pagination.offset(), + sourceSpan(context)); + } + finally { + prohibitedCommonTableScopes.pop(); + commonTableScopes.pop(); + } + } + + private static boolean hasQueryClauses(HogQlQuery query) + { + return !query.orderBy().isEmpty() || query.limit().isPresent() || query.offset().isPresent(); + } + + private QueryOperand buildSetOperand(HogQLParser.SelectStmtWithParensContext context) + { + if (context.selectStmt() != null) { + return new QueryOperand(buildQuery(new SelectedQuery(context.selectStmt(), null, null)), false, sourceSpan(context)); + } + if (context.withClause() != null) { + throw unsupported(context, "non-standard WITH query wrapper"); + } + if (context.selectSetStmt() != null) { + return new QueryOperand(buildSetQuery(context.selectSetStmt()), true, sourceSpan(context)); + } + throw unsupported(context, "placeholder query operand"); + } + + private SetOperator buildSetOperator(HogQLParser.SubsequentSelectSetClauseContext context) + { + if (context.BY() != null) { + throw unsupported(context, "set operation BY NAME"); + } + TerminalNode operator; + SetOperationType type; + if (context.UNION() != null) { + operator = context.UNION(); + type = SetOperationType.UNION; + } + else if (context.INTERSECT() != null) { + operator = context.INTERSECT(); + type = SetOperationType.INTERSECT; + } + else { + operator = context.EXCEPT(); + type = SetOperationType.EXCEPT; + } + Token stop = context.ALL() != null + ? context.ALL().getSymbol() + : context.DISTINCT() != null ? context.DISTINCT().getSymbol() : operator.getSymbol(); + return new SetOperator(type, context.ALL() == null, sourceSpan(operator.getSymbol(), stop)); + } + + private QueryOperand applySetOperationPrecedence(List operands, List operators) + { + List unionAndExceptOperands = new ArrayList<>(); + List unionAndExceptOperators = new ArrayList<>(); + QueryOperand current = operands.getFirst(); + for (int index = 0; index < operators.size(); index++) { + SetOperator operator = operators.get(index); + QueryOperand right = operands.get(index + 1); + if (operator.type() == SetOperationType.INTERSECT) { + current = combineSetOperation(current, operator, right); + } + else { + unionAndExceptOperands.add(current); + unionAndExceptOperators.add(operator); + current = right; + } + } + unionAndExceptOperands.add(current); + + current = unionAndExceptOperands.getFirst(); + for (int index = 0; index < unionAndExceptOperators.size(); ) { + SetOperator operator = unionAndExceptOperators.get(index); + if (operator.type() != SetOperationType.UNION) { + current = combineSetOperation(current, operator, unionAndExceptOperands.get(index + 1)); + index++; + continue; + } + + int runEnd = index + 1; + while (runEnd < unionAndExceptOperators.size()) { + SetOperator next = unionAndExceptOperators.get(runEnd); + if (next.type() != SetOperationType.UNION || next.distinct() != operator.distinct()) { + break; + } + runEnd++; + } + List runOperands = new ArrayList<>(runEnd - index + 1); + runOperands.add(current); + runOperands.addAll(unionAndExceptOperands.subList(index + 1, runEnd + 1)); + current = combineAssociativeSetOperations( + runOperands, + unionAndExceptOperators.subList(index, runEnd), + 0, + runOperands.size()); + index = runEnd; + } + return current; + } + + private QueryOperand combineAssociativeSetOperations(List operands, List operators, int start, int end) + { + if (end - start == 1) { + return operands.get(start); + } + int split = (start + end) / 2; + QueryOperand left = combineAssociativeSetOperations(operands, operators, start, split); + QueryOperand right = combineAssociativeSetOperations(operands, operators, split, end); + return combineSetOperation(left, operators.get(split - 1), right); + } + + private QueryOperand combineSetOperation(QueryOperand left, SetOperator operator, QueryOperand right) + { + SourceSpan span = enclosingSpan(left.span(), right.span()); + SetOperation body = new SetOperation( + operator.type(), + operator.distinct(), + left.query(), + right.query(), + left.parenthesized(), + right.parenthesized(), + operator.span(), + span); + HogQlQuery query = new HogQlQuery( + List.of(), + body, + List.of(), + Optional.empty(), + Optional.empty(), + span); + return new QueryOperand(query, false, span); + } + + private HogQlQuery attachSetLevelClauses(HogQlQuery query, HogQLParser.SelectSetStmtContext context) + { + List outerOrderBy = buildOrderBy(context.orderByClause()); + if (!outerOrderBy.isEmpty() && !query.orderBy().isEmpty()) { + throw unsupported(context.orderByClause(), "multiple ORDER BY clauses"); + } + Pagination outerPagination = context.limitAndOffsetClauseOptional() == null + ? new Pagination(Optional.empty(), Optional.empty()) + : buildPagination(context.limitAndOffsetClauseOptional()); + Pagination pagination = mergePagination( + new Pagination(query.limit(), query.offset()), + outerPagination, + context); + return copyQuery( + query.with(), + query.body(), + outerOrderBy.isEmpty() ? query.orderBy() : outerOrderBy, + pagination.limit(), + pagination.offset(), + sourceSpan(context)); + } + + private static HogQlQuery copyQuery( + List commonTables, + HogQlQuery.QueryBody body, + List orderBy, + Optional limit, + Optional offset, + SourceSpan span) + { + return new HogQlQuery(commonTables, body, orderBy, limit, offset, span); + } + + private static SourceSpan enclosingSpan(SourceSpan left, SourceSpan right) + { + return new SourceSpan( + left.startOffset(), + right.endOffset(), + left.startLine(), + left.startColumn(), + right.endLine(), + right.endColumn()); + } + + private record QueryOperand(HogQlQuery query, boolean parenthesized, SourceSpan span) + { + private QueryOperand withQuery(HogQlQuery query) + { + return new QueryOperand(query, parenthesized, span); + } + } + + private record SetOperator(SetOperationType type, boolean distinct, SourceSpan span) {} + + private HogQlQuery buildQuery(SelectedQuery selectedQuery) + { + HogQLParser.SelectStmtContext select = selectedQuery.statement(); + rejectUnsupportedClauses(select); + commonTableScopes.push(new LinkedHashSet<>()); + prohibitedCommonTableScopes.push(new LinkedHashSet<>()); + withExpressionScopes.push(new LinkedHashMap<>()); + try { + List with = buildCommonTables(select.withClause()); + + List projections = selectColumns(select.selectColumnExprListBeforeFrom()).stream() + .map(this::buildProjection) + .toList(); + Optional from = Optional.ofNullable(select.fromClause()) + .map(HogQLParser.FromClauseContext::joinExpr) + .map(this::buildRelation); + if (select.arrayJoinClause() != null) { + from = Optional.of(buildArrayJoin(from, select.arrayJoinClause())); + } + Optional where = Optional.ofNullable(select.whereClause()) + .map(HogQLParser.WhereClauseContext::columnExpr) + .map(this::buildExpression); + List groupBy = buildGroupBy(select); + Optional having = Optional.ofNullable(select.havingClause()) + .map(HogQLParser.HavingClauseContext::columnExpr) + .map(this::buildExpression); + List windows = buildWindowDefinitions(select.windowClause()); + List orderBy = buildOrderBy(selectedQuery.orderBy() != null ? selectedQuery.orderBy() : select.orderByClause()); + Optional limitBy = Optional.ofNullable(select.limitByClause()).map(this::buildLimitBy); + Pagination pagination = mergePagination( + buildPagination(select.limitAndOffsetClause(), select.offsetOnlyClause()), + selectedQuery.pagination() == null ? new Pagination(Optional.empty(), Optional.empty()) : buildPagination(selectedQuery.pagination()), + select); + return new HogQlQuery( + with, + new HogQlQuery.SelectQueryBody( + select.DISTINCT() != null, + projections, + from, + where, + groupBy, + having, + windows, + limitBy, + sourceSpan(select)), + orderBy, + pagination.limit(), + pagination.offset(), + sourceSpan(select)); + } + finally { + withExpressionScopes.pop(); + prohibitedCommonTableScopes.pop(); + commonTableScopes.pop(); + } + } + + private List buildCommonTables(HogQLParser.WithClauseContext context) + { + if (context == null) { + return List.of(); + } + if (context.RECURSIVE() != null) { + throw unsupported(context, "recursive CTE"); + } + List commonTables = new ArrayList<>(); + Set localNames = commonTableScopes.getFirst(); + for (HogQLParser.WithExprContext expression : context.withExprList().withExpr()) { + if (expression instanceof HogQLParser.WithExprColumnContext column) { + Identifier name = buildIdentifier(column.identifier()); + String canonicalName = canonicalName(name); + if (!localNames.add(canonicalName)) { + throw unsupported(column, "duplicate WITH name"); + } + withExpressionScopes.getFirst().put(canonicalName, buildExpression(column.columnExpr())); + continue; + } + HogQLParser.WithExprSubqueryContext subquery = (HogQLParser.WithExprSubqueryContext) expression; + if (subquery.USING() != null) { + throw unsupported(subquery, "CTE USING KEY"); + } + if (subquery.MATERIALIZED() != null) { + throw unsupported(subquery, "materialized CTE"); + } + Identifier name = buildIdentifier(subquery.identifier()); + String canonicalName = canonicalName(name); + if (localNames.contains(canonicalName)) { + throw unsupported(subquery, "duplicate CTE name"); + } + Set prohibitedNames = prohibitedCommonTableScopes.getFirst(); + prohibitedNames.add(canonicalName); + HogQlQuery query; + try { + query = buildSetQuery(subquery.selectSetStmt()); + } + finally { + prohibitedNames.remove(canonicalName); + } + List columnAliases = subquery.withExprColumnNameList().isEmpty() + ? List.of() + : subquery.withExprColumnNameList().getFirst().identifier().stream() + .map(this::buildIdentifier) + .toList(); + commonTables.add(new CommonTableExpression(name, columnAliases, query, sourceSpan(subquery))); + localNames.add(canonicalName); + } + return List.copyOf(commonTables); + } + + private List selectColumns(HogQLParser.SelectColumnExprListBeforeFromContext context) + { + if (context instanceof HogQLParser.SelectColumnExprListBeforeFromTrailingCommaContext trailingComma) { + return trailingComma.selectColumnExpr(); + } + if (context instanceof HogQLParser.SelectColumnExprListBeforeFromPlainContext plain) { + return plain.selectColumnExprList().selectColumnExpr(); + } + throw unsupported(context, "select list"); + } + + private record SelectedQuery( + HogQLParser.SelectStmtContext statement, + HogQLParser.OrderByClauseContext orderBy, + HogQLParser.LimitAndOffsetClauseOptionalContext pagination) {} + + private void rejectUnsupportedClauses(HogQLParser.SelectStmtContext context) + { + List clauses = new ArrayList<>(); + clauses.add(context.topClause()); + clauses.add(context.prewhereClause()); + clauses.addAll(context.sampleClause()); + clauses.add(context.qualifyClause()); + clauses.add(context.settingsClause()); + Optional firstClause = clauses.stream() + .filter(requireNonNullClause -> requireNonNullClause != null) + .min((left, right) -> Integer.compare(left.getStart().getStartIndex(), right.getStart().getStartIndex())); + if (firstClause.isPresent()) { + throw unsupported(firstClause.orElseThrow(), "query clause"); + } + } + + private LimitBy buildLimitBy(HogQLParser.LimitByClauseContext context) + { + List limitExpressions = context.limitExpr().columnExpr(); + Expression limit = buildPaginationExpression( + limitExpressions.size() == 2 && context.limitExpr().COMMA() != null + ? limitExpressions.getLast() + : limitExpressions.getFirst()); + Optional offset = limitExpressions.size() == 2 + ? Optional.of(buildPaginationExpression( + context.limitExpr().COMMA() != null + ? limitExpressions.getFirst() + : limitExpressions.getLast())) + : Optional.empty(); + return new LimitBy(limit, offset, buildExpressions(context.columnExprList()), sourceSpan(context)); + } + + private Relation buildArrayJoin(Optional from, HogQLParser.ArrayJoinClauseContext context) + { + List expressions = new ArrayList<>(); + List columnAliases = new ArrayList<>(); + for (HogQLParser.ColumnExprContext expression : context.columnExprList().columnExpr()) { + if (!(expression instanceof HogQLParser.ColumnExprAliasContext alias) || alias.identifier() == null) { + throw unsupported(expression, "ARRAY JOIN expression without an identifier alias"); + } + expressions.add(buildExpression(alias.columnExpr())); + columnAliases.add(buildIdentifier(alias.identifier())); + } + Identifier relationAlias = buildIdentifier("__hogql_array_join", context); + UnnestRelation unnest = new UnnestRelation(expressions, relationAlias, columnAliases, sourceSpan(context)); + if (from.isEmpty()) { + if (context.LEFT() != null) { + throw unsupported(context, "LEFT ARRAY JOIN without FROM"); + } + return unnest; + } + JoinType type = context.LEFT() == null ? JoinType.CROSS : JoinType.LEFT; + Optional criteria = context.LEFT() == null + ? Optional.empty() + : Optional.of(new JoinOn(new Literal(HogQlQuery.LiteralKind.BOOLEAN, "true", sourceSpan(context)), sourceSpan(context))); + return new JoinRelation(type, from.orElseThrow(), unnest, criteria, sourceSpan(context)); + } + + private List buildOrderBy(HogQLParser.OrderByClauseContext context) + { + if (context == null) { + return List.of(); + } + if (context.interpolateClause() != null) { + throw unsupported(context.interpolateClause(), "ORDER BY interpolation"); + } + return buildSortItems(context.orderExprList()); + } + + private List buildSortItems(HogQLParser.OrderExprListContext context) + { + return context.orderExpr().stream() + .map(this::buildSortItem) + .toList(); + } + + private List buildGroupBy(HogQLParser.SelectStmtContext select) + { + HogQLParser.GroupByClauseContext groupBy = select.groupByClause(); + if (groupBy == null) { + return List.of(); + } + if (groupBy.columnExprList() == null || + groupBy.ALL() != null || + groupBy.CUBE() != null || + groupBy.ROLLUP() != null || + groupBy.GROUPING() != null || + select.CUBE() != null || + select.ROLLUP() != null || + select.TOTALS() != null) { + throw unsupported(groupBy, "advanced grouping"); + } + return buildExpressions(groupBy.columnExprList()); + } + + private SortItem buildSortItem(HogQLParser.OrderExprContext context) + { + if (context.COLLATE() != null || context.withFillClause() != null) { + throw unsupported(context, "ORDER BY collation or fill"); + } + SortDirection direction = context.DESC() != null || context.DESCENDING() != null + ? SortDirection.DESCENDING + : SortDirection.ASCENDING; + NullPlacement nullPlacement = context.FIRST() != null + ? NullPlacement.FIRST + : context.LAST() != null ? NullPlacement.LAST : NullPlacement.UNDEFINED; + return new SortItem(buildExpression(context.columnExpr()), direction, nullPlacement, sourceSpan(context)); + } + + private Pagination buildPagination( + HogQLParser.LimitAndOffsetClauseContext limitContext, + HogQLParser.OffsetOnlyClauseContext offsetContext) + { + if (limitContext == null) { + return new Pagination(Optional.empty(), Optional.ofNullable(offsetContext) + .map(HogQLParser.OffsetOnlyClauseContext::columnExpr) + .map(this::buildPaginationExpression)); + } + if (limitContext.PERCENT() != null || limitContext.WITH() != null) { + throw unsupported(limitContext, "percent limit or ties"); + } + List expressions = limitContext.columnExpr(); + if (limitContext.COMMA() != null) { + return new Pagination( + Optional.of(buildPaginationExpression(expressions.get(1))), + Optional.of(buildPaginationExpression(expressions.getFirst()))); + } + return new Pagination( + Optional.of(buildPaginationExpression(expressions.getFirst())), + expressions.size() == 2 ? Optional.of(buildPaginationExpression(expressions.get(1))) : Optional.empty()); + } + + private Pagination buildPagination(HogQLParser.LimitAndOffsetClauseOptionalContext context) + { + if (context.PERCENT() != null || context.WITH() != null) { + throw unsupported(context, "percent limit or ties"); + } + List expressions = context.columnExpr(); + if (context.LIMIT() == null) { + return new Pagination(Optional.empty(), Optional.of(buildPaginationExpression(expressions.getFirst()))); + } + if (context.COMMA() != null) { + return new Pagination( + Optional.of(buildPaginationExpression(expressions.get(1))), + Optional.of(buildPaginationExpression(expressions.getFirst()))); + } + return new Pagination( + Optional.of(buildPaginationExpression(expressions.getFirst())), + expressions.size() == 2 ? Optional.of(buildPaginationExpression(expressions.get(1))) : Optional.empty()); + } + + private Pagination mergePagination(Pagination inner, Pagination outer, ParserRuleContext context) + { + if (inner.limit().isPresent() && outer.limit().isPresent() || inner.offset().isPresent() && outer.offset().isPresent()) { + throw unsupported(context, "duplicate pagination"); + } + return new Pagination( + outer.limit().or(() -> inner.limit()), + outer.offset().or(() -> inner.offset())); + } + + private Expression buildPaginationExpression(HogQLParser.ColumnExprContext context) + { + Expression expression = buildExpression(context); + if (expression instanceof Literal literal && literal.kind() == INTEGER || expression instanceof Placeholder) { + return expression; + } + throw unsupported(context, "non-constant pagination"); + } + + private record Pagination(Optional limit, Optional offset) {} + + private Projection buildProjection(HogQLParser.SelectColumnExprContext context) + { + HogQLParser.ColumnExprContext expression; + Optional alias = Optional.empty(); + if (context instanceof HogQLParser.ColumnExprAliasBeforeContext aliased) { + expression = aliased.columnExpr(); + alias = Optional.of(buildIdentifier(aliased.identifier())); + } + else if (context instanceof HogQLParser.ColumnExprAliasImplicitContext aliased) { + expression = aliased.columnExpr(); + alias = Optional.of(buildIdentifier(aliased.implicitAlias().getText(), aliased.implicitAlias())); + } + else if (context instanceof HogQLParser.ColumnExprSelectValueContext selected) { + expression = selected.columnExpr(); + if (expression instanceof HogQLParser.ColumnExprAliasContext aliased) { + expression = aliased.columnExpr(); + if (aliased.identifier() != null) { + alias = Optional.of(buildIdentifier(aliased.identifier())); + } + else { + alias = Optional.of(new Identifier(decodeQuoted(aliased.STRING_LITERAL().getText()), true, sourceSpan(aliased))); + } + } + } + else { + throw unsupported(context, "projection"); + } + + if (expression instanceof HogQLParser.ColumnExprValuePassthroughContext passthrough) { + Projection columns = buildColumnsProjection(passthrough.columnExprValue(), alias); + if (columns != null) { + return columns; + } + } + return new ExpressionProjection(buildExpression(expression), alias); + } + + private Projection buildColumnsProjection(HogQLParser.ColumnExprValueContext context, Optional alias) + { + if (context instanceof HogQLParser.ColumnExprAsteriskContext asterisk) { + requireUnaliasedColumnsProjection(alias, asterisk); + List qualifier = asterisk.tableIdentifier() == null ? List.of() : buildIdentifiers(asterisk.tableIdentifier()); + return new Star(qualifier, buildStarExclusions(asterisk.identifierList()), List.of(), sourceSpan(asterisk)); + } + if (context instanceof HogQLParser.ColumnExprColumnsRegexContext regex) { + requireUnaliasedColumnsProjection(alias, regex); + return new ColumnsRegex(decodeQuoted(regex.STRING_LITERAL().getText()), sourceSpan(regex.STRING_LITERAL().getSymbol(), regex.STRING_LITERAL().getSymbol()), sourceSpan(regex)); + } + if (context instanceof HogQLParser.ColumnExprColumnsListContext columns) { + requireUnaliasedColumnsProjection(alias, columns); + return new ColumnsList(buildExpressions(columns.columnExprList()), sourceSpan(columns)); + } + if (context instanceof HogQLParser.ColumnExprColumnsAllContext columns) { + requireUnaliasedColumnsProjection(alias, columns); + return new Star(List.of(), List.of(), List.of(), sourceSpan(columns)); + } + if (context instanceof HogQLParser.ColumnExprColumnsExcludeContext columns) { + requireUnaliasedColumnsProjection(alias, columns); + return new Star(List.of(), buildStarExclusions(columns.identifierList()), List.of(), sourceSpan(columns)); + } + if (context instanceof HogQLParser.ColumnExprColumnsReplaceContext columns) { + requireUnaliasedColumnsProjection(alias, columns); + return new Star(List.of(), List.of(), buildStarReplacements(columns.columnsReplaceList()), sourceSpan(columns)); + } + if (context instanceof HogQLParser.ColumnExprColumnsExcludeReplaceContext columns) { + requireUnaliasedColumnsProjection(alias, columns); + return new Star(List.of(), buildStarExclusions(columns.identifierList()), buildStarReplacements(columns.columnsReplaceList()), sourceSpan(columns)); + } + if (context instanceof HogQLParser.ColumnExprColumnsQualifiedAllContext columns) { + requireUnaliasedColumnsProjection(alias, columns); + return new Star(List.of(buildIdentifier(columns.identifier())), List.of(), List.of(), sourceSpan(columns)); + } + if (context instanceof HogQLParser.ColumnExprColumnsQualifiedExcludeContext columns) { + requireUnaliasedColumnsProjection(alias, columns); + return new Star(List.of(buildIdentifier(columns.identifier())), buildStarExclusions(columns.identifierList()), List.of(), sourceSpan(columns)); + } + if (context instanceof HogQLParser.ColumnExprColumnsQualifiedReplaceContext columns) { + requireUnaliasedColumnsProjection(alias, columns); + return new Star(List.of(buildIdentifier(columns.identifier())), List.of(), buildStarReplacements(columns.columnsReplaceList()), sourceSpan(columns)); + } + if (context instanceof HogQLParser.ColumnExprColumnsQualifiedExcludeReplaceContext columns) { + requireUnaliasedColumnsProjection(alias, columns); + return new Star( + List.of(buildIdentifier(columns.identifier())), + buildStarExclusions(columns.identifierList()), + buildStarReplacements(columns.columnsReplaceList()), + sourceSpan(columns)); + } + return null; + } + + private void requireUnaliasedColumnsProjection(Optional alias, ParserRuleContext context) + { + if (alias.isPresent()) { + throw unsupported(context, "aliased columns projection"); + } + } + + private List buildStarExclusions(HogQLParser.IdentifierListContext exclusions) + { + return exclusions == null ? List.of() : exclusions.nestedIdentifier().stream() + .map(identifier -> new ColumnReference(buildIdentifiers(identifier), sourceSpan(identifier))) + .toList(); + } + + private List buildStarReplacements(HogQLParser.ColumnsReplaceListContext replacements) + { + return replacements.columnsReplaceItem().stream() + .map(replacement -> new StarReplacement( + buildExpression(replacement.columnExpr()), + buildIdentifier(replacement.identifier()), + sourceSpan(replacement))) + .toList(); + } + + private Expression buildExpression(HogQLParser.ColumnExprContext context) + { + if (context instanceof HogQLParser.ColumnExprAliasContext alias) { + return buildExpression(alias.columnExpr()); + } + if (context instanceof HogQLParser.ColumnExprValuePassthroughContext passthrough) { + return buildExpression(passthrough.columnExprValue()); + } + if (context instanceof HogQLParser.ColumnExprAndContext binary) { + return binary(AND, binary.columnExpr(0), binary.columnExpr(1), binary); + } + if (context instanceof HogQLParser.ColumnExprOrContext binary) { + return binary(OR, binary.columnExpr(0), binary.columnExpr(1), binary); + } + throw unsupported(context, "expression " + context.getClass().getSimpleName()); + } + + private Expression buildExpression(HogQLParser.ColumnExprValueContext context) + { + if (context instanceof HogQLParser.ColumnExprLiteralContext literal) { + return buildLiteral(literal.literal()); + } + if (context instanceof HogQLParser.ColumnExprCaseContext caseExpression) { + return buildCaseExpression(caseExpression); + } + if (context instanceof HogQLParser.ColumnExprCastContext cast) { + return new CastExpression(buildExpression(cast.columnExpr()), buildType(cast.columnTypeExpr()), false, CastTypeDialect.HOGQL, sourceSpan(cast)); + } + if (context instanceof HogQLParser.ColumnExprTryCastContext cast) { + return new CastExpression(buildExpression(cast.columnExpr()), buildType(cast.columnTypeExpr()), true, CastTypeDialect.HOGQL, sourceSpan(cast)); + } + if (context instanceof HogQLParser.ColumnExprIntervalContext interval) { + return new IntervalExpression( + buildExpression(interval.columnExpr()), + buildIntervalUnit(interval.interval().getText(), interval.interval()), + sourceSpan(interval)); + } + if (context instanceof HogQLParser.ColumnExprIntervalStringContext interval) { + return buildStringInterval(interval); + } + if (context instanceof HogQLParser.ColumnExprIdentifierContext identifier) { + return buildColumnReference(identifier.columnIdentifier()); + } + if (context instanceof HogQLParser.ColumnExprSubqueryContext subquery) { + return new ScalarSubqueryExpression(buildSetQuery(subquery.selectSetStmt()), sourceSpan(subquery)); + } + if (context instanceof HogQLParser.ColumnExprParensContext parens) { + return buildExpression(parens.columnExpr()); + } + if (context instanceof HogQLParser.ColumnExprArrayContext array) { + List values = array.columnExprList() == null ? List.of() : buildExpressions(array.columnExprList()); + return new ArrayExpression(values, sourceSpan(array)); + } + if (context instanceof HogQLParser.ColumnExprTemplateStringContext template) { + return buildTemplateString(template.templateString()); + } + if (context instanceof HogQLParser.ColumnExprTagElementContext tag) { + return buildHogQlXTag(tag.hogqlxTagElement()); + } + if (context instanceof HogQLParser.ColumnExprTupleContext tuple) { + return new TupleExpression(buildExpressions(tuple.columnExprList()), sourceSpan(tuple)); + } + if (context instanceof HogQLParser.ColumnExprArrayAccessContext access) { + return new SubscriptExpression( + buildExpression(access.columnExprValue()), + buildExpression(access.columnExpr()), + sourceSpan(access)); + } + if (context instanceof HogQLParser.ColumnExprTupleAccessContext access) { + if (access.DECIMAL_LITERAL().getText().equals("0")) { + throw unsupported(access, "tuple index zero"); + } + return new SubscriptExpression( + buildExpression(access.columnExprValue()), + new Literal(INTEGER, access.DECIMAL_LITERAL().getText(), sourceSpan(access.DECIMAL_LITERAL().getSymbol(), access.DECIMAL_LITERAL().getSymbol())), + sourceSpan(access)); + } + if (context instanceof HogQLParser.ColumnExprPropertyAccessContext access) { + return new MemberAccessExpression( + buildExpression(access.columnExprValue()), + buildIdentifier(access.identifier()), + sourceSpan(access)); + } + if (context instanceof HogQLParser.ColumnExprNegateContext negate) { + return new UnaryExpression(NEGATE, buildExpression(negate.columnExprValue()), sourceSpan(negate)); + } + if (context instanceof HogQLParser.ColumnExprNotContext not) { + return new UnaryExpression(NOT, buildExpression(not.columnExprValue()), sourceSpan(not)); + } + if (context instanceof HogQLParser.ColumnExprPrecedence1Context binary) { + BinaryOperator operator = switch (binary.operator.getType()) { + case HogQLParser.ASTERISK -> MULTIPLY; + case HogQLParser.SLASH -> DIVIDE; + case HogQLParser.PERCENT -> MODULO; + default -> throw unsupported(binary, "multiplicative operator"); + }; + return binary(operator, binary.left, binary.right, binary); + } + if (context instanceof HogQLParser.ColumnExprPrecedence2Context binary) { + BinaryOperator operator = switch (binary.operator.getType()) { + case HogQLParser.PLUS -> ADD; + case HogQLParser.DASH -> SUBTRACT; + case HogQLParser.CONCAT -> CONCAT; + default -> throw unsupported(binary, "additive operator"); + }; + return binary(operator, binary.left, binary.right, binary); + } + if (context instanceof HogQLParser.ColumnExprPrecedence3Context binary) { + if (binary.IN() != null) { + if (binary.right instanceof HogQLParser.ColumnExprSubqueryContext subquery) { + if (binary.COHORT() != null) { + throw unsupported(binary, "IN COHORT subquery"); + } + return new InSubqueryExpression( + buildExpression(binary.left), + buildSetQuery(subquery.selectSetStmt()), + binary.NOT() != null, + sourceSpan(binary.NOT() == null ? binary.IN().getSymbol() : binary.NOT().getSymbol(), binary.getStop()), + sourceSpan(binary)); + } + if (binary.COHORT() != null) { + return new InCohortExpression( + buildExpression(binary.left), + buildExpression(binary.right), + binary.NOT() != null, + sourceSpan(binary.NOT() == null ? binary.IN().getSymbol() : binary.NOT().getSymbol(), binary.getStop()), + sourceSpan(binary)); + } + return new InExpression( + buildExpression(binary.left), + buildInValues(binary.right), + binary.NOT() != null, + sourceSpan(binary.NOT() == null ? binary.IN().getSymbol() : binary.NOT().getSymbol(), binary.getStop()), + sourceSpan(binary)); + } + if (binary.LIKE() != null) { + return binary(binary.NOT() == null ? LIKE : NOT_LIKE, binary.left, binary.right, binary); + } + if (binary.ILIKE() != null) { + return binary(binary.NOT() == null ? ILIKE : NOT_ILIKE, binary.left, binary.right, binary); + } + if (binary.operator == null) { + throw unsupported(binary, "comparison operator"); + } + BinaryOperator operator = switch (binary.operator.getType()) { + case HogQLParser.EQ_DOUBLE, HogQLParser.EQ_SINGLE -> EQUAL; + case HogQLParser.NOT_EQ -> NOT_EQUAL; + case HogQLParser.LT -> LESS_THAN; + case HogQLParser.LT_EQ -> LESS_THAN_OR_EQUAL; + case HogQLParser.GT -> GREATER_THAN; + case HogQLParser.GT_EQ -> GREATER_THAN_OR_EQUAL; + default -> throw unsupported(binary, "comparison operator"); + }; + return binary(operator, binary.left, binary.right, binary); + } + if (context instanceof HogQLParser.ColumnExprIsNullContext isNull) { + return new IsNullExpression( + buildExpression(isNull.columnExprValue()), + isNull.NOT() != null, + sourceSpan(isNull.IS().getSymbol(), isNull.getStop()), + sourceSpan(isNull)); + } + if (context instanceof HogQLParser.ColumnExprNullishContext nullish) { + Token operator = nullish.NULLISH().getSymbol(); + return new FunctionCall( + new Identifier("ifNull", false, sourceSpan(operator, operator)), + List.of(buildExpression(nullish.columnExprValue(0)), buildExpression(nullish.columnExprValue(1))), + false, + List.of(), + Optional.empty(), + sourceSpan(nullish)); + } + if (context instanceof HogQLParser.ColumnExprBetweenContext between) { + return new BetweenExpression( + buildExpression(between.columnExprValue(0)), + buildExpression(between.columnExprValue(1)), + buildExpression(between.columnExprValue(2)), + between.NOT() != null, + sourceSpan(between.NOT() == null ? between.BETWEEN().getSymbol() : between.NOT().getSymbol(), between.getStop()), + sourceSpan(between)); + } + if (context instanceof HogQLParser.ColumnExprIgnoreNullsContext ignoreNulls) { + Expression expression = buildExpression(ignoreNulls.columnExprValue()); + if (!(expression instanceof FunctionCall function) || function.window().isEmpty()) { + throw unsupported(ignoreNulls, "IGNORE NULLS outside window function"); + } + if (function.nullTreatment().isPresent()) { + throw unsupported(ignoreNulls, "duplicate window null treatment"); + } + return new FunctionCall( + function.nameParts(), + function.arguments(), + function.distinct(), + function.orderBy(), + function.filter(), + Optional.of(NullTreatment.IGNORE), + function.window(), + sourceSpan(ignoreNulls)); + } + if (context instanceof HogQLParser.ColumnExprFunctionContext function) { + return buildFunction(function); + } + if (context instanceof HogQLParser.ColumnExprWinFunctionContext function) { + return buildWindowFunction( + function, + function.identifier(), + function.columnExprs, + function.columnArgList, + function.filterExpr, + buildWindowSpecification(function.windowExpr())); + } + if (context instanceof HogQLParser.ColumnExprWinFunctionTargetContext function) { + return buildWindowFunction( + function, + function.identifier(0), + function.columnExprs, + function.columnArgList, + function.filterExpr, + new WindowReference(buildIdentifier(function.identifier(1)), sourceSpan(function.identifier(1)))); + } + if (context instanceof HogQLParser.ColumnExprLambdaContext lambda) { + HogQLParser.ColumnLambdaExprContext lambdaExpression = lambda.columnLambdaExpr(); + if (lambdaExpression instanceof HogQLParser.ArrowLambdaContext arrow) { + if (arrow.block() != null) { + throw unsupported(arrow.block(), "lambda block"); + } + return new LambdaExpression( + arrow.identifier().stream().map(this::buildIdentifier).toList(), + buildExpression(arrow.columnExpr()), + sourceSpan(arrow)); + } + if (lambdaExpression instanceof HogQLParser.ColonLambdaContext colon) { + return new LambdaExpression( + colon.identifier().stream().map(this::buildIdentifier).toList(), + buildExpression(colon.columnExpr()), + sourceSpan(colon)); + } + throw unsupported(lambdaExpression, "lambda expression"); + } + if (context instanceof HogQLParser.ColumnExprColonLambdaContext lambda) { + return new LambdaExpression( + lambda.identifier().stream().map(this::buildIdentifier).toList(), + buildExpression(lambda.columnExpr()), + sourceSpan(lambda)); + } + throw unsupported(context, "expression " + context.getClass().getSimpleName()); + } + + private Expression buildTemplateString(HogQLParser.TemplateStringContext context) + { + List parts = new ArrayList<>(); + boolean interpolated = false; + for (HogQLParser.StringContentsContext contents : context.stringContents()) { + if (contents.STRING_TEXT() != null) { + Token token = contents.STRING_TEXT().getSymbol(); + parts.add(new Literal(STRING, decodeQuoted("'" + token.getText() + "'"), sourceSpan(token, token))); + } + else { + interpolated = true; + Expression value = buildExpression(contents.columnExpr()); + parts.add(new FunctionCall( + new Identifier("toString", false, value.span()), + List.of(value), + false, + List.of(), + Optional.empty(), + value.span())); + } + } + if (!interpolated) { + String value = parts.stream() + .map(Literal.class::cast) + .map(Literal::value) + .collect(joining()); + return new Literal(STRING, value, sourceSpan(context)); + } + if (parts.size() == 1) { + return parts.getFirst(); + } + return new FunctionCall( + new Identifier("concat", false, sourceSpan(context)), + parts, + false, + List.of(), + Optional.empty(), + sourceSpan(context)); + } + + private TupleExpression buildHogQlXTag(HogQLParser.HogqlxTagElementContext context) + { + HogQLParser.IdentifierContext openingIdentifier; + List attributes; + List children = new ArrayList<>(); + if (context instanceof HogQLParser.HogqlxTagElementClosedContext closed) { + openingIdentifier = closed.identifier(); + attributes = closed.hogqlxTagAttribute(); + } + else if (context instanceof HogQLParser.HogqlxTagElementNestedContext nested) { + openingIdentifier = nested.identifier(0); + Identifier opening = buildIdentifier(openingIdentifier); + Identifier closing = buildIdentifier(nested.identifier(1)); + if (!opening.value().equals(closing.value())) { + throw unsupported(nested.identifier(1), "mismatched HogQLX closing tag"); + } + attributes = nested.hogqlxTagAttribute(); + for (HogQLParser.HogqlxChildElementContext child : nested.hogqlxChildElement()) { + if (child.hogqlxTagElement() != null) { + children.add(buildHogQlXTag(child.hogqlxTagElement())); + } + else if (child.hogqlxText() != null) { + String text = child.hogqlxText().getText(); + if (!((text.indexOf('\n') >= 0 || text.indexOf('\r') >= 0) && text.isBlank())) { + children.add(new Literal(STRING, text, sourceSpan(child.hogqlxText()))); + } + } + else { + children.add(buildExpression(child.columnExpr())); + } + } + } + else { + throw unsupported(context, "HogQLX tag"); + } + + List tupleValues = new ArrayList<>(); + tupleValues.add(new Literal(STRING, "__hx_tag", sourceSpan(context))); + tupleValues.add(new Literal(STRING, buildIdentifier(openingIdentifier).value(), sourceSpan(openingIdentifier))); + boolean hasChildrenAttribute = false; + for (HogQLParser.HogqlxTagAttributeContext attribute : attributes) { + Identifier name = buildIdentifier(attribute.identifier()); + hasChildrenAttribute |= name.value().equals("children"); + tupleValues.add(new Literal(STRING, name.value(), sourceSpan(attribute.identifier()))); + tupleValues.add(buildHogQlXAttributeValue(attribute)); + } + if (!children.isEmpty()) { + if (hasChildrenAttribute) { + throw unsupported(context, "HogQLX tag with both nested children and a children attribute"); + } + tupleValues.add(new Literal(STRING, "children", sourceSpan(context))); + tupleValues.add(new TupleExpression(children, sourceSpan(context))); + } + return new TupleExpression(tupleValues, sourceSpan(context)); + } + + private Expression buildHogQlXAttributeValue(HogQLParser.HogqlxTagAttributeContext attribute) + { + if (attribute.string() != null) { + HogQLParser.StringContext value = attribute.string(); + if (value.STRING_LITERAL() != null) { + return new Literal(STRING, decodeQuoted(value.STRING_LITERAL().getText()), sourceSpan(value)); + } + return buildTemplateString(value.templateString()); + } + if (attribute.columnExpr() != null) { + return buildExpression(attribute.columnExpr()); + } + return new Literal(HogQlQuery.LiteralKind.BOOLEAN, "true", sourceSpan(attribute)); + } + + private IntervalExpression buildStringInterval(HogQLParser.ColumnExprIntervalStringContext context) + { + String value = decodeQuoted(context.STRING_LITERAL().getText()); + int separator = value.indexOf(' '); + if (separator < 0) { + throw intervalError(context, "Unsupported interval type: must be in the format ' '"); + } + + String count = value.substring(0, separator); + String unit = value.substring(separator + 1); + if (!count.matches("[0-9]+")) { + throw intervalError(context, "Unsupported interval count: '" + count + "' is not a valid integer"); + } + try { + Long.parseLong(count); + } + catch (NumberFormatException _) { + throw intervalError(context, "Unsupported interval count: '" + count + "' is too large"); + } + + String singularUnit = unit.endsWith("s") ? unit.substring(0, unit.length() - 1) : unit; + IntervalUnit intervalUnit; + try { + intervalUnit = IntervalUnit.valueOf(singularUnit.toUpperCase(Locale.ENGLISH)); + } + catch (IllegalArgumentException _) { + throw intervalError(context, "Unsupported interval unit: " + unit); + } + if (!unit.equals(singularUnit) && !unit.equals(singularUnit + "s")) { + throw intervalError(context, "Unsupported interval unit: " + unit); + } + if (!unit.equals(unit.toLowerCase(Locale.ENGLISH))) { + throw intervalError(context, "Unsupported interval unit: " + unit); + } + return new IntervalExpression( + new Literal(INTEGER, normalizeInteger(count), sourceSpan(context.STRING_LITERAL().getSymbol(), context.STRING_LITERAL().getSymbol())), + intervalUnit, + sourceSpan(context)); + } + + private IntervalUnit buildIntervalUnit(String value, ParserRuleContext context) + { + try { + return IntervalUnit.valueOf(value.toUpperCase(Locale.ENGLISH)); + } + catch (IllegalArgumentException _) { + throw intervalError(context, "Unsupported interval unit: " + value); + } + } + + private CaseExpression buildCaseExpression(HogQLParser.ColumnExprCaseContext context) + { + List expressions = context.columnExpr(); + int index = 0; + Optional operand = Optional.empty(); + if (context.caseExpr != null) { + operand = Optional.of(buildExpression(expressions.get(index++))); + } + + List whenClauses = new ArrayList<>(); + for (int whenIndex = 0; whenIndex < context.WHEN().size(); whenIndex++) { + HogQLParser.ColumnExprContext when = expressions.get(index++); + HogQLParser.ColumnExprContext result = expressions.get(index++); + whenClauses.add(new CaseWhen( + buildExpression(when), + buildExpression(result), + sourceSpan(context.WHEN(whenIndex).getSymbol(), result.getStop()))); + } + Optional defaultValue = context.ELSE() == null ? Optional.empty() : Optional.of(buildExpression(expressions.get(index))); + return new CaseExpression(operand, whenClauses, defaultValue, sourceSpan(context)); + } + + private Identifier buildType(HogQLParser.ColumnTypeExprContext context) + { + SourceSpan span = sourceSpan(context); + return new Identifier(source.substring(span.startOffset(), span.endOffset()), false, span); + } + + private List buildExpressions(HogQLParser.ColumnExprListContext context) + { + return context.columnExpr().stream() + .map(this::buildExpression) + .toList(); + } + + private List buildInValues(HogQLParser.ColumnExprValueContext context) + { + if (context instanceof HogQLParser.ColumnExprArrayContext array) { + return array.columnExprList() == null ? List.of() : buildExpressions(array.columnExprList()); + } + if (context instanceof HogQLParser.ColumnExprTupleContext tuple) { + return buildExpressions(tuple.columnExprList()); + } + if (context instanceof HogQLParser.ColumnExprParensContext parens) { + return List.of(buildExpression(parens.columnExpr())); + } + throw unsupported(context, "IN value list"); + } + + private BinaryExpression binary(BinaryOperator operator, HogQLParser.ColumnExprContext left, HogQLParser.ColumnExprContext right, ParserRuleContext context) + { + return new BinaryExpression(operator, buildExpression(left), buildExpression(right), sourceSpan(context)); + } + + private BinaryExpression binary(BinaryOperator operator, HogQLParser.ColumnExprValueContext left, HogQLParser.ColumnExprValueContext right, ParserRuleContext context) + { + return new BinaryExpression(operator, buildExpression(left), buildExpression(right), sourceSpan(context)); + } + + private FunctionCall buildFunction(HogQLParser.ColumnExprFunctionContext context) + { + Identifier name = buildIdentifier(context.identifier()); + if (context.columnExprs != null && !Set.of("quantile", "quantileexact", "quantileif", "grouparrayif").contains(canonicalName(name))) { + throw unsupported(context, "parametric function"); + } + List arguments = new ArrayList<>(buildFunctionArguments(context)); + if (context.columnExprs != null) { + arguments.addAll(buildExpressions(context.columnExprs)); + } + List orderBy = context.orderExprList() == null ? List.of() : buildSortItems(context.orderExprList()); + Optional filter = Optional.ofNullable(context.filterExpr).map(this::buildExpression); + return new FunctionCall( + name, + List.copyOf(arguments), + context.DISTINCT() != null, + orderBy, + filter, + sourceSpan(context)); + } + + private FunctionCall buildWindowFunction( + ParserRuleContext context, + HogQLParser.IdentifierContext name, + HogQLParser.ColumnExprListContext arguments, + HogQLParser.ColumnExprListContext parametricArguments, + HogQLParser.ColumnExprContext filter, + Window window) + { + if (parametricArguments != null) { + throw unsupported(context, "parametric window function"); + } + List functionArguments; + if (arguments == null) { + functionArguments = List.of(); + } + else if (arguments.columnExpr().size() == 1 && isUnqualifiedStar(arguments.columnExpr().getFirst())) { + functionArguments = List.of(); + } + else { + functionArguments = buildExpressions(arguments); + } + Optional functionFilter = Optional.ofNullable(filter).map(this::buildExpression); + return new FunctionCall( + List.of(buildIdentifier(name)), + functionArguments, + false, + List.of(), + functionFilter, + Optional.empty(), + Optional.of(window), + sourceSpan(context)); + } + + private List buildWindowDefinitions(HogQLParser.WindowClauseContext context) + { + if (context == null) { + return List.of(); + } + List definitions = new ArrayList<>(); + for (int index = 0; index < context.identifier().size(); index++) { + Identifier name = buildIdentifier(context.identifier(index)); + WindowSpecification specification = buildWindowSpecification(context.windowExpr(index)); + definitions.add(new WindowDefinition(name, specification, enclosingSpan(name.span(), specification.span()))); + } + return List.copyOf(definitions); + } + + private WindowSpecification buildWindowSpecification(HogQLParser.WindowExprContext context) + { + List partitionBy = context.winPartitionByClause() == null + ? List.of() + : buildExpressions(context.winPartitionByClause().columnExprList()); + List orderBy = context.winOrderByClause() == null + ? List.of() + : buildSortItems(context.winOrderByClause().orderExprList()); + Optional frame = Optional.ofNullable(context.winFrameClause()).map(this::buildWindowFrame); + return new WindowSpecification(partitionBy, orderBy, frame, sourceSpan(context)); + } + + private WindowFrame buildWindowFrame(HogQLParser.WinFrameClauseContext context) + { + FrameType type = context.ROWS() != null ? FrameType.ROWS : FrameType.RANGE; + if (context.winFrameExtend() instanceof HogQLParser.FrameStartContext start) { + return new WindowFrame(type, buildFrameBound(start.winFrameBound()), Optional.empty(), sourceSpan(context)); + } + HogQLParser.FrameBetweenContext between = (HogQLParser.FrameBetweenContext) context.winFrameExtend(); + return new WindowFrame( + type, + buildFrameBound(between.winFrameBound(0)), + Optional.of(buildFrameBound(between.winFrameBound(1))), + sourceSpan(context)); + } + + private FrameBound buildFrameBound(HogQLParser.WinFrameBoundContext context) + { + if (context.CURRENT() != null) { + return new FrameBound(FrameBoundType.CURRENT_ROW, Optional.empty(), sourceSpan(context)); + } + if (context.UNBOUNDED() != null) { + FrameBoundType type = context.PRECEDING() != null + ? FrameBoundType.UNBOUNDED_PRECEDING + : FrameBoundType.UNBOUNDED_FOLLOWING; + return new FrameBound(type, Optional.empty(), sourceSpan(context)); + } + FrameBoundType type = context.PRECEDING() != null ? FrameBoundType.PRECEDING : FrameBoundType.FOLLOWING; + return new FrameBound(type, Optional.of(buildExpression(context.columnExpr())), sourceSpan(context)); + } + + private List buildFunctionArguments(HogQLParser.ColumnExprFunctionContext context) + { + if (context.columnArgList == null) { + return List.of(); + } + List arguments = context.columnArgList.columnExpr(); + if (arguments.size() == 1 && isUnqualifiedStar(arguments.getFirst())) { + if (context.DISTINCT() != null) { + throw unsupported(context, "DISTINCT star function"); + } + return List.of(); + } + return arguments.stream() + .map(this::buildExpression) + .toList(); + } + + private static boolean isUnqualifiedStar(HogQLParser.ColumnExprContext context) + { + return context instanceof HogQLParser.ColumnExprValuePassthroughContext passthrough && + passthrough.columnExprValue() instanceof HogQLParser.ColumnExprAsteriskContext asterisk && + asterisk.tableIdentifier() == null && + asterisk.EXCLUDE() == null; + } + + private Expression buildLiteral(HogQLParser.LiteralContext context) + { + if (context.NULL_SQL() != null) { + return new Literal(NULL, "null", sourceSpan(context)); + } + if (context.STRING_LITERAL() != null) { + return new Literal(STRING, decodeQuoted(context.getText()), sourceSpan(context)); + } + + String value = context.numberLiteral().getText(); + if (value.matches("[+-]?[0-9]+")) { + if (value.startsWith("-")) { + return new Literal(INTEGER, "-" + normalizeInteger(value.substring(1)), sourceSpan(context)); + } + if (value.startsWith("+")) { + Literal magnitude = new Literal(INTEGER, normalizeInteger(value.substring(1)), sourceSpan(context)); + return new UnaryExpression(POSITIVE, magnitude, sourceSpan(context)); + } + return new Literal(INTEGER, normalizeInteger(value), sourceSpan(context)); + } + if (!value.matches("[+-]?(?:[0-9]+(?:\\.[0-9]*)?|\\.[0-9]+)(?:[eE][+-]?[0-9]+)?")) { + throw unsupported(context, "numeric literal"); + } + if (value.startsWith("-")) { + return new Literal(HogQlQuery.LiteralKind.FLOAT, value, sourceSpan(context)); + } + if (value.startsWith("+")) { + Literal magnitude = new Literal(HogQlQuery.LiteralKind.FLOAT, value.substring(1), sourceSpan(context)); + return new UnaryExpression(POSITIVE, magnitude, sourceSpan(context)); + } + return new Literal(HogQlQuery.LiteralKind.FLOAT, value, sourceSpan(context)); + } + + private Expression buildColumnReference(HogQLParser.ColumnIdentifierContext context) + { + if (context.placeholder() != null) { + return buildPlaceholder(context.placeholder()); + } + List parts = new ArrayList<>(); + if (context.tableIdentifier() != null) { + parts.addAll(buildIdentifiers(context.tableIdentifier())); + } + parts.addAll(buildIdentifiers(context.nestedIdentifier())); + if (parts.size() == 1 && !parts.getFirst().delimited()) { + if (parts.getFirst().value().equalsIgnoreCase("true")) { + return new Literal(HogQlQuery.LiteralKind.BOOLEAN, "true", sourceSpan(context)); + } + if (parts.getFirst().value().equalsIgnoreCase("false")) { + return new Literal(HogQlQuery.LiteralKind.BOOLEAN, "false", sourceSpan(context)); + } + } + if (parts.size() == 1) { + String name = canonicalName(parts.getFirst()); + for (Map scope : withExpressionScopes) { + Expression expression = scope.get(name); + if (expression != null) { + return expression; + } + } + } + return new ColumnReference(parts, sourceSpan(context)); + } + + private Placeholder buildPlaceholder(HogQLParser.PlaceholderContext context) + { + Expression expression = buildExpression(context.columnExpr()); + if (!(expression instanceof ColumnReference reference)) { + throw unsupported(context, "non-name placeholder"); + } + if (reference.parts().size() == 1) { + return new Placeholder(reference.parts().getFirst().value(), sourceSpan(context)); + } + if (reference.parts().size() == 2 && + (reference.parts().getFirst().value().equalsIgnoreCase("variables") || + reference.parts().getFirst().value().equalsIgnoreCase("filters"))) { + return new Placeholder( + (reference.parts().getFirst().value().equalsIgnoreCase("variables") ? "variables." : "filters.") + reference.parts().getLast().value(), + sourceSpan(context)); + } + throw unsupported(context, "non-name placeholder"); + } + + private Relation buildRelation(HogQLParser.JoinExprContext context) + { + if (context instanceof HogQLParser.JoinExprTableContext table) { + if (table.FINAL() != null || table.sampleClause() != null) { + throw unsupported(context, "HogQL-specific table modifier"); + } + return buildTableExpression(table.tableExpr()); + } + if (context instanceof HogQLParser.JoinExprParensContext parens) { + return buildRelation(parens.joinExpr()); + } + if (context instanceof HogQLParser.JoinExprCrossOpContext cross) { + if (cross.joinOpCross().COMMA() != null) { + throw unsupported(context, "implicit comma join"); + } + return new JoinRelation( + JoinType.CROSS, + buildRelation(cross.joinExpr(0)), + buildRelation(cross.joinExpr(1)), + Optional.empty(), + sourceSpan(context)); + } + if (context instanceof HogQLParser.JoinExprOpContext join) { + if (join.NATURAL() != null) { + throw unsupported(context, "natural join"); + } + if (join.joinConstraintClause() == null) { + throw unsupported(context, "join without ON or USING"); + } + return new JoinRelation( + buildJoinType(join.joinOp()), + buildRelation(join.joinExpr(0)), + buildRelation(join.joinExpr(1)), + Optional.of(buildJoinCriteria(join.joinConstraintClause())), + sourceSpan(context)); + } + if (context instanceof HogQLParser.JoinExprPivotContext pivot) { + return buildPivot( + buildRelation(pivot.joinExpr()), + pivot.columnExprList(), + pivot.pivotColumnList(), + pivot.GROUP() != null, + context); + } + throw unsupported(context, "HogQL-specific join variant"); + } + + private Relation buildTableExpression(HogQLParser.TableExprContext context) + { + if (context instanceof HogQLParser.TableExprIdentifierContext identifier) { + List parts = buildIdentifiers(identifier.tableIdentifier()); + if (parts.size() == 1) { + String name = canonicalName(parts.getFirst()); + switch (resolveCommonTableName(name)) { + case PROHIBITED -> throw unsupported(context, "recursive CTE reference"); + case VISIBLE -> { + return new CommonTableReference(parts.getFirst(), sourceSpan(context)); + } + case ABSENT -> {} + } + } + return new TableReference(parts, sourceSpan(context)); + } + if (context instanceof HogQLParser.TableExprFunctionContext function) { + Identifier name = buildIdentifier(function.tableFunctionExpr().identifier()); + if (!canonicalName(name).equals("numbers")) { + throw unsupported(context, "table expression"); + } + List arguments = function.tableFunctionExpr().tableArgList() == null + ? List.of() + : function.tableFunctionExpr().tableArgList().columnExpr(); + if (arguments.size() != 1) { + throw unsupported(context, "numbers table function arity"); + } + SourceSpan span = sourceSpan(context); + FunctionCall range = new FunctionCall( + new Identifier("range", false, span), + List.of(buildExpression(arguments.getFirst())), + false, + List.of(), + Optional.empty(), + span); + return new UnnestRelation( + List.of(range), + new Identifier("numbers", false, span), + List.of(new Identifier("number", false, span)), + span); + } + if (context instanceof HogQLParser.TableExprPlaceholderContext placeholder) { + return new TablePlaceholder(buildPlaceholder(placeholder.placeholder())); + } + if (context instanceof HogQLParser.TableExprSubqueryContext subquery) { + return new SubqueryRelation(buildSetQuery(subquery.selectSetStmt()), sourceSpan(context)); + } + if (context instanceof HogQLParser.TableExprValuesContext values) { + List> rows = values.valuesClause().valuesRow().stream() + .map(row -> row.columnExpr().stream() + .map(this::buildExpression) + .toList()) + .toList(); + int width = rows.getFirst().size(); + for (int index = 1; index < rows.size(); index++) { + if (rows.get(index).size() != width) { + throw unsupported(values.valuesClause().valuesRow(index), "VALUES rows with different column counts"); + } + } + return new ValuesRelation(rows, sourceSpan(context)); + } + if (context instanceof HogQLParser.TableExprPivotContext pivot) { + return buildPivot( + buildTableExpression(pivot.tableExpr()), + pivot.columnExprList(), + pivot.pivotColumnList(), + pivot.GROUP() != null, + context); + } + if (context instanceof HogQLParser.TableExprAliasContext alias) { + Relation relation = buildTableExpression(alias.tableExpr()); + Identifier identifier = alias.identifier() != null + ? buildIdentifier(alias.identifier()) + : buildIdentifier(alias.alias().getText(), alias.alias()); + List columnAliases = alias.columnAliases() == null + ? List.of() + : alias.columnAliases().identifier().stream() + .map(this::buildIdentifier) + .toList(); + if (relation instanceof ValuesRelation values && !columnAliases.isEmpty() && columnAliases.size() != values.columnCount()) { + throw unsupported(alias.columnAliases(), "VALUES column alias count"); + } + return new AliasedRelation(relation, identifier, columnAliases, sourceSpan(context)); + } + throw unsupported(context, "table expression"); + } + + private PivotRelation buildPivot( + Relation input, + List expressionLists, + HogQLParser.PivotColumnListContext pivotColumnList, + boolean hasGroupBy, + ParserRuleContext context) + { + List aggregations = expressionLists.getFirst().columnExpr().stream() + .map(this::buildPivotAggregation) + .toList(); + List pivotColumns = pivotColumnList.pivotColumn(); + if (pivotColumns.size() != 1) { + throw unsupported(pivotColumns.get(1), "multiple PIVOT column clauses"); + } + HogQLParser.PivotColumnContext pivotColumn = pivotColumns.getFirst(); + List keys = buildPivotKeys(pivotColumn.columnExprTupleOrSingle()); + List valueGroups = pivotColumn.columnExprList().columnExpr().stream() + .map(this::buildPivotValueGroup) + .toList(); + List groupBy = hasGroupBy + ? expressionLists.get(1).columnExpr().stream().map(this::buildExpression).toList() + : List.of(); + return new PivotRelation(input, aggregations, keys, valueGroups, groupBy, sourceSpan(context)); + } + + private PivotAggregation buildPivotAggregation(HogQLParser.ColumnExprContext context) + { + PivotExpression expression = buildPivotExpression(context); + return new PivotAggregation(expression.expression(), expression.alias(), sourceSpan(context)); + } + + private List buildPivotKeys(HogQLParser.ColumnExprTupleOrSingleContext context) + { + List expressions = context.columnExprList() == null + ? List.of(context.columnExpr()) + : context.columnExprList().columnExpr(); + return expressions.stream() + .map(expression -> { + Expression key = buildExpression(expression); + if (!(key instanceof ColumnReference)) { + throw unsupported(expression, "non-column PIVOT key"); + } + return key; + }) + .toList(); + } + + private PivotValueGroup buildPivotValueGroup(HogQLParser.ColumnExprContext context) + { + PivotExpression expression = buildPivotExpression(context); + List values = expression.expression() instanceof TupleExpression tuple + ? tuple.values() + : List.of(expression.expression()); + return new PivotValueGroup(values, expression.alias(), sourceSpan(context)); + } + + private PivotExpression buildPivotExpression(HogQLParser.ColumnExprContext context) + { + if (context instanceof HogQLParser.ColumnExprAliasContext alias) { + Identifier identifier = alias.identifier() != null + ? buildIdentifier(alias.identifier()) + : new Identifier( + decodeQuoted(alias.STRING_LITERAL().getText()), + true, + sourceSpan(alias.STRING_LITERAL().getSymbol(), alias.STRING_LITERAL().getSymbol())); + return new PivotExpression(buildExpression(alias.columnExpr()), Optional.of(identifier)); + } + return new PivotExpression(buildExpression(context), Optional.empty()); + } + + private record PivotExpression(Expression expression, Optional alias) {} + + private CommonTableNameResolution resolveCommonTableName(String name) + { + Iterator> visibleScopes = commonTableScopes.iterator(); + Iterator> prohibitedScopes = prohibitedCommonTableScopes.iterator(); + while (visibleScopes.hasNext()) { + Set visibleNames = visibleScopes.next(); + Set prohibitedNames = prohibitedScopes.next(); + if (prohibitedNames.contains(name)) { + return CommonTableNameResolution.PROHIBITED; + } + if (visibleNames.contains(name)) { + return CommonTableNameResolution.VISIBLE; + } + } + return CommonTableNameResolution.ABSENT; + } + + private enum CommonTableNameResolution + { + ABSENT, + PROHIBITED, + VISIBLE, + } + + private static String canonicalName(Identifier identifier) + { + return identifier.delimited() ? identifier.value() : identifier.value().toLowerCase(Locale.ENGLISH); + } + + private void validateUncorrelatedSubqueries(HogQlQuery query) + { + validateQueryScope(query, Set.of()); + } + + private void validateQueryScope(HogQlQuery query, Set forbiddenOuterRelations) + { + query.with().forEach(commonTable -> validateQueryScope(commonTable.query(), forbiddenOuterRelations)); + if (query.body() instanceof SetOperation setOperation) { + query.orderBy().forEach(item -> validateExpressionScope(item.expression(), forbiddenOuterRelations, Set.of("__hogql_set_output"))); + query.limit().ifPresent(expression -> validateExpressionScope(expression, forbiddenOuterRelations, Set.of())); + query.offset().ifPresent(expression -> validateExpressionScope(expression, forbiddenOuterRelations, Set.of())); + validateQueryScope(setOperation.left(), forbiddenOuterRelations); + validateQueryScope(setOperation.right(), forbiddenOuterRelations); + return; + } + + Set localRelations = relationNames(query.from()); + boolean hasLocalRelation = query.from().isPresent(); + query.projections().forEach(projection -> { + if (projection instanceof ExpressionProjection expression) { + validateExpressionScope(expression.expression(), forbiddenOuterRelations, localRelations, hasLocalRelation); + } + }); + query.where().ifPresent(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations, hasLocalRelation)); + query.groupBy().forEach(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations, hasLocalRelation)); + query.having().ifPresent(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations, hasLocalRelation)); + query.orderBy().forEach(item -> validateExpressionScope(item.expression(), forbiddenOuterRelations, localRelations, hasLocalRelation)); + query.limit().ifPresent(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations, hasLocalRelation)); + query.offset().ifPresent(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations, hasLocalRelation)); + query.from().ifPresent(relation -> validateRelationExpressionScopes(relation, forbiddenOuterRelations, localRelations, hasLocalRelation)); + + Set nestedForbiddenRelations = new LinkedHashSet<>(forbiddenOuterRelations); + nestedForbiddenRelations.addAll(localRelations); + query.from().ifPresent(relation -> validateNestedRelationScopes(relation, Set.copyOf(nestedForbiddenRelations))); + } + + private void validateRelationExpressionScopes(Relation relation, Set forbiddenOuterRelations, Set localRelations, boolean hasLocalRelation) + { + switch (relation) { + case AliasedRelation alias -> validateRelationExpressionScopes(alias.relation(), forbiddenOuterRelations, localRelations, hasLocalRelation); + case JoinRelation join -> { + validateRelationExpressionScopes(join.left(), forbiddenOuterRelations, localRelations, hasLocalRelation); + validateRelationExpressionScopes(join.right(), forbiddenOuterRelations, localRelations, hasLocalRelation); + join.criteria().ifPresent(criteria -> { + if (criteria instanceof JoinOn on) { + validateExpressionScope(on.expression(), forbiddenOuterRelations, localRelations, hasLocalRelation); + } + }); + } + case PivotRelation pivot -> { + validateRelationExpressionScopes(pivot.input(), forbiddenOuterRelations, localRelations, hasLocalRelation); + pivot.aggregations().forEach(aggregation -> + validateExpressionScope(aggregation.expression(), forbiddenOuterRelations, localRelations, hasLocalRelation)); + pivot.pivotColumns().forEach(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations, hasLocalRelation)); + pivot.valueGroups().forEach(group -> group.values().forEach(expression -> + validateExpressionScope(expression, forbiddenOuterRelations, localRelations, hasLocalRelation))); + pivot.groupBy().forEach(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations, hasLocalRelation)); + } + case UnnestRelation unnest -> unnest.expressions().forEach(expression -> validateExpressionScope(expression, forbiddenOuterRelations, localRelations, hasLocalRelation)); + case CommonTableReference _, SubqueryRelation _, TablePlaceholder _, TableReference _, ValuesRelation _ -> {} + } + } + + private void validateNestedRelationScopes(Relation relation, Set forbiddenOuterRelations) + { + switch (relation) { + case AliasedRelation alias -> validateNestedRelationScopes(alias.relation(), forbiddenOuterRelations); + case JoinRelation join -> { + validateNestedRelationScopes(join.left(), forbiddenOuterRelations); + validateNestedRelationScopes(join.right(), forbiddenOuterRelations); + } + case PivotRelation pivot -> validateNestedRelationScopes(pivot.input(), forbiddenOuterRelations); + case SubqueryRelation subquery -> validateQueryScope(subquery.query(), forbiddenOuterRelations); + case UnnestRelation _ -> {} + case ValuesRelation values -> values.rows().forEach(row -> row.forEach(expression -> validateExpressionScope(expression, forbiddenOuterRelations, Set.of()))); + case CommonTableReference _, TablePlaceholder _, TableReference _ -> {} + } + } + + private Set relationNames(Optional relation) + { + Set names = new LinkedHashSet<>(); + relation.ifPresent(value -> collectRelationNames(value, names)); + return Set.copyOf(names); + } + + private void collectRelationNames(Relation relation, Set names) + { + switch (relation) { + case AliasedRelation alias -> names.add(canonicalName(alias.alias())); + case CommonTableReference commonTable -> names.add(canonicalName(commonTable.name())); + case JoinRelation join -> { + collectRelationNames(join.left(), names); + collectRelationNames(join.right(), names); + } + case PivotRelation pivot -> collectRelationNames(pivot.input(), names); + case UnnestRelation unnest -> names.add(canonicalName(unnest.alias())); + case SubqueryRelation _, TablePlaceholder _, ValuesRelation _ -> {} + case TableReference table -> names.add(canonicalName(table.parts().getLast())); + } + } + + private void validateExpressionScope(Expression expression, Set forbiddenOuterRelations, Set localRelations) + { + validateExpressionScope(expression, forbiddenOuterRelations, localRelations, !localRelations.isEmpty()); + } + + private void validateExpressionScope(Expression expression, Set forbiddenOuterRelations, Set localRelations, boolean hasLocalRelation) + { + switch (expression) { + case ArrayExpression array -> array.values().forEach(value -> validateExpressionScope(value, forbiddenOuterRelations, localRelations, hasLocalRelation)); + case BetweenExpression between -> { + validateExpressionScope(between.value(), forbiddenOuterRelations, localRelations, hasLocalRelation); + validateExpressionScope(between.min(), forbiddenOuterRelations, localRelations, hasLocalRelation); + validateExpressionScope(between.max(), forbiddenOuterRelations, localRelations, hasLocalRelation); + } + case BinaryExpression binary -> { + validateExpressionScope(binary.left(), forbiddenOuterRelations, localRelations, hasLocalRelation); + validateExpressionScope(binary.right(), forbiddenOuterRelations, localRelations, hasLocalRelation); + } + case CaseExpression caseExpression -> { + caseExpression.operand().ifPresent(value -> validateExpressionScope(value, forbiddenOuterRelations, localRelations, hasLocalRelation)); + caseExpression.whenClauses().forEach(when -> { + validateExpressionScope(when.operand(), forbiddenOuterRelations, localRelations, hasLocalRelation); + validateExpressionScope(when.result(), forbiddenOuterRelations, localRelations, hasLocalRelation); + }); + caseExpression.defaultValue().ifPresent(value -> validateExpressionScope(value, forbiddenOuterRelations, localRelations, hasLocalRelation)); + } + case CastExpression cast -> validateExpressionScope(cast.value(), forbiddenOuterRelations, localRelations, hasLocalRelation); + case ColumnReference reference -> { + if (reference.parts().size() == 1 && !forbiddenOuterRelations.isEmpty() && !hasLocalRelation) { + throw unsupported(reference.span(), "correlated relation subquery"); + } + if (reference.parts().size() > 1) { + String qualifier = canonicalName(reference.parts().getFirst()); + if (forbiddenOuterRelations.contains(qualifier) && !localRelations.contains(qualifier)) { + throw unsupported(reference.span(), "correlated relation subquery"); + } + } + } + case FunctionCall function -> { + function.arguments().forEach(value -> validateExpressionScope(value, forbiddenOuterRelations, localRelations, hasLocalRelation)); + function.orderBy().forEach(item -> validateExpressionScope(item.expression(), forbiddenOuterRelations, localRelations, hasLocalRelation)); + function.filter().ifPresent(value -> validateExpressionScope(value, forbiddenOuterRelations, localRelations, hasLocalRelation)); + } + case InExpression in -> { + validateExpressionScope(in.value(), forbiddenOuterRelations, localRelations, hasLocalRelation); + in.values().forEach(value -> validateExpressionScope(value, forbiddenOuterRelations, localRelations, hasLocalRelation)); + } + case InCohortExpression in -> { + validateExpressionScope(in.value(), forbiddenOuterRelations, localRelations, hasLocalRelation); + validateExpressionScope(in.cohort(), forbiddenOuterRelations, localRelations, hasLocalRelation); + } + case InSubqueryExpression in -> { + validateExpressionScope(in.value(), forbiddenOuterRelations, localRelations, hasLocalRelation); + validateQueryScope(in.query(), Set.of()); + } + case IntervalExpression interval -> validateExpressionScope(interval.value(), forbiddenOuterRelations, localRelations, hasLocalRelation); + case IsNullExpression isNull -> validateExpressionScope(isNull.value(), forbiddenOuterRelations, localRelations, hasLocalRelation); + case LambdaExpression lambda -> validateExpressionScope(lambda.body(), forbiddenOuterRelations, localRelations, hasLocalRelation); + case Literal _, Placeholder _ -> {} + case MemberAccessExpression memberAccess -> validateExpressionScope(memberAccess.base(), forbiddenOuterRelations, localRelations, hasLocalRelation); + case ScalarSubqueryExpression subquery -> validateQueryScope(subquery.query(), Set.of()); + case SubscriptExpression subscript -> { + validateExpressionScope(subscript.base(), forbiddenOuterRelations, localRelations, hasLocalRelation); + validateExpressionScope(subscript.index(), forbiddenOuterRelations, localRelations, hasLocalRelation); + } + case TupleExpression tuple -> tuple.values().forEach(value -> validateExpressionScope(value, forbiddenOuterRelations, localRelations, hasLocalRelation)); + case UnaryExpression unary -> validateExpressionScope(unary.operand(), forbiddenOuterRelations, localRelations, hasLocalRelation); + } + } + + private JoinType buildJoinType(HogQLParser.JoinOpContext context) + { + if (context == null) { + return JoinType.INNER; + } + if (context instanceof HogQLParser.JoinOpInnerContext inner) { + if (inner.ANTI() != null || inner.SEMI() != null || inner.ASOF() != null || inner.ALL() != null) { + throw unsupported(context, "HogQL-specific inner join modifier"); + } + return inner.ANY() == null ? JoinType.INNER : JoinType.INNER_ANY; + } + if (context instanceof HogQLParser.JoinOpLeftRightContext outer) { + if (outer.ANTI() != null || outer.SEMI() != null || outer.ASOF() != null || outer.ALL() != null) { + throw unsupported(context, "HogQL-specific outer join modifier"); + } + if (outer.ANY() != null) { + if (outer.LEFT() == null) { + throw unsupported(context, "RIGHT ANY JOIN"); + } + return JoinType.LEFT_ANY; + } + return outer.LEFT() != null ? JoinType.LEFT : JoinType.RIGHT; + } + if (context instanceof HogQLParser.JoinOpFullContext full) { + if (full.ASOF() != null || full.ALL() != null || full.ANY() != null) { + throw unsupported(context, "HogQL-specific full join modifier"); + } + return JoinType.FULL; + } + throw unsupported(context, "join operator"); + } + + private JoinCriteria buildJoinCriteria(HogQLParser.JoinConstraintClauseContext context) + { + if (context.ON() != null) { + List expressions = context.columnExprList().columnExpr(); + if (expressions.size() != 1) { + throw unsupported(context, "multiple ON expressions"); + } + return new JoinOn(buildExpression(expressions.getFirst()), sourceSpan(context)); + } + List columns = context.columnExprList().columnExpr().stream() + .map(this::buildExpression) + .map(expression -> { + if (!(expression instanceof ColumnReference reference) || reference.parts().size() != 1) { + throw unsupported(context, "non-identifier USING column"); + } + return reference.parts().getFirst(); + }) + .toList(); + return new JoinUsing(columns, sourceSpan(context)); + } + + private List buildIdentifiers(HogQLParser.TableIdentifierContext context) + { + List parts = new ArrayList<>(); + if (context.databaseIdentifier() != null) { + parts.add(buildIdentifier(context.databaseIdentifier().identifier())); + } + parts.addAll(buildIdentifiers(context.nestedIdentifier())); + return List.copyOf(parts); + } + + private List buildIdentifiers(HogQLParser.NestedIdentifierContext context) + { + return context.identifier().stream() + .map(this::buildIdentifier) + .toList(); + } + + private Identifier buildIdentifier(HogQLParser.IdentifierContext context) + { + return buildIdentifier(context.getText(), context); + } + + private Identifier buildIdentifier(String text, ParserRuleContext context) + { + boolean delimited = text.startsWith("`") || text.startsWith("\""); + return new Identifier(delimited ? decodeQuoted(text) : text, delimited, sourceSpan(context)); + } + + private SourceSpan sourceSpan(ParserRuleContext context) + { + return sourceSpan(context.getStart(), context.getStop()); + } + + private SourceSpan sourceSpan(Token start, Token stop) + { + int startOffset = start.getStartIndex(); + int endOffset = stop.getStopIndex() + 1; + return new SourceSpan( + startOffset, + endOffset, + sourcePositions.line(startOffset), + sourcePositions.column(startOffset), + sourcePositions.line(endOffset), + sourcePositions.column(endOffset)); + } + + private HogQlParsingException unsupported(ParserRuleContext context, String feature) + { + return unsupported(sourceSpan(context), feature); + } + + private HogQlParsingException unsupported(SourceSpan span, String feature) + { + return new HogQlParsingException("HogQL feature is not lowered yet: " + feature, null, span.startLine(), span.startColumn()); + } + + private HogQlParsingException intervalError(ParserRuleContext context, String message) + { + SourceSpan span = sourceSpan(context); + return new HogQlParsingException(message, null, span.startLine(), span.startColumn()); + } + + private static String normalizeInteger(String value) + { + int firstNonZero = 0; + while (firstNonZero < value.length() - 1 && value.charAt(firstNonZero) == '0') { + firstNonZero++; + } + return value.substring(firstNonZero); + } + + private static String decodeQuoted(String text) + { + char quote = text.charAt(0); + StringBuilder decoded = new StringBuilder(text.length() - 2); + for (int index = 1; index < text.length() - 1; index++) { + char character = text.charAt(index); + if (character == quote && text.charAt(index + 1) == quote) { + decoded.append(quote); + index++; + } + else if (character != '\\') { + decoded.append(character); + } + else { + char escaped = text.charAt(++index); + if (escaped == 'x') { + decoded.append((char) ((digit(text.charAt(++index), 16) << 4) | digit(text.charAt(++index), 16))); + } + else { + decoded.append(switch (escaped) { + case '0' -> '\0'; + case 'a' -> '\u0007'; + case 'b' -> '\b'; + case 'f' -> '\f'; + case 'n' -> '\n'; + case 'r' -> '\r'; + case 't' -> '\t'; + case 'v' -> '\u000B'; + default -> escaped; + }); + } + } + } + return decoded.toString(); + } + } + + private static final class SourcePositions + { + private final int[] lines; + private final int[] columns; + + private SourcePositions(String source) + { + int length = source.codePointCount(0, source.length()); + lines = new int[length + 1]; + columns = new int[length + 1]; + lines[0] = 1; + columns[0] = 1; + + int line = 1; + int column = 1; + int codePointOffset = 0; + boolean previousWasCarriageReturn = false; + for (int charOffset = 0; charOffset < source.length(); ) { + int codePoint = source.codePointAt(charOffset); + charOffset += Character.charCount(codePoint); + codePointOffset++; + if (codePoint == '\n' && previousWasCarriageReturn) { + previousWasCarriageReturn = false; + } + else if (codePoint == '\n' || codePoint == '\r') { + line++; + column = 1; + previousWasCarriageReturn = codePoint == '\r'; + } + else { + column++; + previousWasCarriageReturn = false; + } + lines[codePointOffset] = line; + columns[codePointOffset] = column; + } + } + + public int line(int offset) + { + return lines[offset]; + } + + public int column(int offset) + { + return columns[offset]; + } + } +} diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParserLimits.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParserLimits.java new file mode 100644 index 000000000000..738ee134d160 --- /dev/null +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParserLimits.java @@ -0,0 +1,37 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +public record HogQlParserLimits(int maxTokens, int maxParseDepth, int maxParseTreeNodes) +{ + private static final HogQlParserLimits DEFAULTS = new HogQlParserLimits(200_000, 256, 500_000); + + public HogQlParserLimits + { + if (maxTokens < 1) { + throw new IllegalArgumentException("maxTokens must be positive"); + } + if (maxParseDepth < 1) { + throw new IllegalArgumentException("maxParseDepth must be positive"); + } + if (maxParseTreeNodes < 1) { + throw new IllegalArgumentException("maxParseTreeNodes must be positive"); + } + } + + public static HogQlParserLimits defaults() + { + return DEFAULTS; + } +} diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParsingException.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParsingException.java new file mode 100644 index 000000000000..7fc12141df41 --- /dev/null +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlParsingException.java @@ -0,0 +1,55 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +public final class HogQlParsingException + extends RuntimeException +{ + private final int line; + private final int column; + + public HogQlParsingException(String message, Throwable cause, int line, int column) + { + super(message, cause); + if (line < 1) { + throw new IllegalArgumentException("line must be positive"); + } + if (column < 1) { + throw new IllegalArgumentException("column must be positive"); + } + this.line = line; + this.column = column; + } + + public int getLineNumber() + { + return line; + } + + public int getColumnNumber() + { + return column; + } + + public String getErrorMessage() + { + return super.getMessage(); + } + + @Override + public String getMessage() + { + return "line %s:%s: %s".formatted(line, column, getErrorMessage()); + } +} diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlSyntaxAstManifest.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlSyntaxAstManifest.java new file mode 100644 index 000000000000..1ecc3e7e4ffd --- /dev/null +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/HogQlSyntaxAstManifest.java @@ -0,0 +1,273 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonValue; +import io.airlift.json.JsonCodec; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HashMap; +import java.util.HexFormat; +import java.util.List; +import java.util.Map; + +import static io.airlift.json.JsonCodec.jsonCodec; +import static java.util.Objects.requireNonNull; + +public record HogQlSyntaxAstManifest( + int schemaVersion, + HogQlLanguageVersion languageVersion, + String grammarSha256, + String grammarFeatureManifestSha256, + String grammarAlternativeIdentityManifestSha256, + SyntaxNodeKind syntaxTreeKind, + SourceSpanGuarantee sourceSpanGuarantee, + List features) +{ + private static final String LANGUAGE_RESOURCE_ROOT = "/io/trino/hogql/parser/language/1.0.0/"; + private static final String CURRENT_RESOURCE = LANGUAGE_RESOURCE_ROOT + "trino-syntax-ast.json"; + private static final JsonCodec MANIFEST_CODEC = jsonCodec(HogQlSyntaxAstManifest.class); + private static final JsonCodec GRAMMAR_FEATURE_MANIFEST_CODEC = jsonCodec(HogQlCompatibilityManifest.PublishedGrammarFeatureManifest.class); + private static final HogQlSyntaxAstManifest CURRENT = loadCurrent(); + + @JsonCreator + public HogQlSyntaxAstManifest( + @JsonProperty("schemaVersion") int schemaVersion, + @JsonProperty("languageVersion") HogQlLanguageVersion languageVersion, + @JsonProperty("grammarSha256") String grammarSha256, + @JsonProperty("grammarFeatureManifestSha256") String grammarFeatureManifestSha256, + @JsonProperty("grammarAlternativeIdentityManifestSha256") String grammarAlternativeIdentityManifestSha256, + @JsonProperty("syntaxTreeKind") SyntaxNodeKind syntaxTreeKind, + @JsonProperty("sourceSpanGuarantee") SourceSpanGuarantee sourceSpanGuarantee, + @JsonProperty("features") List features) + { + if (schemaVersion != 1) { + throw new IllegalArgumentException("unsupported HogQL syntax AST manifest schema: " + schemaVersion); + } + this.schemaVersion = schemaVersion; + this.languageVersion = requireNonNull(languageVersion, "languageVersion is null"); + this.grammarSha256 = requireNonNull(grammarSha256, "grammarSha256 is null"); + this.grammarFeatureManifestSha256 = requireNonNull(grammarFeatureManifestSha256, "grammarFeatureManifestSha256 is null"); + this.grammarAlternativeIdentityManifestSha256 = requireNonNull(grammarAlternativeIdentityManifestSha256, "grammarAlternativeIdentityManifestSha256 is null"); + this.syntaxTreeKind = requireNonNull(syntaxTreeKind, "syntaxTreeKind is null"); + this.sourceSpanGuarantee = requireNonNull(sourceSpanGuarantee, "sourceSpanGuarantee is null"); + this.features = List.copyOf(requireNonNull(features, "features is null")); + if (syntaxTreeKind != SyntaxNodeKind.TREE) { + throw new IllegalArgumentException("HogQL syntax AST root must use the tree node kind"); + } + } + + public static HogQlSyntaxAstManifest current() + { + return CURRENT; + } + + private static HogQlSyntaxAstManifest loadCurrent() + { + HogQlSyntaxAstManifest manifest = MANIFEST_CODEC.fromJson(readResource(CURRENT_RESOURCE)); + HogQlLanguageContract languageContract = HogQlLanguageContract.current(); + if (!manifest.languageVersion().equals(languageContract.languageVersion())) { + throw new IllegalStateException("HogQL syntax AST manifest language version does not match the language contract"); + } + if (!manifest.grammarSha256().equals(languageContract.grammarSha256())) { + throw new IllegalStateException("HogQL syntax AST manifest grammar hash does not match the language contract"); + } + if (!manifest.grammarFeatureManifestSha256().equals(languageContract.grammarFeatureManifest().sha256())) { + throw new IllegalStateException("HogQL syntax AST manifest feature hash does not match the language contract"); + } + if (!manifest.grammarAlternativeIdentityManifestSha256().equals(HogQlGrammarAlternativeIdentities.currentSha256())) { + throw new IllegalStateException("HogQL syntax AST manifest alternative identity hash does not match the published sidecar"); + } + + String featureManifestResource = LANGUAGE_RESOURCE_ROOT + languageContract.grammarFeatureManifest().path(); + byte[] featureManifestBytes = readResourceBytes(featureManifestResource); + if (!sha256(featureManifestBytes).equals(manifest.grammarFeatureManifestSha256())) { + throw new IllegalStateException("published HogQL grammar feature manifest checksum does not match the syntax AST manifest"); + } + HogQlCompatibilityManifest.PublishedGrammarFeatureManifest published = GRAMMAR_FEATURE_MANIFEST_CODEC.fromJson(new String(featureManifestBytes, StandardCharsets.UTF_8)); + validateFeatureCoverage(manifest.features(), published, HogQlGrammarAlternativeIdentities.current()); + return manifest; + } + + static void validateFeatureCoverage( + List features, + HogQlCompatibilityManifest.PublishedGrammarFeatureManifest published, + HogQlGrammarAlternativeIdentities alternativeIdentities) + { + HogQlLanguageContract languageContract = HogQlLanguageContract.current(); + if (!published.languageVersion().equals(languageContract.languageVersion()) || !published.grammarSha256().equals(languageContract.grammarSha256())) { + throw new IllegalStateException("published HogQL grammar features do not match the language contract"); + } + + Map expectedById = new HashMap<>(); + for (HogQlCompatibilityManifest.PublishedFeature feature : published.features()) { + SyntaxNodeKind nodeKind = switch (feature.kind()) { + case "token" -> SyntaxNodeKind.TOKEN; + case "parserRule", "parserAlternative" -> SyntaxNodeKind.RULE; + default -> throw new IllegalStateException("unknown published HogQL grammar feature kind: " + feature.kind()); + }; + if (expectedById.put(feature.id(), nodeKind) != null) { + throw new IllegalStateException("duplicate published HogQL grammar feature: " + feature.id()); + } + } + + Map sourceUnlabeledByRule = new HashMap<>(); + for (HogQlCompatibilityManifest.SourceUnlabeledAlternativeRule rule : published.validationErrors()) { + if (sourceUnlabeledByRule.put(rule.rule(), rule) != null) { + throw new IllegalStateException("duplicate source-unlabeled HogQL grammar alternative rule: " + rule.rule()); + } + } + for (HogQlGrammarAlternativeIdentities.AlternativeRule rule : alternativeIdentities.rules()) { + HogQlCompatibilityManifest.SourceUnlabeledAlternativeRule sourceRule = sourceUnlabeledByRule.remove(rule.rule()); + if (sourceRule == null || sourceRule.alternativeCount() != rule.alternatives().size() || sourceRule.queryReachable() != rule.queryReachable()) { + throw new IllegalStateException("HogQL grammar alternative identities do not match the published rule: " + rule.rule()); + } + for (HogQlGrammarAlternativeIdentities.Alternative alternative : rule.alternatives()) { + if (expectedById.put(alternative.id(), SyntaxNodeKind.RULE) != null) { + throw new IllegalStateException("duplicate published HogQL grammar feature: " + alternative.id()); + } + } + } + if (!sourceUnlabeledByRule.isEmpty()) { + throw new IllegalStateException("HogQL grammar alternative identities do not cover every source-unlabeled rule"); + } + + Map manifestById = new HashMap<>(); + for (Feature feature : features) { + if (manifestById.put(feature.id(), feature) != null) { + throw new IllegalStateException("duplicate HogQL syntax AST feature: " + feature.id()); + } + SyntaxNodeKind expected = expectedById.get(feature.id()); + if (expected == null) { + throw new IllegalStateException("unknown HogQL syntax AST feature: " + feature.id()); + } + if (feature.syntaxNodeKind() != expected) { + throw new IllegalStateException("HogQL syntax AST feature has the wrong node kind: " + feature.id()); + } + } + if (!manifestById.keySet().equals(expectedById.keySet())) { + throw new IllegalStateException("HogQL syntax AST manifest does not account for every grammar feature"); + } + } + + private static String readResource(String path) + { + return new String(readResourceBytes(path), StandardCharsets.UTF_8); + } + + private static byte[] readResourceBytes(String path) + { + try (InputStream input = HogQlSyntaxAstManifest.class.getResourceAsStream(path)) { + if (input == null) { + throw new IllegalStateException("HogQL syntax AST resource is missing: " + path); + } + return input.readAllBytes(); + } + catch (IOException e) { + throw new UncheckedIOException("failed to read HogQL syntax AST resource", e); + } + } + + private static String sha256(byte[] content) + { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(content)); + } + catch (NoSuchAlgorithmException e) { + throw new AssertionError(e); + } + } + + public record Feature(String id, SyntaxNodeKind syntaxNodeKind) + { + @JsonCreator + public Feature( + @JsonProperty("id") String id, + @JsonProperty("syntaxNodeKind") SyntaxNodeKind syntaxNodeKind) + { + this.id = requireNonNull(id, "id is null"); + this.syntaxNodeKind = requireNonNull(syntaxNodeKind, "syntaxNodeKind is null"); + if (id.isBlank() || syntaxNodeKind == SyntaxNodeKind.TREE) { + throw new IllegalArgumentException("invalid HogQL syntax AST feature: " + id); + } + } + } + + public enum SyntaxNodeKind + { + TREE("tree"), + RULE("rule"), + TOKEN("token"); + + private final String value; + + SyntaxNodeKind(String value) + { + this.value = value; + } + + @JsonCreator + public static SyntaxNodeKind fromJson(String value) + { + for (SyntaxNodeKind kind : values()) { + if (kind.value.equals(value)) { + return kind; + } + } + throw new IllegalArgumentException("unknown HogQL syntax node kind: " + value); + } + + @JsonValue + public String toJson() + { + return value; + } + } + + public enum SourceSpanGuarantee + { + CODE_POINT_OFFSETS_END_EXCLUSIVE_ONE_BASED_LINE_COLUMNS("codePointOffsetsEndExclusiveOneBasedLineColumns"); + + private final String value; + + SourceSpanGuarantee(String value) + { + this.value = value; + } + + @JsonCreator + public static SourceSpanGuarantee fromJson(String value) + { + for (SourceSpanGuarantee guarantee : values()) { + if (guarantee.value.equals(value)) { + return guarantee; + } + } + throw new IllegalArgumentException("unknown HogQL source span guarantee: " + value); + } + + @JsonValue + public String toJson() + { + return value; + } + } +} diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java new file mode 100644 index 000000000000..e640e11944f8 --- /dev/null +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlQuery.java @@ -0,0 +1,1056 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser.tree; + +import java.util.List; +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +public record HogQlQuery( + List with, + QueryBody body, + List orderBy, + Optional limit, + Optional offset, + SourceSpan span) +{ + public HogQlQuery + { + with = List.copyOf(requireNonNull(with, "with is null")); + body = requireNonNull(body, "body is null"); + orderBy = List.copyOf(requireNonNull(orderBy, "orderBy is null")); + limit = requireNonNull(limit, "limit is null"); + offset = requireNonNull(offset, "offset is null"); + span = requireNonNull(span, "span is null"); + } + + public HogQlQuery( + List with, + boolean distinct, + List projections, + Optional from, + Optional where, + List groupBy, + Optional having, + List windows, + List orderBy, + Optional limit, + Optional offset, + SourceSpan span) + { + this(with, new SelectQueryBody(distinct, projections, from, where, groupBy, having, windows, span), orderBy, limit, offset, span); + } + + public HogQlQuery( + List with, + boolean distinct, + List projections, + Optional from, + Optional where, + List groupBy, + Optional having, + List orderBy, + Optional limit, + Optional offset, + SourceSpan span) + { + this(with, distinct, projections, from, where, groupBy, having, List.of(), orderBy, limit, offset, span); + } + + public boolean distinct() + { + return selectBody().distinct(); + } + + public List projections() + { + return selectBody().projections(); + } + + public Optional from() + { + return selectBody().from(); + } + + public Optional where() + { + return selectBody().where(); + } + + public List groupBy() + { + return selectBody().groupBy(); + } + + public Optional having() + { + return selectBody().having(); + } + + public Optional limitBy() + { + return selectBody().limitBy(); + } + + public List windows() + { + return selectBody().windows(); + } + + private SelectQueryBody selectBody() + { + if (body instanceof SelectQueryBody select) { + return select; + } + throw new IllegalStateException("query body is not a SELECT"); + } + + public sealed interface QueryBody + permits SelectQueryBody, SetOperation + { + SourceSpan span(); + } + + public record SelectQueryBody( + boolean distinct, + List projections, + Optional from, + Optional where, + List groupBy, + Optional having, + List windows, + Optional limitBy, + SourceSpan span) + implements QueryBody + { + public SelectQueryBody + { + projections = List.copyOf(requireNonNull(projections, "projections is null")); + from = requireNonNull(from, "from is null"); + where = requireNonNull(where, "where is null"); + groupBy = List.copyOf(requireNonNull(groupBy, "groupBy is null")); + having = requireNonNull(having, "having is null"); + windows = List.copyOf(requireNonNull(windows, "windows is null")); + limitBy = requireNonNull(limitBy, "limitBy is null"); + span = requireNonNull(span, "span is null"); + } + + public SelectQueryBody( + boolean distinct, + List projections, + Optional from, + Optional where, + List groupBy, + Optional having, + List windows, + SourceSpan span) + { + this(distinct, projections, from, where, groupBy, having, windows, Optional.empty(), span); + } + + public SelectQueryBody( + boolean distinct, + List projections, + Optional from, + Optional where, + List groupBy, + Optional having, + SourceSpan span) + { + this(distinct, projections, from, where, groupBy, having, List.of(), Optional.empty(), span); + } + } + + public record LimitBy(Expression limit, Optional offset, List partitionBy, SourceSpan span) + { + public LimitBy + { + limit = requireNonNull(limit, "limit is null"); + offset = requireNonNull(offset, "offset is null"); + partitionBy = List.copyOf(requireNonNull(partitionBy, "partitionBy is null")); + if (partitionBy.isEmpty()) { + throw new IllegalArgumentException("partitionBy is empty"); + } + span = requireNonNull(span, "span is null"); + } + } + + public record SetOperation( + SetOperationType type, + boolean distinct, + HogQlQuery left, + HogQlQuery right, + boolean leftParenthesized, + boolean rightParenthesized, + SourceSpan operatorSpan, + SourceSpan span) + implements QueryBody + { + public SetOperation + { + type = requireNonNull(type, "type is null"); + left = requireNonNull(left, "left is null"); + right = requireNonNull(right, "right is null"); + operatorSpan = requireNonNull(operatorSpan, "operatorSpan is null"); + span = requireNonNull(span, "span is null"); + } + } + + public enum SetOperationType + { + EXCEPT, + INTERSECT, + UNION, + } + + public record CommonTableExpression(Identifier name, List columnAliases, HogQlQuery query, SourceSpan span) + { + public CommonTableExpression + { + name = requireNonNull(name, "name is null"); + columnAliases = List.copyOf(requireNonNull(columnAliases, "columnAliases is null")); + query = requireNonNull(query, "query is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record SortItem(Expression expression, SortDirection direction, NullPlacement nullPlacement, SourceSpan span) + { + public SortItem + { + expression = requireNonNull(expression, "expression is null"); + direction = requireNonNull(direction, "direction is null"); + nullPlacement = requireNonNull(nullPlacement, "nullPlacement is null"); + span = requireNonNull(span, "span is null"); + } + } + + public enum SortDirection + { + ASCENDING, + DESCENDING, + } + + public enum NullPlacement + { + FIRST, + LAST, + UNDEFINED, + } + + public sealed interface Projection + permits ColumnsList, + ColumnsRegex, + ExpressionProjection, + Star + { + SourceSpan span(); + } + + public record ColumnsRegex(String pattern, SourceSpan patternSpan, SourceSpan span) + implements Projection + { + public ColumnsRegex + { + pattern = requireNonNull(pattern, "pattern is null"); + patternSpan = requireNonNull(patternSpan, "patternSpan is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record ColumnsList(List expressions, SourceSpan span) + implements Projection + { + public ColumnsList + { + expressions = List.copyOf(requireNonNull(expressions, "expressions is null")); + span = requireNonNull(span, "span is null"); + } + } + + public record Star(List qualifier, List exclusions, List replacements, SourceSpan span) + implements Projection + { + public Star + { + qualifier = List.copyOf(requireNonNull(qualifier, "qualifier is null")); + exclusions = List.copyOf(requireNonNull(exclusions, "exclusions is null")); + replacements = List.copyOf(requireNonNull(replacements, "replacements is null")); + span = requireNonNull(span, "span is null"); + } + + public Star(List qualifier, List exclusions, SourceSpan span) + { + this(qualifier, exclusions, List.of(), span); + } + + public Star(SourceSpan span) + { + this(List.of(), List.of(), List.of(), span); + } + } + + public record StarReplacement(Expression expression, Identifier target, SourceSpan span) + { + public StarReplacement + { + expression = requireNonNull(expression, "expression is null"); + target = requireNonNull(target, "target is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record ExpressionProjection(Expression expression, Optional alias) + implements Projection + { + public ExpressionProjection + { + expression = requireNonNull(expression, "expression is null"); + alias = requireNonNull(alias, "alias is null"); + } + + @Override + public SourceSpan span() + { + return expression.span(); + } + } + + public sealed interface Expression + permits ArrayExpression, + BetweenExpression, + BinaryExpression, + CaseExpression, + CastExpression, + ColumnReference, + FunctionCall, + InCohortExpression, + InExpression, + InSubqueryExpression, + IntervalExpression, + IsNullExpression, + LambdaExpression, + Literal, + MemberAccessExpression, + Placeholder, + ScalarSubqueryExpression, + SubscriptExpression, + TupleExpression, + UnaryExpression + { + SourceSpan span(); + } + + public record CaseExpression(Optional operand, List whenClauses, Optional defaultValue, SourceSpan span) + implements Expression + { + public CaseExpression + { + operand = requireNonNull(operand, "operand is null"); + whenClauses = List.copyOf(requireNonNull(whenClauses, "whenClauses is null")); + if (whenClauses.isEmpty()) { + throw new IllegalArgumentException("whenClauses is empty"); + } + defaultValue = requireNonNull(defaultValue, "defaultValue is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record CaseWhen(Expression operand, Expression result, SourceSpan span) + { + public CaseWhen + { + operand = requireNonNull(operand, "operand is null"); + result = requireNonNull(result, "result is null"); + span = requireNonNull(span, "span is null"); + } + } + + public enum CastTypeDialect + { + HOGQL, + TRINO, + } + + public record CastExpression(Expression value, Identifier type, boolean safe, CastTypeDialect typeDialect, SourceSpan span) + implements Expression + { + public CastExpression + { + value = requireNonNull(value, "value is null"); + type = requireNonNull(type, "type is null"); + typeDialect = requireNonNull(typeDialect, "typeDialect is null"); + span = requireNonNull(span, "span is null"); + } + + public CastExpression(Expression value, Identifier type, boolean safe, SourceSpan span) + { + this(value, type, safe, CastTypeDialect.TRINO, span); + } + } + + public enum IntervalUnit + { + SECOND, + MINUTE, + HOUR, + DAY, + WEEK, + MONTH, + QUARTER, + YEAR, + } + + public record IntervalExpression(Expression value, IntervalUnit unit, SourceSpan span) + implements Expression + { + public IntervalExpression + { + value = requireNonNull(value, "value is null"); + unit = requireNonNull(unit, "unit is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record LambdaExpression(List arguments, Expression body, SourceSpan span) + implements Expression + { + public LambdaExpression + { + arguments = List.copyOf(requireNonNull(arguments, "arguments is null")); + body = requireNonNull(body, "body is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record ArrayExpression(List values, SourceSpan span) + implements Expression + { + public ArrayExpression + { + values = List.copyOf(requireNonNull(values, "values is null")); + span = requireNonNull(span, "span is null"); + } + } + + public record TupleExpression(List values, SourceSpan span) + implements Expression + { + public TupleExpression + { + values = List.copyOf(requireNonNull(values, "values is null")); + if (values.isEmpty()) { + throw new IllegalArgumentException("values is empty"); + } + span = requireNonNull(span, "span is null"); + } + } + + public record SubscriptExpression(Expression base, Expression index, SourceSpan span) + implements Expression + { + public SubscriptExpression + { + base = requireNonNull(base, "base is null"); + index = requireNonNull(index, "index is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record MemberAccessExpression(Expression base, Identifier member, SourceSpan span) + implements Expression + { + public MemberAccessExpression + { + base = requireNonNull(base, "base is null"); + member = requireNonNull(member, "member is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record BetweenExpression(Expression value, Expression min, Expression max, boolean negated, SourceSpan predicateSpan, SourceSpan span) + implements Expression + { + public BetweenExpression + { + value = requireNonNull(value, "value is null"); + min = requireNonNull(min, "min is null"); + max = requireNonNull(max, "max is null"); + predicateSpan = requireNonNull(predicateSpan, "predicateSpan is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record InExpression(Expression value, List values, boolean negated, SourceSpan predicateSpan, SourceSpan span) + implements Expression + { + public InExpression + { + value = requireNonNull(value, "value is null"); + values = List.copyOf(requireNonNull(values, "values is null")); + predicateSpan = requireNonNull(predicateSpan, "predicateSpan is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record InCohortExpression(Expression value, Expression cohort, boolean negated, SourceSpan predicateSpan, SourceSpan span) + implements Expression + { + public InCohortExpression + { + value = requireNonNull(value, "value is null"); + cohort = requireNonNull(cohort, "cohort is null"); + predicateSpan = requireNonNull(predicateSpan, "predicateSpan is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record InSubqueryExpression(Expression value, HogQlQuery query, boolean negated, SourceSpan predicateSpan, SourceSpan span) + implements Expression + { + public InSubqueryExpression + { + value = requireNonNull(value, "value is null"); + query = requireNonNull(query, "query is null"); + predicateSpan = requireNonNull(predicateSpan, "predicateSpan is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record ScalarSubqueryExpression(HogQlQuery query, SourceSpan span) + implements Expression + { + public ScalarSubqueryExpression + { + query = requireNonNull(query, "query is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record IsNullExpression(Expression value, boolean negated, SourceSpan predicateSpan, SourceSpan span) + implements Expression + { + public IsNullExpression + { + value = requireNonNull(value, "value is null"); + predicateSpan = requireNonNull(predicateSpan, "predicateSpan is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record UnaryExpression(UnaryOperator operator, Expression operand, SourceSpan span) + implements Expression + { + public UnaryExpression + { + operator = requireNonNull(operator, "operator is null"); + operand = requireNonNull(operand, "operand is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record BinaryExpression(BinaryOperator operator, Expression left, Expression right, SourceSpan span) + implements Expression + { + public BinaryExpression + { + operator = requireNonNull(operator, "operator is null"); + left = requireNonNull(left, "left is null"); + right = requireNonNull(right, "right is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record FunctionCall( + List nameParts, + List arguments, + boolean distinct, + List orderBy, + Optional filter, + Optional nullTreatment, + Optional window, + SourceSpan span) + implements Expression + { + public FunctionCall + { + nameParts = List.copyOf(requireNonNull(nameParts, "nameParts is null")); + if (nameParts.isEmpty()) { + throw new IllegalArgumentException("nameParts is empty"); + } + arguments = List.copyOf(requireNonNull(arguments, "arguments is null")); + orderBy = List.copyOf(requireNonNull(orderBy, "orderBy is null")); + filter = requireNonNull(filter, "filter is null"); + nullTreatment = requireNonNull(nullTreatment, "nullTreatment is null"); + window = requireNonNull(window, "window is null"); + span = requireNonNull(span, "span is null"); + } + + public FunctionCall( + List nameParts, + List arguments, + boolean distinct, + List orderBy, + Optional filter, + SourceSpan span) + { + this(nameParts, arguments, distinct, orderBy, filter, Optional.empty(), Optional.empty(), span); + } + + public FunctionCall( + Identifier name, + List arguments, + boolean distinct, + List orderBy, + Optional filter, + SourceSpan span) + { + this(List.of(name), arguments, distinct, orderBy, filter, Optional.empty(), Optional.empty(), span); + } + + public Identifier name() + { + return nameParts.getLast(); + } + } + + public enum NullTreatment + { + IGNORE + } + + public record WindowDefinition(Identifier name, WindowSpecification specification, SourceSpan span) + { + public WindowDefinition + { + name = requireNonNull(name, "name is null"); + specification = requireNonNull(specification, "specification is null"); + span = requireNonNull(span, "span is null"); + } + } + + public sealed interface Window + permits WindowReference, WindowSpecification + { + SourceSpan span(); + } + + public record WindowReference(Identifier name, SourceSpan span) + implements Window + { + public WindowReference + { + name = requireNonNull(name, "name is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record WindowSpecification( + List partitionBy, + List orderBy, + Optional frame, + SourceSpan span) + implements Window + { + public WindowSpecification + { + partitionBy = List.copyOf(requireNonNull(partitionBy, "partitionBy is null")); + orderBy = List.copyOf(requireNonNull(orderBy, "orderBy is null")); + frame = requireNonNull(frame, "frame is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record WindowFrame(FrameType type, FrameBound start, Optional end, SourceSpan span) + { + public WindowFrame + { + type = requireNonNull(type, "type is null"); + start = requireNonNull(start, "start is null"); + end = requireNonNull(end, "end is null"); + span = requireNonNull(span, "span is null"); + } + } + + public enum FrameType + { + RANGE, + ROWS, + } + + public record FrameBound(FrameBoundType type, Optional value, SourceSpan span) + { + public FrameBound + { + type = requireNonNull(type, "type is null"); + value = requireNonNull(value, "value is null"); + span = requireNonNull(span, "span is null"); + boolean valueRequired = type == FrameBoundType.PRECEDING || type == FrameBoundType.FOLLOWING; + if (value.isPresent() != valueRequired) { + throw new IllegalArgumentException("window frame bound value does not match bound type"); + } + } + } + + public enum FrameBoundType + { + CURRENT_ROW, + FOLLOWING, + PRECEDING, + UNBOUNDED_FOLLOWING, + UNBOUNDED_PRECEDING, + } + + public enum UnaryOperator + { + NEGATE, + NOT, + POSITIVE, + } + + public enum BinaryOperator + { + ADD, + AND, + CONCAT, + DIVIDE, + EQUAL, + GREATER_THAN, + GREATER_THAN_OR_EQUAL, + ILIKE, + LESS_THAN, + LESS_THAN_OR_EQUAL, + LIKE, + MODULO, + MULTIPLY, + NOT_ILIKE, + NOT_LIKE, + NOT_EQUAL, + OR, + SUBTRACT, + } + + public record ColumnReference(List parts, SourceSpan span) + implements Expression + { + public ColumnReference + { + parts = List.copyOf(requireNonNull(parts, "parts is null")); + if (parts.isEmpty()) { + throw new IllegalArgumentException("parts is empty"); + } + span = requireNonNull(span, "span is null"); + } + } + + public record Placeholder(String name, SourceSpan span) + implements Expression + { + public Placeholder + { + name = requireNonNull(name, "name is null"); + if (name.isEmpty()) { + throw new IllegalArgumentException("name is empty"); + } + span = requireNonNull(span, "span is null"); + } + } + + public record Literal(LiteralKind kind, String value, SourceSpan span) + implements Expression + { + public Literal + { + kind = requireNonNull(kind, "kind is null"); + value = requireNonNull(value, "value is null"); + span = requireNonNull(span, "span is null"); + } + } + + public enum LiteralKind + { + BOOLEAN, + FLOAT, + INTEGER, + NULL, + STRING, + } + + public sealed interface Relation + permits AliasedRelation, + CommonTableReference, + JoinRelation, + PivotRelation, + SubqueryRelation, + TablePlaceholder, + TableReference, + UnnestRelation, + ValuesRelation + { + SourceSpan span(); + } + + public record AliasedRelation(Relation relation, Identifier alias, List columnAliases, SourceSpan span) + implements Relation + { + public AliasedRelation(Relation relation, Identifier alias, SourceSpan span) + { + this(relation, alias, List.of(), span); + } + + public AliasedRelation + { + relation = requireNonNull(relation, "relation is null"); + alias = requireNonNull(alias, "alias is null"); + columnAliases = List.copyOf(requireNonNull(columnAliases, "columnAliases is null")); + span = requireNonNull(span, "span is null"); + } + } + + public record UnnestRelation(List expressions, Identifier alias, List columnAliases, SourceSpan span) + implements Relation + { + public UnnestRelation + { + expressions = List.copyOf(requireNonNull(expressions, "expressions is null")); + if (expressions.isEmpty()) { + throw new IllegalArgumentException("expressions is empty"); + } + alias = requireNonNull(alias, "alias is null"); + columnAliases = List.copyOf(requireNonNull(columnAliases, "columnAliases is null")); + if (expressions.size() != columnAliases.size()) { + throw new IllegalArgumentException("expressions and columnAliases sizes differ"); + } + span = requireNonNull(span, "span is null"); + } + } + + public record CommonTableReference(Identifier name, SourceSpan span) + implements Relation + { + public CommonTableReference + { + name = requireNonNull(name, "name is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record SubqueryRelation(HogQlQuery query, SourceSpan span) + implements Relation + { + public SubqueryRelation + { + query = requireNonNull(query, "query is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record ValuesRelation(List> rows, SourceSpan span) + implements Relation + { + public ValuesRelation + { + rows = requireNonNull(rows, "rows is null").stream() + .map(row -> List.copyOf(requireNonNull(row, "row is null"))) + .toList(); + if (rows.isEmpty()) { + throw new IllegalArgumentException("rows is empty"); + } + int columnCount = rows.getFirst().size(); + if (columnCount == 0 || rows.stream().anyMatch(row -> row.size() != columnCount)) { + throw new IllegalArgumentException("VALUES rows must have the same non-zero column count"); + } + span = requireNonNull(span, "span is null"); + } + + public int columnCount() + { + return rows.getFirst().size(); + } + } + + public enum JoinType + { + CROSS, + INNER, + INNER_ANY, + LEFT, + LEFT_ANY, + RIGHT, + FULL, + } + + public sealed interface JoinCriteria + permits JoinOn, JoinUsing + { + SourceSpan span(); + } + + public record JoinOn(Expression expression, SourceSpan span) + implements JoinCriteria + { + public JoinOn + { + expression = requireNonNull(expression, "expression is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record JoinUsing(List columns, SourceSpan span) + implements JoinCriteria + { + public JoinUsing + { + columns = List.copyOf(requireNonNull(columns, "columns is null")); + if (columns.isEmpty()) { + throw new IllegalArgumentException("columns is empty"); + } + span = requireNonNull(span, "span is null"); + } + } + + public record JoinRelation(JoinType type, Relation left, Relation right, Optional criteria, SourceSpan span) + implements Relation + { + public JoinRelation + { + type = requireNonNull(type, "type is null"); + left = requireNonNull(left, "left is null"); + right = requireNonNull(right, "right is null"); + criteria = requireNonNull(criteria, "criteria is null"); + span = requireNonNull(span, "span is null"); + if ((type == JoinType.CROSS) == criteria.isPresent()) { + throw new IllegalArgumentException("cross joins must omit criteria and qualified joins must provide criteria"); + } + } + } + + public record PivotRelation( + Relation input, + List aggregations, + List pivotColumns, + List valueGroups, + List groupBy, + SourceSpan span) + implements Relation + { + public PivotRelation + { + input = requireNonNull(input, "input is null"); + aggregations = List.copyOf(requireNonNull(aggregations, "aggregations is null")); + pivotColumns = List.copyOf(requireNonNull(pivotColumns, "pivotColumns is null")); + valueGroups = List.copyOf(requireNonNull(valueGroups, "valueGroups is null")); + groupBy = List.copyOf(requireNonNull(groupBy, "groupBy is null")); + span = requireNonNull(span, "span is null"); + if (aggregations.isEmpty()) { + throw new IllegalArgumentException("aggregations is empty"); + } + if (pivotColumns.isEmpty()) { + throw new IllegalArgumentException("pivotColumns is empty"); + } + if (valueGroups.isEmpty()) { + throw new IllegalArgumentException("valueGroups is empty"); + } + } + } + + public record PivotAggregation(Expression expression, Optional alias, SourceSpan span) + { + public PivotAggregation + { + expression = requireNonNull(expression, "expression is null"); + alias = requireNonNull(alias, "alias is null"); + span = requireNonNull(span, "span is null"); + } + } + + public record PivotValueGroup(List values, Optional alias, SourceSpan span) + { + public PivotValueGroup + { + values = List.copyOf(requireNonNull(values, "values is null")); + alias = requireNonNull(alias, "alias is null"); + span = requireNonNull(span, "span is null"); + if (values.isEmpty()) { + throw new IllegalArgumentException("values is empty"); + } + } + } + + public record TableReference(List parts, SourceSpan span) + implements Relation + { + public TableReference + { + parts = List.copyOf(requireNonNull(parts, "parts is null")); + if (parts.isEmpty()) { + throw new IllegalArgumentException("parts is empty"); + } + span = requireNonNull(span, "span is null"); + } + } + + public record TablePlaceholder(Placeholder placeholder) + implements Relation + { + public TablePlaceholder + { + placeholder = requireNonNull(placeholder, "placeholder is null"); + } + + @Override + public SourceSpan span() + { + return placeholder.span(); + } + } + + public record Identifier(String value, boolean delimited, SourceSpan span) + { + public Identifier + { + value = requireNonNull(value, "value is null"); + if (value.isEmpty()) { + throw new IllegalArgumentException("value is empty"); + } + span = requireNonNull(span, "span is null"); + } + } + + public record SourceSpan(int startOffset, int endOffset, int startLine, int startColumn, int endLine, int endColumn) + { + public SourceSpan + { + if (startOffset < 0) { + throw new IllegalArgumentException("startOffset is negative"); + } + if (endOffset < startOffset) { + throw new IllegalArgumentException("endOffset is before startOffset"); + } + if (startLine < 1) { + throw new IllegalArgumentException("startLine must be positive"); + } + if (startColumn < 1) { + throw new IllegalArgumentException("startColumn must be positive"); + } + if (endLine < startLine) { + throw new IllegalArgumentException("endLine is before startLine"); + } + if (endColumn < 1) { + throw new IllegalArgumentException("endColumn must be positive"); + } + } + } +} diff --git a/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlSyntaxTree.java b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlSyntaxTree.java new file mode 100644 index 000000000000..74cdf2516667 --- /dev/null +++ b/core/trino-hogql-parser/src/main/java/io/trino/hogql/parser/tree/HogQlSyntaxTree.java @@ -0,0 +1,70 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser.tree; + +import java.util.List; +import java.util.Optional; + +import static java.util.Objects.requireNonNull; + +public record HogQlSyntaxTree(LanguageClass languageClass, Node root) +{ + public HogQlSyntaxTree + { + languageClass = requireNonNull(languageClass, "languageClass is null"); + root = requireNonNull(root, "root is null"); + } + + public enum LanguageClass + { + READ_ONLY_QUERY, + HOGQLX, + PROCEDURAL, + } + + public sealed interface Element + permits Node, Token + { + HogQlQuery.SourceSpan span(); + } + + public record Node(String rule, Optional alternative, List children, HogQlQuery.SourceSpan span) + implements Element + { + public Node + { + rule = requireNonNull(rule, "rule is null"); + alternative = requireNonNull(alternative, "alternative is null"); + children = List.copyOf(requireNonNull(children, "children is null")); + span = requireNonNull(span, "span is null"); + if (rule.isBlank() || alternative.stream().anyMatch(String::isBlank)) { + throw new IllegalArgumentException("syntax node identity is empty"); + } + } + } + + public record Token(String type, String text, HogQlQuery.SourceSpan span) + implements Element + { + public Token + { + type = requireNonNull(type, "type is null"); + text = requireNonNull(text, "text is null"); + span = requireNonNull(span, "span is null"); + if (type.isBlank()) { + throw new IllegalArgumentException("token type is empty"); + } + } + } +} diff --git a/core/trino-hogql-parser/src/main/resources/META-INF/LICENSE-posthog-hogql.txt b/core/trino-hogql-parser/src/main/resources/META-INF/LICENSE-posthog-hogql.txt new file mode 100644 index 000000000000..46ad1d9ce1ee --- /dev/null +++ b/core/trino-hogql-parser/src/main/resources/META-INF/LICENSE-posthog-hogql.txt @@ -0,0 +1,19 @@ +Copyright (c) 2020-2026 PostHog Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/corpus/expr_select_cases.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/corpus/expr_select_cases.json new file mode 100644 index 000000000000..9b2540ca6161 --- /dev/null +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/corpus/expr_select_cases.json @@ -0,0 +1,210 @@ +{ + "cases": [ + { + "accepted": true, + "category": "arithmetic-expression", + "entryPoint": "expression", + "id": "expr.accept.arithmetic", + "source": "-value * 2 + 1" + }, + { + "accepted": true, + "category": "array-expression", + "entryPoint": "expression", + "id": "expr.accept.array", + "source": "[1, 2, 3]" + }, + { + "accepted": true, + "category": "between-predicate", + "entryPoint": "expression", + "id": "expr.accept.between", + "source": "value BETWEEN 1 AND 10" + }, + { + "accepted": true, + "category": "boolean-expression", + "entryPoint": "expression", + "id": "expr.accept.boolean", + "source": "NOT enabled OR value >= 10" + }, + { + "accepted": true, + "category": "case-expression", + "entryPoint": "expression", + "id": "expr.accept.case", + "source": "CASE WHEN enabled THEN 1 ELSE 0 END" + }, + { + "accepted": true, + "category": "cast-expression", + "entryPoint": "expression", + "id": "expr.accept.cast", + "source": "CAST(value AS INT)" + }, + { + "accepted": true, + "category": "field-reference", + "entryPoint": "expression", + "id": "expr.accept.dotted-field", + "source": "properties.plan" + }, + { + "accepted": true, + "category": "function-call", + "entryPoint": "expression", + "id": "expr.accept.function", + "source": "coalesce(value, 0)" + }, + { + "accepted": true, + "category": "in-predicate", + "entryPoint": "expression", + "id": "expr.accept.in-list", + "source": "value IN (1, 2, 3)" + }, + { + "accepted": true, + "category": "literal-expression", + "entryPoint": "expression", + "id": "expr.accept.literal-boolean", + "source": "true" + }, + { + "accepted": true, + "category": "literal-expression", + "entryPoint": "expression", + "id": "expr.accept.literal-null", + "source": "null" + }, + { + "accepted": true, + "category": "literal-expression", + "entryPoint": "expression", + "id": "expr.accept.literal-number", + "source": "42" + }, + { + "accepted": true, + "category": "literal-expression", + "entryPoint": "expression", + "id": "expr.accept.literal-string", + "source": "'example'" + }, + { + "accepted": true, + "category": "null-predicate", + "entryPoint": "expression", + "id": "expr.accept.null-predicate", + "source": "value IS NULL" + }, + { + "accepted": true, + "category": "tuple-expression", + "entryPoint": "expression", + "id": "expr.accept.tuple", + "source": "(1, 'example')" + }, + { + "accepted": false, + "category": "invalid-expression", + "entryPoint": "expression", + "errorCategory": "incomplete-input", + "id": "expr.reject.incomplete-binary", + "source": "1 +" + }, + { + "accepted": false, + "category": "invalid-expression", + "entryPoint": "expression", + "errorCategory": "unclosed-delimiter", + "id": "expr.reject.unclosed-tuple", + "source": "(1, 2" + }, + { + "accepted": false, + "category": "invalid-expression", + "entryPoint": "expression", + "errorCategory": "unexpected-character", + "id": "expr.reject.unexpected-character", + "source": "value @ 1" + }, + { + "accepted": true, + "category": "select-aggregation", + "entryPoint": "query", + "id": "query.accept.aggregate", + "source": "SELECT event, count() AS total FROM events GROUP BY event HAVING total > 1" + }, + { + "accepted": true, + "category": "select-query", + "entryPoint": "query", + "id": "query.accept.constant", + "source": "SELECT 1" + }, + { + "accepted": true, + "category": "select-query", + "entryPoint": "query", + "id": "query.accept.multiline", + "source": "SELECT event,\n value\nFROM events" + }, + { + "accepted": true, + "category": "select-ordering", + "entryPoint": "query", + "id": "query.accept.order-limit-offset", + "source": "SELECT event FROM events ORDER BY event DESC LIMIT 10 OFFSET 2" + }, + { + "accepted": true, + "category": "select-query", + "entryPoint": "query", + "id": "query.accept.projection-from", + "source": "SELECT event, value + 1 AS next_value FROM events" + }, + { + "accepted": true, + "category": "select-filter", + "entryPoint": "query", + "id": "query.accept.where", + "source": "SELECT event FROM events WHERE event = '$pageview'" + }, + { + "accepted": false, + "category": "invalid-query", + "entryPoint": "query", + "errorCategory": "missing-projection", + "id": "query.reject.missing-projection", + "source": "SELECT" + }, + { + "accepted": false, + "category": "invalid-query", + "entryPoint": "query", + "errorCategory": "missing-source", + "id": "query.reject.missing-source", + "source": "SELECT 1 FROM" + }, + { + "accepted": false, + "category": "invalid-query", + "entryPoint": "query", + "errorCategory": "missing-predicate", + "id": "query.reject.missing-where-expression", + "source": "SELECT 1 WHERE" + }, + { + "accepted": false, + "category": "invalid-query", + "entryPoint": "query", + "errorCategory": "incomplete-input", + "id": "query.reject.trailing-comparison", + "source": "SELECT 1 FROM events WHERE value =" + } + ], + "provenance": "synthetic-public-safe", + "schemaVersion": 1, + "slice": "expression-and-plain-select" +} diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/corpus/expr_select_cpp_oracle.jsonl b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/corpus/expr_select_cpp_oracle.jsonl new file mode 100644 index 000000000000..1be8c076a4b6 --- /dev/null +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/corpus/expr_select_cpp_oracle.jsonl @@ -0,0 +1,28 @@ +{"accepted":true,"ast":{"end":{"column":14,"line":1,"offset":14},"left":{"end":{"column":10,"line":1,"offset":10},"left":{"end":{"column":6,"line":1,"offset":6},"left":{"node":"Constant","value":0},"node":"ArithmeticOperation","op":"-","right":{"chain":["value"],"end":{"column":6,"line":1,"offset":6},"node":"Field","start":{"column":1,"line":1,"offset":1}},"start":{"column":0,"line":1,"offset":0}},"node":"ArithmeticOperation","op":"*","right":{"end":{"column":10,"line":1,"offset":10},"node":"Constant","start":{"column":9,"line":1,"offset":9},"value":2},"start":{"column":0,"line":1,"offset":0}},"node":"ArithmeticOperation","op":"+","right":{"end":{"column":14,"line":1,"offset":14},"node":"Constant","start":{"column":13,"line":1,"offset":13},"value":1},"start":{"column":0,"line":1,"offset":0}},"category":"arithmetic-expression","entryPoint":"expression","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.accept.arithmetic","languageVersion":"1.0.0","schemaVersion":1,"source":"-value * 2 + 1","span":{"end":{"column":14,"line":1,"offset":14},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"end":{"column":9,"line":1,"offset":9},"exprs":[{"end":{"column":2,"line":1,"offset":2},"node":"Constant","start":{"column":1,"line":1,"offset":1},"value":1},{"end":{"column":5,"line":1,"offset":5},"node":"Constant","start":{"column":4,"line":1,"offset":4},"value":2},{"end":{"column":8,"line":1,"offset":8},"node":"Constant","start":{"column":7,"line":1,"offset":7},"value":3}],"node":"Array","start":{"column":0,"line":1,"offset":0}},"category":"array-expression","entryPoint":"expression","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.accept.array","languageVersion":"1.0.0","schemaVersion":1,"source":"[1, 2, 3]","span":{"end":{"column":9,"line":1,"offset":9},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"end":{"column":22,"line":1,"offset":22},"expr":{"chain":["value"],"end":{"column":5,"line":1,"offset":5},"node":"Field","start":{"column":0,"line":1,"offset":0}},"high":{"end":{"column":22,"line":1,"offset":22},"node":"Constant","start":{"column":20,"line":1,"offset":20},"value":10},"low":{"end":{"column":15,"line":1,"offset":15},"node":"Constant","start":{"column":14,"line":1,"offset":14},"value":1},"negated":false,"node":"BetweenExpr","start":{"column":0,"line":1,"offset":0}},"category":"between-predicate","entryPoint":"expression","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.accept.between","languageVersion":"1.0.0","schemaVersion":1,"source":"value BETWEEN 1 AND 10","span":{"end":{"column":22,"line":1,"offset":22},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"end":{"column":26,"line":1,"offset":26},"exprs":[{"end":{"column":11,"line":1,"offset":11},"expr":{"chain":["enabled"],"end":{"column":11,"line":1,"offset":11},"node":"Field","start":{"column":4,"line":1,"offset":4}},"node":"Not","start":{"column":0,"line":1,"offset":0}},{"end":{"column":26,"line":1,"offset":26},"left":{"chain":["value"],"end":{"column":20,"line":1,"offset":20},"node":"Field","start":{"column":15,"line":1,"offset":15}},"node":"CompareOperation","op":">=","right":{"end":{"column":26,"line":1,"offset":26},"node":"Constant","start":{"column":24,"line":1,"offset":24},"value":10},"start":{"column":15,"line":1,"offset":15}}],"node":"Or","start":{"column":0,"line":1,"offset":0}},"category":"boolean-expression","entryPoint":"expression","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.accept.boolean","languageVersion":"1.0.0","schemaVersion":1,"source":"NOT enabled OR value >= 10","span":{"end":{"column":26,"line":1,"offset":26},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"args":[{"chain":["enabled"],"end":{"column":17,"line":1,"offset":17},"node":"Field","start":{"column":10,"line":1,"offset":10}},{"end":{"column":24,"line":1,"offset":24},"node":"Constant","start":{"column":23,"line":1,"offset":23},"value":1},{"end":{"column":31,"line":1,"offset":31},"node":"Constant","start":{"column":30,"line":1,"offset":30},"value":0}],"end":{"column":35,"line":1,"offset":35},"name":"if","node":"Call","start":{"column":0,"line":1,"offset":0}},"category":"case-expression","entryPoint":"expression","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.accept.case","languageVersion":"1.0.0","schemaVersion":1,"source":"CASE WHEN enabled THEN 1 ELSE 0 END","span":{"end":{"column":35,"line":1,"offset":35},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"end":{"column":18,"line":1,"offset":18},"expr":{"chain":["value"],"end":{"column":10,"line":1,"offset":10},"node":"Field","start":{"column":5,"line":1,"offset":5}},"node":"TypeCast","start":{"column":0,"line":1,"offset":0},"type_name":"int"},"category":"cast-expression","entryPoint":"expression","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.accept.cast","languageVersion":"1.0.0","schemaVersion":1,"source":"CAST(value AS INT)","span":{"end":{"column":18,"line":1,"offset":18},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"chain":["properties","plan"],"end":{"column":15,"line":1,"offset":15},"node":"Field","start":{"column":0,"line":1,"offset":0}},"category":"field-reference","entryPoint":"expression","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.accept.dotted-field","languageVersion":"1.0.0","schemaVersion":1,"source":"properties.plan","span":{"end":{"column":15,"line":1,"offset":15},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"args":[{"chain":["value"],"end":{"column":14,"line":1,"offset":14},"node":"Field","start":{"column":9,"line":1,"offset":9}},{"end":{"column":17,"line":1,"offset":17},"node":"Constant","start":{"column":16,"line":1,"offset":16},"value":0}],"distinct":false,"end":{"column":18,"line":1,"offset":18},"filter_expr":null,"name":"coalesce","node":"Call","order_by":null,"params":null,"start":{"column":0,"line":1,"offset":0}},"category":"function-call","entryPoint":"expression","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.accept.function","languageVersion":"1.0.0","schemaVersion":1,"source":"coalesce(value, 0)","span":{"end":{"column":18,"line":1,"offset":18},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"end":{"column":18,"line":1,"offset":18},"left":{"chain":["value"],"end":{"column":5,"line":1,"offset":5},"node":"Field","start":{"column":0,"line":1,"offset":0}},"node":"CompareOperation","op":"in","right":{"end":{"column":18,"line":1,"offset":18},"exprs":[{"end":{"column":11,"line":1,"offset":11},"node":"Constant","start":{"column":10,"line":1,"offset":10},"value":1},{"end":{"column":14,"line":1,"offset":14},"node":"Constant","start":{"column":13,"line":1,"offset":13},"value":2},{"end":{"column":17,"line":1,"offset":17},"node":"Constant","start":{"column":16,"line":1,"offset":16},"value":3}],"node":"Tuple","start":{"column":9,"line":1,"offset":9}},"start":{"column":0,"line":1,"offset":0}},"category":"in-predicate","entryPoint":"expression","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.accept.in-list","languageVersion":"1.0.0","schemaVersion":1,"source":"value IN (1, 2, 3)","span":{"end":{"column":18,"line":1,"offset":18},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"end":{"column":4,"line":1,"offset":4},"node":"Constant","start":{"column":0,"line":1,"offset":0},"value":true},"category":"literal-expression","entryPoint":"expression","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.accept.literal-boolean","languageVersion":"1.0.0","schemaVersion":1,"source":"true","span":{"end":{"column":4,"line":1,"offset":4},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"end":{"column":4,"line":1,"offset":4},"node":"Constant","start":{"column":0,"line":1,"offset":0},"value":null},"category":"literal-expression","entryPoint":"expression","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.accept.literal-null","languageVersion":"1.0.0","schemaVersion":1,"source":"null","span":{"end":{"column":4,"line":1,"offset":4},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"end":{"column":2,"line":1,"offset":2},"node":"Constant","start":{"column":0,"line":1,"offset":0},"value":42},"category":"literal-expression","entryPoint":"expression","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.accept.literal-number","languageVersion":"1.0.0","schemaVersion":1,"source":"42","span":{"end":{"column":2,"line":1,"offset":2},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"end":{"column":9,"line":1,"offset":9},"node":"Constant","start":{"column":0,"line":1,"offset":0},"value":"example"},"category":"literal-expression","entryPoint":"expression","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.accept.literal-string","languageVersion":"1.0.0","schemaVersion":1,"source":"'example'","span":{"end":{"column":9,"line":1,"offset":9},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"end":{"column":13,"line":1,"offset":13},"is_null_comparison_style":true,"left":{"chain":["value"],"end":{"column":5,"line":1,"offset":5},"node":"Field","start":{"column":0,"line":1,"offset":0}},"node":"CompareOperation","op":"==","right":{"node":"Constant","value":null},"start":{"column":0,"line":1,"offset":0}},"category":"null-predicate","entryPoint":"expression","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.accept.null-predicate","languageVersion":"1.0.0","schemaVersion":1,"source":"value IS NULL","span":{"end":{"column":13,"line":1,"offset":13},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"end":{"column":14,"line":1,"offset":14},"exprs":[{"end":{"column":2,"line":1,"offset":2},"node":"Constant","start":{"column":1,"line":1,"offset":1},"value":1},{"end":{"column":13,"line":1,"offset":13},"node":"Constant","start":{"column":4,"line":1,"offset":4},"value":"example"}],"node":"Tuple","start":{"column":0,"line":1,"offset":0}},"category":"tuple-expression","entryPoint":"expression","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.accept.tuple","languageVersion":"1.0.0","schemaVersion":1,"source":"(1, 'example')","span":{"end":{"column":14,"line":1,"offset":14},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":false,"category":"invalid-expression","entryPoint":"expression","errorCategory":"incomplete-input","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.reject.incomplete-binary","languageVersion":"1.0.0","schemaVersion":1,"source":"1 +","span":{"end":{"column":3,"line":1,"offset":3},"start":{"column":3,"line":1,"offset":3}}} +{"accepted":false,"category":"invalid-expression","entryPoint":"expression","errorCategory":"unclosed-delimiter","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.reject.unclosed-tuple","languageVersion":"1.0.0","schemaVersion":1,"source":"(1, 2","span":{"end":{"column":5,"line":1,"offset":5},"start":{"column":5,"line":1,"offset":5}}} +{"accepted":false,"category":"invalid-expression","entryPoint":"expression","errorCategory":"unexpected-character","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"expr.reject.unexpected-character","languageVersion":"1.0.0","schemaVersion":1,"source":"value @ 1","span":{"end":{"column":9,"line":1,"offset":9},"start":{"column":6,"line":1,"offset":6}}} +{"accepted":true,"ast":{"ctes":null,"distinct":null,"end":{"column":74,"line":1,"offset":74},"group_by":[{"chain":["event"],"end":{"column":57,"line":1,"offset":57},"node":"Field","start":{"column":52,"line":1,"offset":52}}],"having":{"end":{"column":74,"line":1,"offset":74},"left":{"chain":["total"],"end":{"column":70,"line":1,"offset":70},"node":"Field","start":{"column":65,"line":1,"offset":65}},"node":"CompareOperation","op":">","right":{"end":{"column":74,"line":1,"offset":74},"node":"Constant","start":{"column":73,"line":1,"offset":73},"value":1},"start":{"column":65,"line":1,"offset":65}},"node":"SelectQuery","prewhere":null,"qualify":null,"select":[{"chain":["event"],"end":{"column":12,"line":1,"offset":12},"node":"Field","start":{"column":7,"line":1,"offset":7}},{"alias":"total","end":{"column":30,"line":1,"offset":30},"expr":{"args":[],"distinct":false,"end":{"column":21,"line":1,"offset":21},"filter_expr":null,"name":"count","node":"Call","order_by":null,"params":null,"start":{"column":14,"line":1,"offset":14}},"node":"Alias","start":{"column":14,"line":1,"offset":14}}],"select_from":{"alias":null,"end":{"column":42,"line":1,"offset":42},"next_join":null,"node":"JoinExpr","sample":null,"start":{"column":36,"line":1,"offset":36},"table":{"chain":["events"],"end":{"column":42,"line":1,"offset":42},"node":"Field","start":{"column":36,"line":1,"offset":36}},"table_final":null},"start":{"column":0,"line":1,"offset":0},"where":null},"category":"select-aggregation","entryPoint":"query","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"query.accept.aggregate","languageVersion":"1.0.0","schemaVersion":1,"source":"SELECT event, count() AS total FROM events GROUP BY event HAVING total > 1","span":{"end":{"column":74,"line":1,"offset":74},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"ctes":null,"distinct":null,"end":{"column":8,"line":1,"offset":8},"group_by":null,"having":null,"node":"SelectQuery","prewhere":null,"qualify":null,"select":[{"end":{"column":8,"line":1,"offset":8},"node":"Constant","start":{"column":7,"line":1,"offset":7},"value":1}],"select_from":null,"start":{"column":0,"line":1,"offset":0},"where":null},"category":"select-query","entryPoint":"query","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"query.accept.constant","languageVersion":"1.0.0","schemaVersion":1,"source":"SELECT 1","span":{"end":{"column":8,"line":1,"offset":8},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"ctes":null,"distinct":null,"end":{"column":11,"line":3,"offset":35},"group_by":null,"having":null,"node":"SelectQuery","prewhere":null,"qualify":null,"select":[{"chain":["event"],"end":{"column":12,"line":1,"offset":12},"node":"Field","start":{"column":7,"line":1,"offset":7}},{"chain":["value"],"end":{"column":9,"line":2,"offset":23},"node":"Field","start":{"column":4,"line":2,"offset":18}}],"select_from":{"alias":null,"end":{"column":11,"line":3,"offset":35},"next_join":null,"node":"JoinExpr","sample":null,"start":{"column":5,"line":3,"offset":29},"table":{"chain":["events"],"end":{"column":11,"line":3,"offset":35},"node":"Field","start":{"column":5,"line":3,"offset":29}},"table_final":null},"start":{"column":0,"line":1,"offset":0},"where":null},"category":"select-query","entryPoint":"query","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"query.accept.multiline","languageVersion":"1.0.0","schemaVersion":1,"source":"SELECT event,\n value\nFROM events","span":{"end":{"column":11,"line":3,"offset":35},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"ctes":null,"distinct":null,"end":{"column":53,"line":1,"offset":53},"group_by":null,"having":null,"limit":{"end":{"column":53,"line":1,"offset":53},"node":"Constant","start":{"column":51,"line":1,"offset":51},"value":10},"node":"SelectQuery","offset":{"end":{"column":62,"line":1,"offset":62},"node":"Constant","start":{"column":61,"line":1,"offset":61},"value":2},"order_by":[{"end":{"column":44,"line":1,"offset":44},"expr":{"chain":["event"],"end":{"column":39,"line":1,"offset":39},"node":"Field","start":{"column":34,"line":1,"offset":34}},"node":"OrderExpr","order":"DESC","start":{"column":34,"line":1,"offset":34}}],"prewhere":null,"qualify":null,"select":[{"chain":["event"],"end":{"column":12,"line":1,"offset":12},"node":"Field","start":{"column":7,"line":1,"offset":7}}],"select_from":{"alias":null,"end":{"column":24,"line":1,"offset":24},"next_join":null,"node":"JoinExpr","sample":null,"start":{"column":18,"line":1,"offset":18},"table":{"chain":["events"],"end":{"column":24,"line":1,"offset":24},"node":"Field","start":{"column":18,"line":1,"offset":18}},"table_final":null},"start":{"column":0,"line":1,"offset":0},"where":null},"category":"select-ordering","entryPoint":"query","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"query.accept.order-limit-offset","languageVersion":"1.0.0","schemaVersion":1,"source":"SELECT event FROM events ORDER BY event DESC LIMIT 10 OFFSET 2","span":{"end":{"column":53,"line":1,"offset":53},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"ctes":null,"distinct":null,"end":{"column":49,"line":1,"offset":49},"group_by":null,"having":null,"node":"SelectQuery","prewhere":null,"qualify":null,"select":[{"chain":["event"],"end":{"column":12,"line":1,"offset":12},"node":"Field","start":{"column":7,"line":1,"offset":7}},{"alias":"next_value","end":{"column":37,"line":1,"offset":37},"expr":{"end":{"column":23,"line":1,"offset":23},"left":{"chain":["value"],"end":{"column":19,"line":1,"offset":19},"node":"Field","start":{"column":14,"line":1,"offset":14}},"node":"ArithmeticOperation","op":"+","right":{"end":{"column":23,"line":1,"offset":23},"node":"Constant","start":{"column":22,"line":1,"offset":22},"value":1},"start":{"column":14,"line":1,"offset":14}},"node":"Alias","start":{"column":14,"line":1,"offset":14}}],"select_from":{"alias":null,"end":{"column":49,"line":1,"offset":49},"next_join":null,"node":"JoinExpr","sample":null,"start":{"column":43,"line":1,"offset":43},"table":{"chain":["events"],"end":{"column":49,"line":1,"offset":49},"node":"Field","start":{"column":43,"line":1,"offset":43}},"table_final":null},"start":{"column":0,"line":1,"offset":0},"where":null},"category":"select-query","entryPoint":"query","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"query.accept.projection-from","languageVersion":"1.0.0","schemaVersion":1,"source":"SELECT event, value + 1 AS next_value FROM events","span":{"end":{"column":49,"line":1,"offset":49},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":true,"ast":{"ctes":null,"distinct":null,"end":{"column":50,"line":1,"offset":50},"group_by":null,"having":null,"node":"SelectQuery","prewhere":null,"qualify":null,"select":[{"chain":["event"],"end":{"column":12,"line":1,"offset":12},"node":"Field","start":{"column":7,"line":1,"offset":7}}],"select_from":{"alias":null,"end":{"column":24,"line":1,"offset":24},"next_join":null,"node":"JoinExpr","sample":null,"start":{"column":18,"line":1,"offset":18},"table":{"chain":["events"],"end":{"column":24,"line":1,"offset":24},"node":"Field","start":{"column":18,"line":1,"offset":18}},"table_final":null},"start":{"column":0,"line":1,"offset":0},"where":{"end":{"column":50,"line":1,"offset":50},"left":{"chain":["event"],"end":{"column":36,"line":1,"offset":36},"node":"Field","start":{"column":31,"line":1,"offset":31}},"node":"CompareOperation","op":"==","right":{"end":{"column":50,"line":1,"offset":50},"node":"Constant","start":{"column":39,"line":1,"offset":39},"value":"$pageview"},"start":{"column":31,"line":1,"offset":31}}},"category":"select-filter","entryPoint":"query","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"query.accept.where","languageVersion":"1.0.0","schemaVersion":1,"source":"SELECT event FROM events WHERE event = '$pageview'","span":{"end":{"column":50,"line":1,"offset":50},"start":{"column":0,"line":1,"offset":0}}} +{"accepted":false,"category":"invalid-query","entryPoint":"query","errorCategory":"missing-projection","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"query.reject.missing-projection","languageVersion":"1.0.0","schemaVersion":1,"source":"SELECT","span":{"end":{"column":6,"line":1,"offset":6},"start":{"column":6,"line":1,"offset":6}}} +{"accepted":false,"category":"invalid-query","entryPoint":"query","errorCategory":"missing-source","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"query.reject.missing-source","languageVersion":"1.0.0","schemaVersion":1,"source":"SELECT 1 FROM","span":{"end":{"column":13,"line":1,"offset":13},"start":{"column":13,"line":1,"offset":13}}} +{"accepted":false,"category":"invalid-query","entryPoint":"query","errorCategory":"missing-predicate","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"query.reject.missing-where-expression","languageVersion":"1.0.0","schemaVersion":1,"source":"SELECT 1 WHERE","span":{"end":{"column":14,"line":1,"offset":14},"start":{"column":14,"line":1,"offset":14}}} +{"accepted":false,"category":"invalid-query","entryPoint":"query","errorCategory":"incomplete-input","grammarSha256":"c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242","id":"query.reject.trailing-comparison","languageVersion":"1.0.0","schemaVersion":1,"source":"SELECT 1 FROM events WHERE value =","span":{"end":{"column":34,"line":1,"offset":34},"start":{"column":34,"line":1,"offset":34}}} diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar-alternative-identities.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar-alternative-identities.json new file mode 100644 index 000000000000..04296fa8f4bc --- /dev/null +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar-alternative-identities.json @@ -0,0 +1,393 @@ +{ + "grammarSha256": "c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242", + "languageVersion": "1.0.0", + "rules": [ + { + "alternatives": [ + { + "id": "alternative:alias:keyword", + "structuralFingerprint": "165f32605f10e147a97ee2d4f2f93bceeb5f6d6eb98f2ef8727355cb59c05bc3" + }, + { + "id": "alternative:alias:quoted-identifier", + "structuralFingerprint": "0abeb8c8fdc827ac36ae2e683162ce80f95e5072fadbcfccd45390ecc8ad2f67" + }, + { + "id": "alternative:alias:unquoted-identifier", + "structuralFingerprint": "73733fbd5549f93da457208e6512bd7d993c6bcaa6c383faac1b225b157ffce2" + } + ], + "queryReachable": true, + "rule": "alias" + }, + { + "alternatives": [ + { + "id": "alternative:columnExprTupleOrSingle:scalar", + "structuralFingerprint": "802b98aaa0c9404525d83dacb80290db7fdff49d34dba43e0e8c6518536197e6" + }, + { + "id": "alternative:columnExprTupleOrSingle:tuple", + "structuralFingerprint": "2b49f26a20cf05b31e6f7071f13a014eca01e5711bad4fcf3825b08d49e63464" + } + ], + "queryReachable": true, + "rule": "columnExprTupleOrSingle" + }, + { + "alternatives": [ + { + "id": "alternative:columnIdentifier:field-path", + "structuralFingerprint": "a4d34630e93d0f240a41eb30229aec9c6363cb7de0b70ebd303df105b5c1a6e4" + }, + { + "id": "alternative:columnIdentifier:placeholder", + "structuralFingerprint": "ac7187ca5c2d10611ac749649461be8e95b68c896f26c5e03b426f4d45a26531" + } + ], + "queryReachable": true, + "rule": "columnIdentifier" + }, + { + "alternatives": [ + { + "id": "alternative:columnTypeCastIdentifier:interval-keyword", + "structuralFingerprint": "30dfdd38947f7e4f75b790b875db86a7f630c63352fbd145fe885fa1a4fe6a1c" + }, + { + "id": "alternative:columnTypeCastIdentifier:quoted-identifier", + "structuralFingerprint": "0abeb8c8fdc827ac36ae2e683162ce80f95e5072fadbcfccd45390ecc8ad2f67" + }, + { + "id": "alternative:columnTypeCastIdentifier:type-keyword", + "structuralFingerprint": "5d58081b9087f3da068f1ad6b457461b5d8deaae59f21cd53b85fcd1bd1f1331" + }, + { + "id": "alternative:columnTypeCastIdentifier:unquoted-identifier", + "structuralFingerprint": "73733fbd5549f93da457208e6512bd7d993c6bcaa6c383faac1b225b157ffce2" + } + ], + "queryReachable": true, + "rule": "columnTypeCastIdentifier" + }, + { + "alternatives": [ + { + "id": "alternative:declaration:statement", + "structuralFingerprint": "5f99fed7cff39aef459246e59fb6f13a5413ddeea7adad8864398797455986f9" + }, + { + "id": "alternative:declaration:variable", + "structuralFingerprint": "4e7fc741b05bde1862641b814dccef54f6927a80c6d9e349ef7684664d44fd82" + } + ], + "queryReachable": true, + "rule": "declaration" + }, + { + "alternatives": [ + { + "id": "alternative:floatingLiteral:floating-token", + "structuralFingerprint": "c88b9f8fead5722931e73444ea92d55372027c411f4f89618d6e0418310ab819" + }, + { + "id": "alternative:floatingLiteral:leading-decimal-point", + "structuralFingerprint": "5b339918b24e7eb9571fa5c007b3b65a6f4c6d3b9c0562e21dcccdd3fd64cb84" + }, + { + "id": "alternative:floatingLiteral:trailing-decimal-point", + "structuralFingerprint": "304631df88730bc6643859c047b4d5a3414e318bd46efc9830a3b292e1312786" + } + ], + "queryReachable": true, + "rule": "floatingLiteral" + }, + { + "alternatives": [ + { + "id": "alternative:hogqlxChildElement:expression", + "structuralFingerprint": "92f247a454640b4807cf6c5f79cf4869c396255034642c950ed2ff7b256c4894" + }, + { + "id": "alternative:hogqlxChildElement:tag", + "structuralFingerprint": "638198a63464a5654b4202c71ae03a9d8abb39b3f6727aeb46165c1635d4410b" + }, + { + "id": "alternative:hogqlxChildElement:text", + "structuralFingerprint": "1acc0716f32f7f1346f847725d21990fd6861790bdbd91c30054c4a35c912d57" + } + ], + "queryReachable": true, + "rule": "hogqlxChildElement" + }, + { + "alternatives": [ + { + "id": "alternative:hogqlxTagAttribute:boolean", + "structuralFingerprint": "f9ec2f5892a5ee11ecee8b7898937fc268f9b2b758ea9a2b4a295f95b0d8e18e" + }, + { + "id": "alternative:hogqlxTagAttribute:expression", + "structuralFingerprint": "7c8aa67035024acc0a0829432ea042be18da3d8ee08fe8000fb60020e9aacbf1" + }, + { + "id": "alternative:hogqlxTagAttribute:string", + "structuralFingerprint": "88dfc10a32800ba48ebe8400d56499520c4ad6601dcc8e7befad72eed782b50e" + } + ], + "queryReachable": true, + "rule": "hogqlxTagAttribute" + }, + { + "alternatives": [ + { + "id": "alternative:identifier:interval-keyword", + "structuralFingerprint": "30dfdd38947f7e4f75b790b875db86a7f630c63352fbd145fe885fa1a4fe6a1c" + }, + { + "id": "alternative:identifier:keyword", + "structuralFingerprint": "bdc4cccbde824cbcccd1a6102bd60f67ab401a5cc6058526f5ac571269ef6030" + }, + { + "id": "alternative:identifier:quoted", + "structuralFingerprint": "0abeb8c8fdc827ac36ae2e683162ce80f95e5072fadbcfccd45390ecc8ad2f67" + }, + { + "id": "alternative:identifier:unquoted", + "structuralFingerprint": "73733fbd5549f93da457208e6512bd7d993c6bcaa6c383faac1b225b157ffce2" + } + ], + "queryReachable": true, + "rule": "identifier" + }, + { + "alternatives": [ + { + "id": "alternative:implicitAlias:keyword", + "structuralFingerprint": "c7b9f7299ab2aa1875c1ee89b24483887dec9de9834dd720cf174b9e104e9f25" + }, + { + "id": "alternative:implicitAlias:quoted-identifier", + "structuralFingerprint": "0abeb8c8fdc827ac36ae2e683162ce80f95e5072fadbcfccd45390ecc8ad2f67" + }, + { + "id": "alternative:implicitAlias:unquoted-identifier", + "structuralFingerprint": "73733fbd5549f93da457208e6512bd7d993c6bcaa6c383faac1b225b157ffce2" + } + ], + "queryReachable": true, + "rule": "implicitAlias" + }, + { + "alternatives": [ + { + "id": "alternative:joinConstraintClause:on", + "structuralFingerprint": "07ff66fad5bfb8968fb35cc93e06ee8bd49028e1854a50b7a40ecaf87a8a97e4" + }, + { + "id": "alternative:joinConstraintClause:using", + "structuralFingerprint": "2754567ba69ffb7298adf0d55cda11bd4b16ec0f9ed66692882b6f958e008327" + }, + { + "id": "alternative:joinConstraintClause:using-parenthesized", + "structuralFingerprint": "c53b790c3e3f8f17d08669249304555a30541a4b37868d86043a794e5eff94e9" + } + ], + "queryReachable": true, + "rule": "joinConstraintClause" + }, + { + "alternatives": [ + { + "id": "alternative:joinOpCross:comma", + "structuralFingerprint": "bcf9583b8a06a904da0c0ea03e289809ead51e2cda7d4084edc1ce18c4f2e025" + }, + { + "id": "alternative:joinOpCross:cross-join", + "structuralFingerprint": "21eefbaad9b940960d7c860ca5bac0ab771e7f71f33b5dc9b9fb9250b4e7be95" + } + ], + "queryReachable": true, + "rule": "joinOpCross" + }, + { + "alternatives": [ + { + "id": "alternative:limitAndOffsetClause:compact", + "structuralFingerprint": "8766cba8afea27bbf1418b00176a7981db472ab89a25675f1a415461a19d9fea" + }, + { + "id": "alternative:limitAndOffsetClause:with-offset", + "structuralFingerprint": "b08e7c7ac1035886c8d50c94cac6b85ed8d5b8a169512b4d3601b87209495c49" + } + ], + "queryReachable": true, + "rule": "limitAndOffsetClause" + }, + { + "alternatives": [ + { + "id": "alternative:limitAndOffsetClauseOptional:limit", + "structuralFingerprint": "8766cba8afea27bbf1418b00176a7981db472ab89a25675f1a415461a19d9fea" + }, + { + "id": "alternative:limitAndOffsetClauseOptional:limit-with-offset", + "structuralFingerprint": "b08e7c7ac1035886c8d50c94cac6b85ed8d5b8a169512b4d3601b87209495c49" + }, + { + "id": "alternative:limitAndOffsetClauseOptional:offset-only", + "structuralFingerprint": "3d9f0279272167d83cd5f876e7cf17e262cb676205fb7bd28f6e570596f0e383" + } + ], + "queryReachable": true, + "rule": "limitAndOffsetClauseOptional" + }, + { + "alternatives": [ + { + "id": "alternative:literal:null", + "structuralFingerprint": "7da720bab69a8562e12edf2c66b42136fbcc198035a585ca5a893d200e5b677a" + }, + { + "id": "alternative:literal:number", + "structuralFingerprint": "56c96aa898d8b78e683c7c140904730921bb5c67201b2be9dc07a804535aba3a" + }, + { + "id": "alternative:literal:string", + "structuralFingerprint": "aab18d5f0771f8fd97a25ffd2112e667e894acf92a430b02dcd8be9522140e85" + } + ], + "queryReachable": true, + "rule": "literal" + }, + { + "alternatives": [ + { + "id": "alternative:ratioExpr:numeric", + "structuralFingerprint": "77d0012bf666d0471f0c5fd47f2f749b3654961ab020288c8b5e524d218bc0a0" + }, + { + "id": "alternative:ratioExpr:placeholder", + "structuralFingerprint": "ac7187ca5c2d10611ac749649461be8e95b68c896f26c5e03b426f4d45a26531" + } + ], + "queryReachable": true, + "rule": "ratioExpr" + }, + { + "alternatives": [ + { + "id": "alternative:selectStmtWithParens:direct", + "structuralFingerprint": "d5e2b3fcc7a599513aa9c07f0a080bc96138881bda01b9885f451cdc4f47c9f1" + }, + { + "id": "alternative:selectStmtWithParens:parenthesized", + "structuralFingerprint": "a70746a6394bdd78e9a2bf2f3b95085009b3c95808b9e2d491c1507129fa1766" + }, + { + "id": "alternative:selectStmtWithParens:placeholder", + "structuralFingerprint": "ac7187ca5c2d10611ac749649461be8e95b68c896f26c5e03b426f4d45a26531" + }, + { + "id": "alternative:selectStmtWithParens:with-parenthesized", + "structuralFingerprint": "e459d96cb925b44a2cc38e03a5dffa2719f59eada2bf788ed797a41540efbfc4" + } + ], + "queryReachable": true, + "rule": "selectStmtWithParens" + }, + { + "alternatives": [ + { + "id": "alternative:statement:block", + "structuralFingerprint": "5acb58cd596f9ea97bc696bddd4f1a5113266af83e22c8f076e96eb5a25fd9bb" + }, + { + "id": "alternative:statement:empty", + "structuralFingerprint": "1d7ce4f5e96efc68d3a3d0388001437e449425f07fbdb3a9bbf02289e6055489" + }, + { + "id": "alternative:statement:expression", + "structuralFingerprint": "792d983175ddbb24989b35af49d0c512d46945699df2423920d79d6a84749f6f" + }, + { + "id": "alternative:statement:for", + "structuralFingerprint": "e70251237e3fd8baf257e1e5e3d675d79bbbbb6ed301885637730065d8f15a58" + }, + { + "id": "alternative:statement:for-in", + "structuralFingerprint": "8d49fe14f87bfe8baf981431e57d79f2c92516fdba0f2cdef472969ab5735d35" + }, + { + "id": "alternative:statement:function", + "structuralFingerprint": "d319855a6ecd0c15fbef3322531043be6aec21545e25f2a4e32392a3e61f0453" + }, + { + "id": "alternative:statement:if", + "structuralFingerprint": "912215ae9664d6d31e8d5102cabc877b5ebbfe6adac5e3261bff90f20475947c" + }, + { + "id": "alternative:statement:return", + "structuralFingerprint": "24012c3b96984fd79cafc9d9a956d947e7ad6fdb21c1f3a2cbc968f80c141cfa" + }, + { + "id": "alternative:statement:throw", + "structuralFingerprint": "85bf0d409a8909ef496fb3219b6d2e38ae9c3c6906953855c9326fcd00f1985a" + }, + { + "id": "alternative:statement:try-catch", + "structuralFingerprint": "fa0c783e64388c6f9a5b32852170481d7b837dfea973e3e37cdc131b7f912f80" + }, + { + "id": "alternative:statement:while", + "structuralFingerprint": "4ce361ee94a08639a64c93c45a3b6bdef28723e71ab7b4c5aaba60698b571477" + } + ], + "queryReachable": true, + "rule": "statement" + }, + { + "alternatives": [ + { + "id": "alternative:string:literal", + "structuralFingerprint": "aab18d5f0771f8fd97a25ffd2112e667e894acf92a430b02dcd8be9522140e85" + }, + { + "id": "alternative:string:template", + "structuralFingerprint": "a16ebcac7c3b3a8da8a3d035c74039354e90500773df3c4c2067bd96d5d43f9b" + } + ], + "queryReachable": true, + "rule": "string" + }, + { + "alternatives": [ + { + "id": "alternative:stringContents:interpolation", + "structuralFingerprint": "c162361fa3590d43840dae422ae3f80c406018732b902d01b5188d263b9dea29" + }, + { + "id": "alternative:stringContents:text", + "structuralFingerprint": "18cf885a4e28d578222b6878f728ac5f9065790527e565818e2a87a761b6bfdf" + } + ], + "queryReachable": true, + "rule": "stringContents" + }, + { + "alternatives": [ + { + "id": "alternative:stringContentsFull:interpolation", + "structuralFingerprint": "df68917607042cf2b91ee6c6f141fe1fcb4d578d2bb356001f5caec3c898bf2f" + }, + { + "id": "alternative:stringContentsFull:text", + "structuralFingerprint": "1abe294c80a61ee18740150362bdafcbb0b7a344d69860589932df63cb1ccab8" + } + ], + "queryReachable": false, + "rule": "stringContentsFull" + } + ], + "schemaVersion": 1 +} diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar-features.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar-features.json new file mode 100644 index 000000000000..11180dae076e --- /dev/null +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar-features.json @@ -0,0 +1,3352 @@ +{ + "features": [ + { + "id": "alternative:arrayJoinClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "arrayJoinClause" + }, + { + "id": "alternative:block", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "block" + }, + { + "id": "alternative:catchBlock", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "catchBlock" + }, + { + "id": "alternative:columnAliases", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "columnAliases" + }, + { + "id": "alternative:columnExpr:ColumnExprAlias", + "kind": "parserAlternative", + "label": "ColumnExprAlias", + "queryReachable": true, + "rule": "columnExpr" + }, + { + "id": "alternative:columnExpr:ColumnExprAnd", + "kind": "parserAlternative", + "label": "ColumnExprAnd", + "queryReachable": true, + "rule": "columnExpr" + }, + { + "id": "alternative:columnExpr:ColumnExprOr", + "kind": "parserAlternative", + "label": "ColumnExprOr", + "queryReachable": true, + "rule": "columnExpr" + }, + { + "id": "alternative:columnExpr:ColumnExprTernaryOp", + "kind": "parserAlternative", + "label": "ColumnExprTernaryOp", + "queryReachable": true, + "rule": "columnExpr" + }, + { + "id": "alternative:columnExpr:ColumnExprValuePassthrough", + "kind": "parserAlternative", + "label": "ColumnExprValuePassthrough", + "queryReachable": true, + "rule": "columnExpr" + }, + { + "id": "alternative:columnExprList", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "columnExprList" + }, + { + "id": "alternative:columnExprValue:ColumnExprArray", + "kind": "parserAlternative", + "label": "ColumnExprArray", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprArrayAccess", + "kind": "parserAlternative", + "label": "ColumnExprArrayAccess", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprArraySlice", + "kind": "parserAlternative", + "label": "ColumnExprArraySlice", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprAsterisk", + "kind": "parserAlternative", + "label": "ColumnExprAsterisk", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprBetween", + "kind": "parserAlternative", + "label": "ColumnExprBetween", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprCall", + "kind": "parserAlternative", + "label": "ColumnExprCall", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprCallSelect", + "kind": "parserAlternative", + "label": "ColumnExprCallSelect", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprCase", + "kind": "parserAlternative", + "label": "ColumnExprCase", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprCast", + "kind": "parserAlternative", + "label": "ColumnExprCast", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprColonLambda", + "kind": "parserAlternative", + "label": "ColumnExprColonLambda", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsAll", + "kind": "parserAlternative", + "label": "ColumnExprColumnsAll", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsExclude", + "kind": "parserAlternative", + "label": "ColumnExprColumnsExclude", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsExcludeReplace", + "kind": "parserAlternative", + "label": "ColumnExprColumnsExcludeReplace", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsList", + "kind": "parserAlternative", + "label": "ColumnExprColumnsList", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsQualifiedAll", + "kind": "parserAlternative", + "label": "ColumnExprColumnsQualifiedAll", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsQualifiedExclude", + "kind": "parserAlternative", + "label": "ColumnExprColumnsQualifiedExclude", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsQualifiedExcludeReplace", + "kind": "parserAlternative", + "label": "ColumnExprColumnsQualifiedExcludeReplace", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsQualifiedReplace", + "kind": "parserAlternative", + "label": "ColumnExprColumnsQualifiedReplace", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsRegex", + "kind": "parserAlternative", + "label": "ColumnExprColumnsRegex", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsReplace", + "kind": "parserAlternative", + "label": "ColumnExprColumnsReplace", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprDate", + "kind": "parserAlternative", + "label": "ColumnExprDate", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprDict", + "kind": "parserAlternative", + "label": "ColumnExprDict", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprFunction", + "kind": "parserAlternative", + "label": "ColumnExprFunction", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprFunctionWithinGroup", + "kind": "parserAlternative", + "label": "ColumnExprFunctionWithinGroup", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprIdentifier", + "kind": "parserAlternative", + "label": "ColumnExprIdentifier", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprIgnoreNulls", + "kind": "parserAlternative", + "label": "ColumnExprIgnoreNulls", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprInterval", + "kind": "parserAlternative", + "label": "ColumnExprInterval", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprIntervalString", + "kind": "parserAlternative", + "label": "ColumnExprIntervalString", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprIsDistinctFrom", + "kind": "parserAlternative", + "label": "ColumnExprIsDistinctFrom", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprIsNull", + "kind": "parserAlternative", + "label": "ColumnExprIsNull", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprLambda", + "kind": "parserAlternative", + "label": "ColumnExprLambda", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprLiteral", + "kind": "parserAlternative", + "label": "ColumnExprLiteral", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprNamedArg", + "kind": "parserAlternative", + "label": "ColumnExprNamedArg", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprNegate", + "kind": "parserAlternative", + "label": "ColumnExprNegate", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprNot", + "kind": "parserAlternative", + "label": "ColumnExprNot", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprNullArrayAccess", + "kind": "parserAlternative", + "label": "ColumnExprNullArrayAccess", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprNullPropertyAccess", + "kind": "parserAlternative", + "label": "ColumnExprNullPropertyAccess", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprNullSafeEq", + "kind": "parserAlternative", + "label": "ColumnExprNullSafeEq", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprNullTupleAccess", + "kind": "parserAlternative", + "label": "ColumnExprNullTupleAccess", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprNullish", + "kind": "parserAlternative", + "label": "ColumnExprNullish", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprParens", + "kind": "parserAlternative", + "label": "ColumnExprParens", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprPositional", + "kind": "parserAlternative", + "label": "ColumnExprPositional", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprPrecedence1", + "kind": "parserAlternative", + "label": "ColumnExprPrecedence1", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprPrecedence2", + "kind": "parserAlternative", + "label": "ColumnExprPrecedence2", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprPrecedence3", + "kind": "parserAlternative", + "label": "ColumnExprPrecedence3", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprPropertyAccess", + "kind": "parserAlternative", + "label": "ColumnExprPropertyAccess", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprSpreadColumnsList", + "kind": "parserAlternative", + "label": "ColumnExprSpreadColumnsList", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprSpreadColumnsRegex", + "kind": "parserAlternative", + "label": "ColumnExprSpreadColumnsRegex", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprSubquery", + "kind": "parserAlternative", + "label": "ColumnExprSubquery", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprSubstring", + "kind": "parserAlternative", + "label": "ColumnExprSubstring", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprTagElement", + "kind": "parserAlternative", + "label": "ColumnExprTagElement", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprTemplateString", + "kind": "parserAlternative", + "label": "ColumnExprTemplateString", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprTimestamp", + "kind": "parserAlternative", + "label": "ColumnExprTimestamp", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprTrim", + "kind": "parserAlternative", + "label": "ColumnExprTrim", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprTryCast", + "kind": "parserAlternative", + "label": "ColumnExprTryCast", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprTuple", + "kind": "parserAlternative", + "label": "ColumnExprTuple", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprTupleAccess", + "kind": "parserAlternative", + "label": "ColumnExprTupleAccess", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprTypeCast", + "kind": "parserAlternative", + "label": "ColumnExprTypeCast", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprWinFunction", + "kind": "parserAlternative", + "label": "ColumnExprWinFunction", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnExprValue:ColumnExprWinFunctionTarget", + "kind": "parserAlternative", + "label": "ColumnExprWinFunctionTarget", + "queryReachable": true, + "rule": "columnExprValue" + }, + { + "id": "alternative:columnLambdaExpr:ArrowLambda", + "kind": "parserAlternative", + "label": "ArrowLambda", + "queryReachable": true, + "rule": "columnLambdaExpr" + }, + { + "id": "alternative:columnLambdaExpr:ColonLambda", + "kind": "parserAlternative", + "label": "ColonLambda", + "queryReachable": true, + "rule": "columnLambdaExpr" + }, + { + "id": "alternative:columnTypeCastExpr:ColumnTypeCastExprSimple", + "kind": "parserAlternative", + "label": "ColumnTypeCastExprSimple", + "queryReachable": true, + "rule": "columnTypeCastExpr" + }, + { + "id": "alternative:columnTypeCastExpr:ColumnTypeCastExprWithTimeZone", + "kind": "parserAlternative", + "label": "ColumnTypeCastExprWithTimeZone", + "queryReachable": true, + "rule": "columnTypeCastExpr" + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprArray", + "kind": "parserAlternative", + "label": "ColumnTypeExprArray", + "queryReachable": true, + "rule": "columnTypeExpr" + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprComplex", + "kind": "parserAlternative", + "label": "ColumnTypeExprComplex", + "queryReachable": true, + "rule": "columnTypeExpr" + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprCompound", + "kind": "parserAlternative", + "label": "ColumnTypeExprCompound", + "queryReachable": true, + "rule": "columnTypeExpr" + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprEnum", + "kind": "parserAlternative", + "label": "ColumnTypeExprEnum", + "queryReachable": true, + "rule": "columnTypeExpr" + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprNested", + "kind": "parserAlternative", + "label": "ColumnTypeExprNested", + "queryReachable": true, + "rule": "columnTypeExpr" + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprParam", + "kind": "parserAlternative", + "label": "ColumnTypeExprParam", + "queryReachable": true, + "rule": "columnTypeExpr" + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprSimple", + "kind": "parserAlternative", + "label": "ColumnTypeExprSimple", + "queryReachable": true, + "rule": "columnTypeExpr" + }, + { + "id": "alternative:columnsReplaceItem", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "columnsReplaceItem" + }, + { + "id": "alternative:columnsReplaceList", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "columnsReplaceList" + }, + { + "id": "alternative:databaseIdentifier", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "databaseIdentifier" + }, + { + "id": "alternative:emptyStmt", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "emptyStmt" + }, + { + "id": "alternative:enumValue", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "enumValue" + }, + { + "id": "alternative:expr", + "kind": "parserAlternative", + "queryReachable": false, + "rule": "expr" + }, + { + "id": "alternative:exprStmt", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "exprStmt" + }, + { + "id": "alternative:expression", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "expression" + }, + { + "id": "alternative:forInStmt", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "forInStmt" + }, + { + "id": "alternative:forStmt", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "forStmt" + }, + { + "id": "alternative:fromClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "fromClause" + }, + { + "id": "alternative:fullTemplateString", + "kind": "parserAlternative", + "queryReachable": false, + "rule": "fullTemplateString" + }, + { + "id": "alternative:funcStmt", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "funcStmt" + }, + { + "id": "alternative:groupByClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "groupByClause" + }, + { + "id": "alternative:groupingSet", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "groupingSet" + }, + { + "id": "alternative:groupingSetList", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "groupingSetList" + }, + { + "id": "alternative:havingClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "havingClause" + }, + { + "id": "alternative:hogqlxTagElement:HogqlxTagElementClosed", + "kind": "parserAlternative", + "label": "HogqlxTagElementClosed", + "queryReachable": true, + "rule": "hogqlxTagElement" + }, + { + "id": "alternative:hogqlxTagElement:HogqlxTagElementNested", + "kind": "parserAlternative", + "label": "HogqlxTagElementNested", + "queryReachable": true, + "rule": "hogqlxTagElement" + }, + { + "id": "alternative:hogqlxText", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "hogqlxText" + }, + { + "id": "alternative:identifierList", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "identifierList" + }, + { + "id": "alternative:ifStmt", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "ifStmt" + }, + { + "id": "alternative:interpolateClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "interpolateClause" + }, + { + "id": "alternative:interpolateExpr", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "interpolateExpr" + }, + { + "id": "alternative:interval", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "interval" + }, + { + "id": "alternative:joinExpr:JoinExprCrossOp", + "kind": "parserAlternative", + "label": "JoinExprCrossOp", + "queryReachable": true, + "rule": "joinExpr" + }, + { + "id": "alternative:joinExpr:JoinExprOp", + "kind": "parserAlternative", + "label": "JoinExprOp", + "queryReachable": true, + "rule": "joinExpr" + }, + { + "id": "alternative:joinExpr:JoinExprParens", + "kind": "parserAlternative", + "label": "JoinExprParens", + "queryReachable": true, + "rule": "joinExpr" + }, + { + "id": "alternative:joinExpr:JoinExprPivot", + "kind": "parserAlternative", + "label": "JoinExprPivot", + "queryReachable": true, + "rule": "joinExpr" + }, + { + "id": "alternative:joinExpr:JoinExprPositional", + "kind": "parserAlternative", + "label": "JoinExprPositional", + "queryReachable": true, + "rule": "joinExpr" + }, + { + "id": "alternative:joinExpr:JoinExprTable", + "kind": "parserAlternative", + "label": "JoinExprTable", + "queryReachable": true, + "rule": "joinExpr" + }, + { + "id": "alternative:joinExpr:JoinExprUnpivot", + "kind": "parserAlternative", + "label": "JoinExprUnpivot", + "queryReachable": true, + "rule": "joinExpr" + }, + { + "id": "alternative:joinOp:JoinOpFull", + "kind": "parserAlternative", + "label": "JoinOpFull", + "queryReachable": true, + "rule": "joinOp" + }, + { + "id": "alternative:joinOp:JoinOpInner", + "kind": "parserAlternative", + "label": "JoinOpInner", + "queryReachable": true, + "rule": "joinOp" + }, + { + "id": "alternative:joinOp:JoinOpLeftRight", + "kind": "parserAlternative", + "label": "JoinOpLeftRight", + "queryReachable": true, + "rule": "joinOp" + }, + { + "id": "alternative:keyword", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "keyword" + }, + { + "id": "alternative:keywordForAlias", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "keywordForAlias" + }, + { + "id": "alternative:keywordForImplicitAlias", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "keywordForImplicitAlias" + }, + { + "id": "alternative:keywordForTypeCast", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "keywordForTypeCast" + }, + { + "id": "alternative:kvPair", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "kvPair" + }, + { + "id": "alternative:kvPairList", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "kvPairList" + }, + { + "id": "alternative:limitByClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "limitByClause" + }, + { + "id": "alternative:limitExpr", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "limitExpr" + }, + { + "id": "alternative:nestedIdentifier", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "nestedIdentifier" + }, + { + "id": "alternative:numberLiteral", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "numberLiteral" + }, + { + "id": "alternative:offsetOnlyClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "offsetOnlyClause" + }, + { + "id": "alternative:orderByClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "orderByClause" + }, + { + "id": "alternative:orderExpr", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "orderExpr" + }, + { + "id": "alternative:orderExprList", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "orderExprList" + }, + { + "id": "alternative:pivotColumn", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "pivotColumn" + }, + { + "id": "alternative:pivotColumnList", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "pivotColumnList" + }, + { + "id": "alternative:placeholder", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "placeholder" + }, + { + "id": "alternative:prewhereClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "prewhereClause" + }, + { + "id": "alternative:program", + "kind": "parserAlternative", + "queryReachable": false, + "rule": "program" + }, + { + "id": "alternative:projectionOrderByClause", + "kind": "parserAlternative", + "queryReachable": false, + "rule": "projectionOrderByClause" + }, + { + "id": "alternative:qualifyClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "qualifyClause" + }, + { + "id": "alternative:returnStmt", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "returnStmt" + }, + { + "id": "alternative:sampleClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "sampleClause" + }, + { + "id": "alternative:select", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "select" + }, + { + "id": "alternative:selectColumnExpr:ColumnExprAliasBefore", + "kind": "parserAlternative", + "label": "ColumnExprAliasBefore", + "queryReachable": true, + "rule": "selectColumnExpr" + }, + { + "id": "alternative:selectColumnExpr:ColumnExprAliasImplicit", + "kind": "parserAlternative", + "label": "ColumnExprAliasImplicit", + "queryReachable": true, + "rule": "selectColumnExpr" + }, + { + "id": "alternative:selectColumnExpr:ColumnExprInvalidFromImplicitAlias", + "kind": "parserAlternative", + "label": "ColumnExprInvalidFromImplicitAlias", + "queryReachable": true, + "rule": "selectColumnExpr" + }, + { + "id": "alternative:selectColumnExpr:ColumnExprSelectValue", + "kind": "parserAlternative", + "label": "ColumnExprSelectValue", + "queryReachable": true, + "rule": "selectColumnExpr" + }, + { + "id": "alternative:selectColumnExprList", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "selectColumnExprList" + }, + { + "id": "alternative:selectColumnExprListBeforeFrom:SelectColumnExprListBeforeFromPlain", + "kind": "parserAlternative", + "label": "SelectColumnExprListBeforeFromPlain", + "queryReachable": true, + "rule": "selectColumnExprListBeforeFrom" + }, + { + "id": "alternative:selectColumnExprListBeforeFrom:SelectColumnExprListBeforeFromTrailingComma", + "kind": "parserAlternative", + "label": "SelectColumnExprListBeforeFromTrailingComma", + "queryReachable": true, + "rule": "selectColumnExprListBeforeFrom" + }, + { + "id": "alternative:selectSetStmt", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "selectSetStmt" + }, + { + "id": "alternative:selectStmt", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "selectStmt" + }, + { + "id": "alternative:settingExpr", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "settingExpr" + }, + { + "id": "alternative:settingExprList", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "settingExprList" + }, + { + "id": "alternative:settingsClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "settingsClause" + }, + { + "id": "alternative:subsequentSelectSetClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "subsequentSelectSetClause" + }, + { + "id": "alternative:tableArgList", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "tableArgList" + }, + { + "id": "alternative:tableExpr:TableExprAlias", + "kind": "parserAlternative", + "label": "TableExprAlias", + "queryReachable": true, + "rule": "tableExpr" + }, + { + "id": "alternative:tableExpr:TableExprFunction", + "kind": "parserAlternative", + "label": "TableExprFunction", + "queryReachable": true, + "rule": "tableExpr" + }, + { + "id": "alternative:tableExpr:TableExprIdentifier", + "kind": "parserAlternative", + "label": "TableExprIdentifier", + "queryReachable": true, + "rule": "tableExpr" + }, + { + "id": "alternative:tableExpr:TableExprPivot", + "kind": "parserAlternative", + "label": "TableExprPivot", + "queryReachable": true, + "rule": "tableExpr" + }, + { + "id": "alternative:tableExpr:TableExprPlaceholder", + "kind": "parserAlternative", + "label": "TableExprPlaceholder", + "queryReachable": true, + "rule": "tableExpr" + }, + { + "id": "alternative:tableExpr:TableExprSubquery", + "kind": "parserAlternative", + "label": "TableExprSubquery", + "queryReachable": true, + "rule": "tableExpr" + }, + { + "id": "alternative:tableExpr:TableExprTag", + "kind": "parserAlternative", + "label": "TableExprTag", + "queryReachable": true, + "rule": "tableExpr" + }, + { + "id": "alternative:tableExpr:TableExprUnpivot", + "kind": "parserAlternative", + "label": "TableExprUnpivot", + "queryReachable": true, + "rule": "tableExpr" + }, + { + "id": "alternative:tableExpr:TableExprValues", + "kind": "parserAlternative", + "label": "TableExprValues", + "queryReachable": true, + "rule": "tableExpr" + }, + { + "id": "alternative:tableFunctionExpr", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "tableFunctionExpr" + }, + { + "id": "alternative:tableIdentifier", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "tableIdentifier" + }, + { + "id": "alternative:templateString", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "templateString" + }, + { + "id": "alternative:throwStmt", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "throwStmt" + }, + { + "id": "alternative:topClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "topClause" + }, + { + "id": "alternative:tryCatchStmt", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "tryCatchStmt" + }, + { + "id": "alternative:unpivotColumn", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "unpivotColumn" + }, + { + "id": "alternative:unpivotColumnList", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "unpivotColumnList" + }, + { + "id": "alternative:valuesClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "valuesClause" + }, + { + "id": "alternative:valuesRow", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "valuesRow" + }, + { + "id": "alternative:varAssignment", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "varAssignment" + }, + { + "id": "alternative:varDecl", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "varDecl" + }, + { + "id": "alternative:whereClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "whereClause" + }, + { + "id": "alternative:whileStmt", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "whileStmt" + }, + { + "id": "alternative:winFrameBound", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "winFrameBound" + }, + { + "id": "alternative:winFrameClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "winFrameClause" + }, + { + "id": "alternative:winFrameExtend:frameBetween", + "kind": "parserAlternative", + "label": "frameBetween", + "queryReachable": true, + "rule": "winFrameExtend" + }, + { + "id": "alternative:winFrameExtend:frameStart", + "kind": "parserAlternative", + "label": "frameStart", + "queryReachable": true, + "rule": "winFrameExtend" + }, + { + "id": "alternative:winOrderByClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "winOrderByClause" + }, + { + "id": "alternative:winPartitionByClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "winPartitionByClause" + }, + { + "id": "alternative:windowClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "windowClause" + }, + { + "id": "alternative:windowExpr", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "windowExpr" + }, + { + "id": "alternative:withClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "withClause" + }, + { + "id": "alternative:withExpr:WithExprColumn", + "kind": "parserAlternative", + "label": "WithExprColumn", + "queryReachable": true, + "rule": "withExpr" + }, + { + "id": "alternative:withExpr:WithExprSubquery", + "kind": "parserAlternative", + "label": "WithExprSubquery", + "queryReachable": true, + "rule": "withExpr" + }, + { + "id": "alternative:withExprColumnNameList", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "withExprColumnNameList" + }, + { + "id": "alternative:withExprList", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "withExprList" + }, + { + "id": "alternative:withFillClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "withFillClause" + }, + { + "id": "alternative:withinGroupClause", + "kind": "parserAlternative", + "queryReachable": true, + "rule": "withinGroupClause" + }, + { + "id": "rule:alias", + "kind": "parserRule", + "name": "alias", + "queryReachable": true + }, + { + "id": "rule:arrayJoinClause", + "kind": "parserRule", + "name": "arrayJoinClause", + "queryReachable": true + }, + { + "id": "rule:block", + "kind": "parserRule", + "name": "block", + "queryReachable": true + }, + { + "id": "rule:catchBlock", + "kind": "parserRule", + "name": "catchBlock", + "queryReachable": true + }, + { + "id": "rule:columnAliases", + "kind": "parserRule", + "name": "columnAliases", + "queryReachable": true + }, + { + "id": "rule:columnExpr", + "kind": "parserRule", + "name": "columnExpr", + "queryReachable": true + }, + { + "id": "rule:columnExprList", + "kind": "parserRule", + "name": "columnExprList", + "queryReachable": true + }, + { + "id": "rule:columnExprTupleOrSingle", + "kind": "parserRule", + "name": "columnExprTupleOrSingle", + "queryReachable": true + }, + { + "id": "rule:columnExprValue", + "kind": "parserRule", + "name": "columnExprValue", + "queryReachable": true + }, + { + "id": "rule:columnIdentifier", + "kind": "parserRule", + "name": "columnIdentifier", + "queryReachable": true + }, + { + "id": "rule:columnLambdaExpr", + "kind": "parserRule", + "name": "columnLambdaExpr", + "queryReachable": true + }, + { + "id": "rule:columnTypeCastExpr", + "kind": "parserRule", + "name": "columnTypeCastExpr", + "queryReachable": true + }, + { + "id": "rule:columnTypeCastIdentifier", + "kind": "parserRule", + "name": "columnTypeCastIdentifier", + "queryReachable": true + }, + { + "id": "rule:columnTypeExpr", + "kind": "parserRule", + "name": "columnTypeExpr", + "queryReachable": true + }, + { + "id": "rule:columnsReplaceItem", + "kind": "parserRule", + "name": "columnsReplaceItem", + "queryReachable": true + }, + { + "id": "rule:columnsReplaceList", + "kind": "parserRule", + "name": "columnsReplaceList", + "queryReachable": true + }, + { + "id": "rule:databaseIdentifier", + "kind": "parserRule", + "name": "databaseIdentifier", + "queryReachable": true + }, + { + "id": "rule:declaration", + "kind": "parserRule", + "name": "declaration", + "queryReachable": true + }, + { + "id": "rule:emptyStmt", + "kind": "parserRule", + "name": "emptyStmt", + "queryReachable": true + }, + { + "id": "rule:enumValue", + "kind": "parserRule", + "name": "enumValue", + "queryReachable": true + }, + { + "id": "rule:expr", + "kind": "parserRule", + "name": "expr", + "queryReachable": false + }, + { + "id": "rule:exprStmt", + "kind": "parserRule", + "name": "exprStmt", + "queryReachable": true + }, + { + "id": "rule:expression", + "kind": "parserRule", + "name": "expression", + "queryReachable": true + }, + { + "id": "rule:floatingLiteral", + "kind": "parserRule", + "name": "floatingLiteral", + "queryReachable": true + }, + { + "id": "rule:forInStmt", + "kind": "parserRule", + "name": "forInStmt", + "queryReachable": true + }, + { + "id": "rule:forStmt", + "kind": "parserRule", + "name": "forStmt", + "queryReachable": true + }, + { + "id": "rule:fromClause", + "kind": "parserRule", + "name": "fromClause", + "queryReachable": true + }, + { + "id": "rule:fullTemplateString", + "kind": "parserRule", + "name": "fullTemplateString", + "queryReachable": false + }, + { + "id": "rule:funcStmt", + "kind": "parserRule", + "name": "funcStmt", + "queryReachable": true + }, + { + "id": "rule:groupByClause", + "kind": "parserRule", + "name": "groupByClause", + "queryReachable": true + }, + { + "id": "rule:groupingSet", + "kind": "parserRule", + "name": "groupingSet", + "queryReachable": true + }, + { + "id": "rule:groupingSetList", + "kind": "parserRule", + "name": "groupingSetList", + "queryReachable": true + }, + { + "id": "rule:havingClause", + "kind": "parserRule", + "name": "havingClause", + "queryReachable": true + }, + { + "id": "rule:hogqlxChildElement", + "kind": "parserRule", + "name": "hogqlxChildElement", + "queryReachable": true + }, + { + "id": "rule:hogqlxTagAttribute", + "kind": "parserRule", + "name": "hogqlxTagAttribute", + "queryReachable": true + }, + { + "id": "rule:hogqlxTagElement", + "kind": "parserRule", + "name": "hogqlxTagElement", + "queryReachable": true + }, + { + "id": "rule:hogqlxText", + "kind": "parserRule", + "name": "hogqlxText", + "queryReachable": true + }, + { + "id": "rule:identifier", + "kind": "parserRule", + "name": "identifier", + "queryReachable": true + }, + { + "id": "rule:identifierList", + "kind": "parserRule", + "name": "identifierList", + "queryReachable": true + }, + { + "id": "rule:ifStmt", + "kind": "parserRule", + "name": "ifStmt", + "queryReachable": true + }, + { + "id": "rule:implicitAlias", + "kind": "parserRule", + "name": "implicitAlias", + "queryReachable": true + }, + { + "id": "rule:interpolateClause", + "kind": "parserRule", + "name": "interpolateClause", + "queryReachable": true + }, + { + "id": "rule:interpolateExpr", + "kind": "parserRule", + "name": "interpolateExpr", + "queryReachable": true + }, + { + "id": "rule:interval", + "kind": "parserRule", + "name": "interval", + "queryReachable": true + }, + { + "id": "rule:joinConstraintClause", + "kind": "parserRule", + "name": "joinConstraintClause", + "queryReachable": true + }, + { + "id": "rule:joinExpr", + "kind": "parserRule", + "name": "joinExpr", + "queryReachable": true + }, + { + "id": "rule:joinOp", + "kind": "parserRule", + "name": "joinOp", + "queryReachable": true + }, + { + "id": "rule:joinOpCross", + "kind": "parserRule", + "name": "joinOpCross", + "queryReachable": true + }, + { + "id": "rule:keyword", + "kind": "parserRule", + "name": "keyword", + "queryReachable": true + }, + { + "id": "rule:keywordForAlias", + "kind": "parserRule", + "name": "keywordForAlias", + "queryReachable": true + }, + { + "id": "rule:keywordForImplicitAlias", + "kind": "parserRule", + "name": "keywordForImplicitAlias", + "queryReachable": true + }, + { + "id": "rule:keywordForTypeCast", + "kind": "parserRule", + "name": "keywordForTypeCast", + "queryReachable": true + }, + { + "id": "rule:kvPair", + "kind": "parserRule", + "name": "kvPair", + "queryReachable": true + }, + { + "id": "rule:kvPairList", + "kind": "parserRule", + "name": "kvPairList", + "queryReachable": true + }, + { + "id": "rule:limitAndOffsetClause", + "kind": "parserRule", + "name": "limitAndOffsetClause", + "queryReachable": true + }, + { + "id": "rule:limitAndOffsetClauseOptional", + "kind": "parserRule", + "name": "limitAndOffsetClauseOptional", + "queryReachable": true + }, + { + "id": "rule:limitByClause", + "kind": "parserRule", + "name": "limitByClause", + "queryReachable": true + }, + { + "id": "rule:limitExpr", + "kind": "parserRule", + "name": "limitExpr", + "queryReachable": true + }, + { + "id": "rule:literal", + "kind": "parserRule", + "name": "literal", + "queryReachable": true + }, + { + "id": "rule:nestedIdentifier", + "kind": "parserRule", + "name": "nestedIdentifier", + "queryReachable": true + }, + { + "id": "rule:numberLiteral", + "kind": "parserRule", + "name": "numberLiteral", + "queryReachable": true + }, + { + "id": "rule:offsetOnlyClause", + "kind": "parserRule", + "name": "offsetOnlyClause", + "queryReachable": true + }, + { + "id": "rule:orderByClause", + "kind": "parserRule", + "name": "orderByClause", + "queryReachable": true + }, + { + "id": "rule:orderExpr", + "kind": "parserRule", + "name": "orderExpr", + "queryReachable": true + }, + { + "id": "rule:orderExprList", + "kind": "parserRule", + "name": "orderExprList", + "queryReachable": true + }, + { + "id": "rule:pivotColumn", + "kind": "parserRule", + "name": "pivotColumn", + "queryReachable": true + }, + { + "id": "rule:pivotColumnList", + "kind": "parserRule", + "name": "pivotColumnList", + "queryReachable": true + }, + { + "id": "rule:placeholder", + "kind": "parserRule", + "name": "placeholder", + "queryReachable": true + }, + { + "id": "rule:prewhereClause", + "kind": "parserRule", + "name": "prewhereClause", + "queryReachable": true + }, + { + "id": "rule:program", + "kind": "parserRule", + "name": "program", + "queryReachable": false + }, + { + "id": "rule:projectionOrderByClause", + "kind": "parserRule", + "name": "projectionOrderByClause", + "queryReachable": false + }, + { + "id": "rule:qualifyClause", + "kind": "parserRule", + "name": "qualifyClause", + "queryReachable": true + }, + { + "id": "rule:ratioExpr", + "kind": "parserRule", + "name": "ratioExpr", + "queryReachable": true + }, + { + "id": "rule:returnStmt", + "kind": "parserRule", + "name": "returnStmt", + "queryReachable": true + }, + { + "id": "rule:sampleClause", + "kind": "parserRule", + "name": "sampleClause", + "queryReachable": true + }, + { + "id": "rule:select", + "kind": "parserRule", + "name": "select", + "queryReachable": true + }, + { + "id": "rule:selectColumnExpr", + "kind": "parserRule", + "name": "selectColumnExpr", + "queryReachable": true + }, + { + "id": "rule:selectColumnExprList", + "kind": "parserRule", + "name": "selectColumnExprList", + "queryReachable": true + }, + { + "id": "rule:selectColumnExprListBeforeFrom", + "kind": "parserRule", + "name": "selectColumnExprListBeforeFrom", + "queryReachable": true + }, + { + "id": "rule:selectSetStmt", + "kind": "parserRule", + "name": "selectSetStmt", + "queryReachable": true + }, + { + "id": "rule:selectStmt", + "kind": "parserRule", + "name": "selectStmt", + "queryReachable": true + }, + { + "id": "rule:selectStmtWithParens", + "kind": "parserRule", + "name": "selectStmtWithParens", + "queryReachable": true + }, + { + "id": "rule:settingExpr", + "kind": "parserRule", + "name": "settingExpr", + "queryReachable": true + }, + { + "id": "rule:settingExprList", + "kind": "parserRule", + "name": "settingExprList", + "queryReachable": true + }, + { + "id": "rule:settingsClause", + "kind": "parserRule", + "name": "settingsClause", + "queryReachable": true + }, + { + "id": "rule:statement", + "kind": "parserRule", + "name": "statement", + "queryReachable": true + }, + { + "id": "rule:string", + "kind": "parserRule", + "name": "string", + "queryReachable": true + }, + { + "id": "rule:stringContents", + "kind": "parserRule", + "name": "stringContents", + "queryReachable": true + }, + { + "id": "rule:stringContentsFull", + "kind": "parserRule", + "name": "stringContentsFull", + "queryReachable": false + }, + { + "id": "rule:subsequentSelectSetClause", + "kind": "parserRule", + "name": "subsequentSelectSetClause", + "queryReachable": true + }, + { + "id": "rule:tableArgList", + "kind": "parserRule", + "name": "tableArgList", + "queryReachable": true + }, + { + "id": "rule:tableExpr", + "kind": "parserRule", + "name": "tableExpr", + "queryReachable": true + }, + { + "id": "rule:tableFunctionExpr", + "kind": "parserRule", + "name": "tableFunctionExpr", + "queryReachable": true + }, + { + "id": "rule:tableIdentifier", + "kind": "parserRule", + "name": "tableIdentifier", + "queryReachable": true + }, + { + "id": "rule:templateString", + "kind": "parserRule", + "name": "templateString", + "queryReachable": true + }, + { + "id": "rule:throwStmt", + "kind": "parserRule", + "name": "throwStmt", + "queryReachable": true + }, + { + "id": "rule:topClause", + "kind": "parserRule", + "name": "topClause", + "queryReachable": true + }, + { + "id": "rule:tryCatchStmt", + "kind": "parserRule", + "name": "tryCatchStmt", + "queryReachable": true + }, + { + "id": "rule:unpivotColumn", + "kind": "parserRule", + "name": "unpivotColumn", + "queryReachable": true + }, + { + "id": "rule:unpivotColumnList", + "kind": "parserRule", + "name": "unpivotColumnList", + "queryReachable": true + }, + { + "id": "rule:valuesClause", + "kind": "parserRule", + "name": "valuesClause", + "queryReachable": true + }, + { + "id": "rule:valuesRow", + "kind": "parserRule", + "name": "valuesRow", + "queryReachable": true + }, + { + "id": "rule:varAssignment", + "kind": "parserRule", + "name": "varAssignment", + "queryReachable": true + }, + { + "id": "rule:varDecl", + "kind": "parserRule", + "name": "varDecl", + "queryReachable": true + }, + { + "id": "rule:whereClause", + "kind": "parserRule", + "name": "whereClause", + "queryReachable": true + }, + { + "id": "rule:whileStmt", + "kind": "parserRule", + "name": "whileStmt", + "queryReachable": true + }, + { + "id": "rule:winFrameBound", + "kind": "parserRule", + "name": "winFrameBound", + "queryReachable": true + }, + { + "id": "rule:winFrameClause", + "kind": "parserRule", + "name": "winFrameClause", + "queryReachable": true + }, + { + "id": "rule:winFrameExtend", + "kind": "parserRule", + "name": "winFrameExtend", + "queryReachable": true + }, + { + "id": "rule:winOrderByClause", + "kind": "parserRule", + "name": "winOrderByClause", + "queryReachable": true + }, + { + "id": "rule:winPartitionByClause", + "kind": "parserRule", + "name": "winPartitionByClause", + "queryReachable": true + }, + { + "id": "rule:windowClause", + "kind": "parserRule", + "name": "windowClause", + "queryReachable": true + }, + { + "id": "rule:windowExpr", + "kind": "parserRule", + "name": "windowExpr", + "queryReachable": true + }, + { + "id": "rule:withClause", + "kind": "parserRule", + "name": "withClause", + "queryReachable": true + }, + { + "id": "rule:withExpr", + "kind": "parserRule", + "name": "withExpr", + "queryReachable": true + }, + { + "id": "rule:withExprColumnNameList", + "kind": "parserRule", + "name": "withExprColumnNameList", + "queryReachable": true + }, + { + "id": "rule:withExprList", + "kind": "parserRule", + "name": "withExprList", + "queryReachable": true + }, + { + "id": "rule:withFillClause", + "kind": "parserRule", + "name": "withFillClause", + "queryReachable": true + }, + { + "id": "rule:withinGroupClause", + "kind": "parserRule", + "name": "withinGroupClause", + "queryReachable": true + }, + { + "id": "token:ALL", + "kind": "token", + "name": "ALL", + "queryReachable": true + }, + { + "id": "token:AND", + "kind": "token", + "name": "AND", + "queryReachable": true + }, + { + "id": "token:ANTI", + "kind": "token", + "name": "ANTI", + "queryReachable": true + }, + { + "id": "token:ANY", + "kind": "token", + "name": "ANY", + "queryReachable": true + }, + { + "id": "token:ARRAY", + "kind": "token", + "name": "ARRAY", + "queryReachable": true + }, + { + "id": "token:ARROW", + "kind": "token", + "name": "ARROW", + "queryReachable": true + }, + { + "id": "token:AS", + "kind": "token", + "name": "AS", + "queryReachable": true + }, + { + "id": "token:ASCENDING", + "kind": "token", + "name": "ASCENDING", + "queryReachable": true + }, + { + "id": "token:ASOF", + "kind": "token", + "name": "ASOF", + "queryReachable": true + }, + { + "id": "token:ASTERISK", + "kind": "token", + "name": "ASTERISK", + "queryReachable": true + }, + { + "id": "token:BACKQUOTE", + "kind": "token", + "name": "BACKQUOTE", + "queryReachable": false + }, + { + "id": "token:BACKSLASH", + "kind": "token", + "name": "BACKSLASH", + "queryReachable": false + }, + { + "id": "token:BETWEEN", + "kind": "token", + "name": "BETWEEN", + "queryReachable": true + }, + { + "id": "token:BINARY_LITERAL", + "kind": "token", + "name": "BINARY_LITERAL", + "queryReachable": true + }, + { + "id": "token:BOTH", + "kind": "token", + "name": "BOTH", + "queryReachable": true + }, + { + "id": "token:BY", + "kind": "token", + "name": "BY", + "queryReachable": true + }, + { + "id": "token:CASE", + "kind": "token", + "name": "CASE", + "queryReachable": true + }, + { + "id": "token:CAST", + "kind": "token", + "name": "CAST", + "queryReachable": true + }, + { + "id": "token:CATCH", + "kind": "token", + "name": "CATCH", + "queryReachable": true + }, + { + "id": "token:COHORT", + "kind": "token", + "name": "COHORT", + "queryReachable": true + }, + { + "id": "token:COLLATE", + "kind": "token", + "name": "COLLATE", + "queryReachable": true + }, + { + "id": "token:COLON", + "kind": "token", + "name": "COLON", + "queryReachable": true + }, + { + "id": "token:COLONEQUALS", + "kind": "token", + "name": "COLONEQUALS", + "queryReachable": true + }, + { + "id": "token:COLUMNS", + "kind": "token", + "name": "COLUMNS", + "queryReachable": true + }, + { + "id": "token:COMMA", + "kind": "token", + "name": "COMMA", + "queryReachable": true + }, + { + "id": "token:CONCAT", + "kind": "token", + "name": "CONCAT", + "queryReachable": true + }, + { + "id": "token:CROSS", + "kind": "token", + "name": "CROSS", + "queryReachable": true + }, + { + "id": "token:CUBE", + "kind": "token", + "name": "CUBE", + "queryReachable": true + }, + { + "id": "token:CURRENT", + "kind": "token", + "name": "CURRENT", + "queryReachable": true + }, + { + "id": "token:DASH", + "kind": "token", + "name": "DASH", + "queryReachable": true + }, + { + "id": "token:DATE", + "kind": "token", + "name": "DATE", + "queryReachable": true + }, + { + "id": "token:DAY", + "kind": "token", + "name": "DAY", + "queryReachable": true + }, + { + "id": "token:DECIMAL_LITERAL", + "kind": "token", + "name": "DECIMAL_LITERAL", + "queryReachable": true + }, + { + "id": "token:DESC", + "kind": "token", + "name": "DESC", + "queryReachable": true + }, + { + "id": "token:DESCENDING", + "kind": "token", + "name": "DESCENDING", + "queryReachable": true + }, + { + "id": "token:DISTINCT", + "kind": "token", + "name": "DISTINCT", + "queryReachable": true + }, + { + "id": "token:DOLLAR", + "kind": "token", + "name": "DOLLAR", + "queryReachable": false + }, + { + "id": "token:DOT", + "kind": "token", + "name": "DOT", + "queryReachable": true + }, + { + "id": "token:DOUBLECOLON", + "kind": "token", + "name": "DOUBLECOLON", + "queryReachable": true + }, + { + "id": "token:ELSE", + "kind": "token", + "name": "ELSE", + "queryReachable": true + }, + { + "id": "token:END", + "kind": "token", + "name": "END", + "queryReachable": true + }, + { + "id": "token:EQ_DOUBLE", + "kind": "token", + "name": "EQ_DOUBLE", + "queryReachable": true + }, + { + "id": "token:EQ_SINGLE", + "kind": "token", + "name": "EQ_SINGLE", + "queryReachable": true + }, + { + "id": "token:ESCAPE_CHAR_COMMON", + "kind": "token", + "name": "ESCAPE_CHAR_COMMON", + "queryReachable": false + }, + { + "id": "token:EXCEPT", + "kind": "token", + "name": "EXCEPT", + "queryReachable": true + }, + { + "id": "token:EXCLUDE", + "kind": "token", + "name": "EXCLUDE", + "queryReachable": true + }, + { + "id": "token:EXTRACT", + "kind": "token", + "name": "EXTRACT", + "queryReachable": true + }, + { + "id": "token:FILL", + "kind": "token", + "name": "FILL", + "queryReachable": true + }, + { + "id": "token:FILTER", + "kind": "token", + "name": "FILTER", + "queryReachable": true + }, + { + "id": "token:FINAL", + "kind": "token", + "name": "FINAL", + "queryReachable": true + }, + { + "id": "token:FINALLY", + "kind": "token", + "name": "FINALLY", + "queryReachable": true + }, + { + "id": "token:FIRST", + "kind": "token", + "name": "FIRST", + "queryReachable": true + }, + { + "id": "token:FLOATING_LITERAL", + "kind": "token", + "name": "FLOATING_LITERAL", + "queryReachable": true + }, + { + "id": "token:FN", + "kind": "token", + "name": "FN", + "queryReachable": true + }, + { + "id": "token:FOLLOWING", + "kind": "token", + "name": "FOLLOWING", + "queryReachable": true + }, + { + "id": "token:FOR", + "kind": "token", + "name": "FOR", + "queryReachable": true + }, + { + "id": "token:FROM", + "kind": "token", + "name": "FROM", + "queryReachable": true + }, + { + "id": "token:FULL", + "kind": "token", + "name": "FULL", + "queryReachable": true + }, + { + "id": "token:FULL_STRING_ESCAPE_TRIGGER", + "kind": "token", + "name": "FULL_STRING_ESCAPE_TRIGGER", + "queryReachable": false + }, + { + "id": "token:FULL_STRING_TEXT", + "kind": "token", + "name": "FULL_STRING_TEXT", + "queryReachable": false + }, + { + "id": "token:FUN", + "kind": "token", + "name": "FUN", + "queryReachable": true + }, + { + "id": "token:GROUP", + "kind": "token", + "name": "GROUP", + "queryReachable": true + }, + { + "id": "token:GROUPING", + "kind": "token", + "name": "GROUPING", + "queryReachable": true + }, + { + "id": "token:GT", + "kind": "token", + "name": "GT", + "queryReachable": true + }, + { + "id": "token:GT_EQ", + "kind": "token", + "name": "GT_EQ", + "queryReachable": true + }, + { + "id": "token:HASH", + "kind": "token", + "name": "HASH", + "queryReachable": true + }, + { + "id": "token:HASH_COMMENT", + "kind": "token", + "name": "HASH_COMMENT", + "queryReachable": false + }, + { + "id": "token:HAVING", + "kind": "token", + "name": "HAVING", + "queryReachable": true + }, + { + "id": "token:HEXADECIMAL_LITERAL", + "kind": "token", + "name": "HEXADECIMAL_LITERAL", + "queryReachable": true + }, + { + "id": "token:HOGQLX_TEXT_TEXT", + "kind": "token", + "name": "HOGQLX_TEXT_TEXT", + "queryReachable": true + }, + { + "id": "token:HOGQLX_TEXT_WS", + "kind": "token", + "name": "HOGQLX_TEXT_WS", + "queryReachable": false + }, + { + "id": "token:HOUR", + "kind": "token", + "name": "HOUR", + "queryReachable": true + }, + { + "id": "token:ID", + "kind": "token", + "name": "ID", + "queryReachable": true + }, + { + "id": "token:IDENTIFIER", + "kind": "token", + "name": "IDENTIFIER", + "queryReachable": true + }, + { + "id": "token:IF", + "kind": "token", + "name": "IF", + "queryReachable": true + }, + { + "id": "token:IGNORE", + "kind": "token", + "name": "IGNORE", + "queryReachable": true + }, + { + "id": "token:ILIKE", + "kind": "token", + "name": "ILIKE", + "queryReachable": true + }, + { + "id": "token:IN", + "kind": "token", + "name": "IN", + "queryReachable": true + }, + { + "id": "token:INCLUDE", + "kind": "token", + "name": "INCLUDE", + "queryReachable": true + }, + { + "id": "token:INF", + "kind": "token", + "name": "INF", + "queryReachable": true + }, + { + "id": "token:INNER", + "kind": "token", + "name": "INNER", + "queryReachable": true + }, + { + "id": "token:INTERPOLATE", + "kind": "token", + "name": "INTERPOLATE", + "queryReachable": true + }, + { + "id": "token:INTERSECT", + "kind": "token", + "name": "INTERSECT", + "queryReachable": true + }, + { + "id": "token:INTERVAL", + "kind": "token", + "name": "INTERVAL", + "queryReachable": true + }, + { + "id": "token:IREGEX_DOUBLE", + "kind": "token", + "name": "IREGEX_DOUBLE", + "queryReachable": true + }, + { + "id": "token:IREGEX_SINGLE", + "kind": "token", + "name": "IREGEX_SINGLE", + "queryReachable": true + }, + { + "id": "token:IS", + "kind": "token", + "name": "IS", + "queryReachable": true + }, + { + "id": "token:JOIN", + "kind": "token", + "name": "JOIN", + "queryReachable": true + }, + { + "id": "token:KEY", + "kind": "token", + "name": "KEY", + "queryReachable": true + }, + { + "id": "token:LAMBDA", + "kind": "token", + "name": "LAMBDA", + "queryReachable": true + }, + { + "id": "token:LAST", + "kind": "token", + "name": "LAST", + "queryReachable": true + }, + { + "id": "token:LBRACE", + "kind": "token", + "name": "LBRACE", + "queryReachable": true + }, + { + "id": "token:LBRACKET", + "kind": "token", + "name": "LBRACKET", + "queryReachable": true + }, + { + "id": "token:LEADING", + "kind": "token", + "name": "LEADING", + "queryReachable": true + }, + { + "id": "token:LEFT", + "kind": "token", + "name": "LEFT", + "queryReachable": true + }, + { + "id": "token:LET", + "kind": "token", + "name": "LET", + "queryReachable": true + }, + { + "id": "token:LIKE", + "kind": "token", + "name": "LIKE", + "queryReachable": true + }, + { + "id": "token:LIMIT", + "kind": "token", + "name": "LIMIT", + "queryReachable": true + }, + { + "id": "token:LOCAL", + "kind": "token", + "name": "LOCAL", + "queryReachable": true + }, + { + "id": "token:LPAREN", + "kind": "token", + "name": "LPAREN", + "queryReachable": true + }, + { + "id": "token:LT", + "kind": "token", + "name": "LT", + "queryReachable": true + }, + { + "id": "token:LT_EQ", + "kind": "token", + "name": "LT_EQ", + "queryReachable": true + }, + { + "id": "token:LT_SLASH", + "kind": "token", + "name": "LT_SLASH", + "queryReachable": true + }, + { + "id": "token:MALFORMED_BINARY_LITERAL", + "kind": "token", + "name": "MALFORMED_BINARY_LITERAL", + "queryReachable": false + }, + { + "id": "token:MATERIALIZED", + "kind": "token", + "name": "MATERIALIZED", + "queryReachable": true + }, + { + "id": "token:MINUTE", + "kind": "token", + "name": "MINUTE", + "queryReachable": true + }, + { + "id": "token:MONTH", + "kind": "token", + "name": "MONTH", + "queryReachable": true + }, + { + "id": "token:MULTI_LINE_COMMENT", + "kind": "token", + "name": "MULTI_LINE_COMMENT", + "queryReachable": false + }, + { + "id": "token:NAME", + "kind": "token", + "name": "NAME", + "queryReachable": true + }, + { + "id": "token:NAN_SQL", + "kind": "token", + "name": "NAN_SQL", + "queryReachable": true + }, + { + "id": "token:NATURAL", + "kind": "token", + "name": "NATURAL", + "queryReachable": true + }, + { + "id": "token:NOT", + "kind": "token", + "name": "NOT", + "queryReachable": true + }, + { + "id": "token:NOT_EQ", + "kind": "token", + "name": "NOT_EQ", + "queryReachable": true + }, + { + "id": "token:NOT_IREGEX", + "kind": "token", + "name": "NOT_IREGEX", + "queryReachable": true + }, + { + "id": "token:NOT_REGEX", + "kind": "token", + "name": "NOT_REGEX", + "queryReachable": true + }, + { + "id": "token:NULLISH", + "kind": "token", + "name": "NULLISH", + "queryReachable": true + }, + { + "id": "token:NULLS", + "kind": "token", + "name": "NULLS", + "queryReachable": true + }, + { + "id": "token:NULL_PROPERTY", + "kind": "token", + "name": "NULL_PROPERTY", + "queryReachable": true + }, + { + "id": "token:NULL_SAFE_EQ", + "kind": "token", + "name": "NULL_SAFE_EQ", + "queryReachable": true + }, + { + "id": "token:NULL_SQL", + "kind": "token", + "name": "NULL_SQL", + "queryReachable": true + }, + { + "id": "token:OCTAL_LITERAL", + "kind": "token", + "name": "OCTAL_LITERAL", + "queryReachable": true + }, + { + "id": "token:OCTAL_PREFIX_LITERAL", + "kind": "token", + "name": "OCTAL_PREFIX_LITERAL", + "queryReachable": true + }, + { + "id": "token:OFFSET", + "kind": "token", + "name": "OFFSET", + "queryReachable": true + }, + { + "id": "token:ON", + "kind": "token", + "name": "ON", + "queryReachable": true + }, + { + "id": "token:OR", + "kind": "token", + "name": "OR", + "queryReachable": true + }, + { + "id": "token:ORDER", + "kind": "token", + "name": "ORDER", + "queryReachable": true + }, + { + "id": "token:OUTER", + "kind": "token", + "name": "OUTER", + "queryReachable": true + }, + { + "id": "token:OVER", + "kind": "token", + "name": "OVER", + "queryReachable": true + }, + { + "id": "token:PARTITION", + "kind": "token", + "name": "PARTITION", + "queryReachable": true + }, + { + "id": "token:PERCENT", + "kind": "token", + "name": "PERCENT", + "queryReachable": true + }, + { + "id": "token:PIVOT", + "kind": "token", + "name": "PIVOT", + "queryReachable": true + }, + { + "id": "token:PLUS", + "kind": "token", + "name": "PLUS", + "queryReachable": true + }, + { + "id": "token:POSITIONAL", + "kind": "token", + "name": "POSITIONAL", + "queryReachable": true + }, + { + "id": "token:PRECEDING", + "kind": "token", + "name": "PRECEDING", + "queryReachable": true + }, + { + "id": "token:PREWHERE", + "kind": "token", + "name": "PREWHERE", + "queryReachable": true + }, + { + "id": "token:QUALIFY", + "kind": "token", + "name": "QUALIFY", + "queryReachable": true + }, + { + "id": "token:QUARTER", + "kind": "token", + "name": "QUARTER", + "queryReachable": true + }, + { + "id": "token:QUERY", + "kind": "token", + "name": "QUERY", + "queryReachable": true + }, + { + "id": "token:QUOTED_IDENTIFIER", + "kind": "token", + "name": "QUOTED_IDENTIFIER", + "queryReachable": true + }, + { + "id": "token:QUOTE_DOUBLE", + "kind": "token", + "name": "QUOTE_DOUBLE", + "queryReachable": false + }, + { + "id": "token:QUOTE_SINGLE", + "kind": "token", + "name": "QUOTE_SINGLE", + "queryReachable": true + }, + { + "id": "token:QUOTE_SINGLE_TEMPLATE", + "kind": "token", + "name": "QUOTE_SINGLE_TEMPLATE", + "queryReachable": true + }, + { + "id": "token:QUOTE_SINGLE_TEMPLATE_FULL", + "kind": "token", + "name": "QUOTE_SINGLE_TEMPLATE_FULL", + "queryReachable": false + }, + { + "id": "token:RANGE", + "kind": "token", + "name": "RANGE", + "queryReachable": true + }, + { + "id": "token:RBRACE", + "kind": "token", + "name": "RBRACE", + "queryReachable": true + }, + { + "id": "token:RBRACKET", + "kind": "token", + "name": "RBRACKET", + "queryReachable": true + }, + { + "id": "token:RECURSIVE", + "kind": "token", + "name": "RECURSIVE", + "queryReachable": true + }, + { + "id": "token:REGEX_DOUBLE", + "kind": "token", + "name": "REGEX_DOUBLE", + "queryReachable": true + }, + { + "id": "token:REGEX_SINGLE", + "kind": "token", + "name": "REGEX_SINGLE", + "queryReachable": true + }, + { + "id": "token:REPLACE", + "kind": "token", + "name": "REPLACE", + "queryReachable": true + }, + { + "id": "token:RETURN", + "kind": "token", + "name": "RETURN", + "queryReachable": true + }, + { + "id": "token:RIGHT", + "kind": "token", + "name": "RIGHT", + "queryReachable": true + }, + { + "id": "token:ROLLUP", + "kind": "token", + "name": "ROLLUP", + "queryReachable": true + }, + { + "id": "token:ROW", + "kind": "token", + "name": "ROW", + "queryReachable": true + }, + { + "id": "token:ROWS", + "kind": "token", + "name": "ROWS", + "queryReachable": true + }, + { + "id": "token:RPAREN", + "kind": "token", + "name": "RPAREN", + "queryReachable": true + }, + { + "id": "token:SAMPLE", + "kind": "token", + "name": "SAMPLE", + "queryReachable": true + }, + { + "id": "token:SECOND", + "kind": "token", + "name": "SECOND", + "queryReachable": true + }, + { + "id": "token:SELECT", + "kind": "token", + "name": "SELECT", + "queryReachable": true + }, + { + "id": "token:SEMI", + "kind": "token", + "name": "SEMI", + "queryReachable": true + }, + { + "id": "token:SEMICOLON", + "kind": "token", + "name": "SEMICOLON", + "queryReachable": true + }, + { + "id": "token:SETS", + "kind": "token", + "name": "SETS", + "queryReachable": true + }, + { + "id": "token:SETTINGS", + "kind": "token", + "name": "SETTINGS", + "queryReachable": true + }, + { + "id": "token:SINGLE_LINE_COMMENT", + "kind": "token", + "name": "SINGLE_LINE_COMMENT", + "queryReachable": false + }, + { + "id": "token:SLASH", + "kind": "token", + "name": "SLASH", + "queryReachable": true + }, + { + "id": "token:SLASH_GT", + "kind": "token", + "name": "SLASH_GT", + "queryReachable": true + }, + { + "id": "token:STEP", + "kind": "token", + "name": "STEP", + "queryReachable": true + }, + { + "id": "token:STRING_ESCAPE_TRIGGER", + "kind": "token", + "name": "STRING_ESCAPE_TRIGGER", + "queryReachable": true + }, + { + "id": "token:STRING_LITERAL", + "kind": "token", + "name": "STRING_LITERAL", + "queryReachable": true + }, + { + "id": "token:STRING_TEXT", + "kind": "token", + "name": "STRING_TEXT", + "queryReachable": true + }, + { + "id": "token:SUBSTRING", + "kind": "token", + "name": "SUBSTRING", + "queryReachable": true + }, + { + "id": "token:TAGC_MULTI_LINE_COMMENT", + "kind": "token", + "name": "TAGC_MULTI_LINE_COMMENT", + "queryReachable": false + }, + { + "id": "token:TAGC_SINGLE_LINE_COMMENT", + "kind": "token", + "name": "TAGC_SINGLE_LINE_COMMENT", + "queryReachable": false + }, + { + "id": "token:TAGC_WS", + "kind": "token", + "name": "TAGC_WS", + "queryReachable": false + }, + { + "id": "token:TAG_MULTI_LINE_COMMENT", + "kind": "token", + "name": "TAG_MULTI_LINE_COMMENT", + "queryReachable": false + }, + { + "id": "token:TAG_SINGLE_LINE_COMMENT", + "kind": "token", + "name": "TAG_SINGLE_LINE_COMMENT", + "queryReachable": false + }, + { + "id": "token:TAG_WS", + "kind": "token", + "name": "TAG_WS", + "queryReachable": false + }, + { + "id": "token:THEN", + "kind": "token", + "name": "THEN", + "queryReachable": true + }, + { + "id": "token:THROW", + "kind": "token", + "name": "THROW", + "queryReachable": true + }, + { + "id": "token:TIES", + "kind": "token", + "name": "TIES", + "queryReachable": true + }, + { + "id": "token:TIME", + "kind": "token", + "name": "TIME", + "queryReachable": true + }, + { + "id": "token:TIMESTAMP", + "kind": "token", + "name": "TIMESTAMP", + "queryReachable": true + }, + { + "id": "token:TO", + "kind": "token", + "name": "TO", + "queryReachable": true + }, + { + "id": "token:TOP", + "kind": "token", + "name": "TOP", + "queryReachable": true + }, + { + "id": "token:TOTALS", + "kind": "token", + "name": "TOTALS", + "queryReachable": true + }, + { + "id": "token:TRAILING", + "kind": "token", + "name": "TRAILING", + "queryReachable": true + }, + { + "id": "token:TRIM", + "kind": "token", + "name": "TRIM", + "queryReachable": true + }, + { + "id": "token:TRUNCATE", + "kind": "token", + "name": "TRUNCATE", + "queryReachable": true + }, + { + "id": "token:TRY", + "kind": "token", + "name": "TRY", + "queryReachable": true + }, + { + "id": "token:TRY_CAST", + "kind": "token", + "name": "TRY_CAST", + "queryReachable": true + }, + { + "id": "token:UNBOUNDED", + "kind": "token", + "name": "UNBOUNDED", + "queryReachable": true + }, + { + "id": "token:UNDERSCORE", + "kind": "token", + "name": "UNDERSCORE", + "queryReachable": false + }, + { + "id": "token:UNEXPECTED_CHARACTER", + "kind": "token", + "name": "UNEXPECTED_CHARACTER", + "queryReachable": false + }, + { + "id": "token:UNION", + "kind": "token", + "name": "UNION", + "queryReachable": true + }, + { + "id": "token:UNPIVOT", + "kind": "token", + "name": "UNPIVOT", + "queryReachable": true + }, + { + "id": "token:USING", + "kind": "token", + "name": "USING", + "queryReachable": true + }, + { + "id": "token:VALUES", + "kind": "token", + "name": "VALUES", + "queryReachable": true + }, + { + "id": "token:WEEK", + "kind": "token", + "name": "WEEK", + "queryReachable": true + }, + { + "id": "token:WHEN", + "kind": "token", + "name": "WHEN", + "queryReachable": true + }, + { + "id": "token:WHERE", + "kind": "token", + "name": "WHERE", + "queryReachable": true + }, + { + "id": "token:WHILE", + "kind": "token", + "name": "WHILE", + "queryReachable": true + }, + { + "id": "token:WHITESPACE", + "kind": "token", + "name": "WHITESPACE", + "queryReachable": false + }, + { + "id": "token:WINDOW", + "kind": "token", + "name": "WINDOW", + "queryReachable": true + }, + { + "id": "token:WITH", + "kind": "token", + "name": "WITH", + "queryReachable": true + }, + { + "id": "token:WITHIN", + "kind": "token", + "name": "WITHIN", + "queryReachable": true + }, + { + "id": "token:YEAR", + "kind": "token", + "name": "YEAR", + "queryReachable": true + }, + { + "id": "token:ZONE", + "kind": "token", + "name": "ZONE", + "queryReachable": true + } + ], + "grammarSha256": "c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242", + "languageVersion": "1.0.0", + "queryEntryPoint": "select", + "schemaVersion": 1, + "valid": false, + "validationErrors": [ + { + "alternativeCount": 3, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "alias" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "columnExprTupleOrSingle" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "columnIdentifier" + }, + { + "alternativeCount": 4, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "columnTypeCastIdentifier" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "declaration" + }, + { + "alternativeCount": 3, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "floatingLiteral" + }, + { + "alternativeCount": 3, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "hogqlxChildElement" + }, + { + "alternativeCount": 3, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "hogqlxTagAttribute" + }, + { + "alternativeCount": 4, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "identifier" + }, + { + "alternativeCount": 3, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "implicitAlias" + }, + { + "alternativeCount": 3, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "joinConstraintClause" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "joinOpCross" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "limitAndOffsetClause" + }, + { + "alternativeCount": 3, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "limitAndOffsetClauseOptional" + }, + { + "alternativeCount": 3, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "literal" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "ratioExpr" + }, + { + "alternativeCount": 4, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "selectStmtWithParens" + }, + { + "alternativeCount": 11, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "statement" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "string" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "stringContents" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": false, + "rule": "stringContentsFull" + } + ] +} diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar/HogQLLexer.common.g4 b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar/HogQLLexer.common.g4 new file mode 100644 index 000000000000..6525c654f2a2 --- /dev/null +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar/HogQLLexer.common.g4 @@ -0,0 +1,357 @@ +lexer grammar HogQLLexer; + +// NB! We cat either HogQLLexter.cpp.g4 or HogQLLexter.python.g4 when generating the grammar. + +// NOTE: don't forget to add new keywords to the parser rule "keyword"! + +// Keywords + +ALL: A L L; +AND: A N D; +ANTI: A N T I; +ANY: A N Y; +ARRAY: A R R A Y; +AS: A S; +ASCENDING: A S C | A S C E N D I N G; +ASOF: A S O F; +BETWEEN: B E T W E E N; +BOTH: B O T H; +BY: B Y; +CASE: C A S E; +CAST: C A S T; +CATCH: C A T C H; +COHORT: C O H O R T; +COLLATE: C O L L A T E; +COLUMNS: C O L U M N S; +CROSS: C R O S S; +CUBE: C U B E; +CURRENT: C U R R E N T; +DATE: D A T E; +DAY: D A Y; +DESC: D E S C; +DESCENDING: D E S C E N D I N G; +DISTINCT: D I S T I N C T; +ELSE: E L S E; +END: E N D; +EXCEPT: E X C E P T; +EXCLUDE: E X C L U D E; +EXTRACT: E X T R A C T; +FINAL: F I N A L; +FILL: F I L L; +FILTER: F I L T E R; +FINALLY: F I N A L L Y; +FIRST: F I R S T; +FN: F N; +FOLLOWING: F O L L O W I N G; +FOR: F O R; +FROM: F R O M; +FULL: F U L L; +FUN: F U N; +GROUP: G R O U P; +GROUPING: G R O U P I N G; +HAVING: H A V I N G; +HOUR: H O U R; +ID: I D; +IF: I F; +ILIKE: I L I K E; +IGNORE: I G N O R E; +INCLUDE: I N C L U D E; +IN: I N; +INF: I N F | I N F I N I T Y; +INNER: I N N E R; +INTERSECT: I N T E R S E C T; +INTERPOLATE: I N T E R P O L A T E; +INTERVAL: I N T E R V A L; +IS: I S; +JOIN: J O I N; +KEY: K E Y; +LAMBDA: L A M B D A; +LAST: L A S T; +LEADING: L E A D I N G; +LEFT: L E F T; +LET: L E T; +LIKE: L I K E; +LIMIT: L I M I T; +MATERIALIZED: M A T E R I A L I Z E D; +MINUTE: M I N U T E; +MONTH: M O N T H; +NAME: N A M E; +NATURAL: N A T U R A L; +NAN_SQL: N A N; // conflicts with macro NAN +NOT: N O T; +NULL_SQL: N U L L; // conflicts with macro NULL +NULLS: N U L L S; +OFFSET: O F F S E T; +ON: O N; +OR: O R; +ORDER: O R D E R; +OUTER: O U T E R; +OVER: O V E R; +PARTITION: P A R T I T I O N; +PIVOT: P I V O T; +POSITIONAL: P O S I T I O N A L; +PRECEDING: P R E C E D I N G; +PREWHERE: P R E W H E R E; +QUALIFY: Q U A L I F Y; +QUARTER: Q U A R T E R; +RANGE: R A N G E; +RECURSIVE: R E C U R S I V E; +REPLACE: R E P L A C E; +RETURN: R E T U R N; +RIGHT: R I G H T; +ROLLUP: R O L L U P; +ROW: R O W; +ROWS: R O W S; +SAMPLE: S A M P L E; +SECOND: S E C O N D; +SELECT: S E L E C T; +SEMI: S E M I; +SETS: S E T S; +SETTINGS: S E T T I N G S; +STEP: S T E P; +SUBSTRING: S U B S T R I N G; +THEN: T H E N; +THROW: T H R O W; +TIES: T I E S; +TIMESTAMP: T I M E S T A M P; +TIME: T I M E; +LOCAL: L O C A L; +ZONE: Z O N E; +TO: T O; +TOP: T O P; +TOTALS: T O T A L S; +TRAILING: T R A I L I N G; +TRIM: T R I M; +TRUNCATE: T R U N C A T E; +TRY: T R Y; +TRY_CAST: T R Y '_' C A S T; +UNBOUNDED: U N B O U N D E D; +UNION: U N I O N; +UNPIVOT: U N P I V O T; +USING: U S I N G; +VALUES: V A L U E S; +WEEK: W E E K; +WHEN: W H E N; +WHERE: W H E R E; +WHILE: W H I L E; +WINDOW: W I N D O W; +WITH: W I T H; +WITHIN: W I T H I N; +YEAR: Y E A R | Y Y Y Y; + +// Tokens + +// copied from clickhouse_driver/util/escape.py +ESCAPE_CHAR_COMMON + : BACKSLASH B + | BACKSLASH F + | BACKSLASH R + | BACKSLASH N + | BACKSLASH T + | BACKSLASH '0' + | BACKSLASH A + | BACKSLASH V + | BACKSLASH BACKSLASH + | BACKSLASH X HEX_DIGIT HEX_DIGIT; + +IDENTIFIER + : (LETTER | UNDERSCORE | DOLLAR) (LETTER | UNDERSCORE | DEC_DIGIT | DOLLAR)* + ; +QUOTED_IDENTIFIER + : BACKQUOTE ( ~([\\`]) | ESCAPE_CHAR_COMMON | BACKSLASH QUOTE_SINGLE | (BACKQUOTE BACKQUOTE) )* BACKQUOTE + | QUOTE_DOUBLE ( ~([\\"]) | ESCAPE_CHAR_COMMON | BACKSLASH QUOTE_DOUBLE | (QUOTE_DOUBLE QUOTE_DOUBLE) )* QUOTE_DOUBLE + ; +FLOATING_LITERAL + // Hex-float exponent: strict C99 `p`/`P` only — `e`/`E` stays a hex digit, so `0x1e5` is 485, not a float. + : HEXADECIMAL_LITERAL DOT HEX_DIGIT* P (PLUS | DASH)? DEC_DIGIT+ + | HEXADECIMAL_LITERAL P (PLUS | DASH)? DEC_DIGIT+ + | DECIMAL_LITERAL DOT DEC_DIGIT* E (PLUS | DASH)? DEC_DIGIT+ + | DOT DECIMAL_LITERAL E (PLUS | DASH)? DEC_DIGIT+ + | DECIMAL_LITERAL E (PLUS | DASH)? DEC_DIGIT+ + ; +// Binary literals (`0b1010`). Declared first so it wins the length-tie against MALFORMED_BINARY_LITERAL. +BINARY_LITERAL: '0' B BIN_DIGIT+; +OCTAL_LITERAL: '0' OCT_DIGIT+; +DECIMAL_LITERAL: DEC_DIGIT+; +HEXADECIMAL_LITERAL: '0' X HEX_DIGIT+; +// Postgres-16 `0o` octal — unsupported; lexed as a real token so the visitor can reject it clearly. +OCTAL_PREFIX_LITERAL: '0' [oO] DEC_DIGIT+; +// Malformed binary (`0b22`) BINARY_LITERAL didn't consume — caught so it can't re-tokenise as `0` + IDENTIFIER. +MALFORMED_BINARY_LITERAL: '0' [bB] DEC_DIGIT+; + +// It's important that quote-symbol is a single character. +STRING_LITERAL: QUOTE_SINGLE ( ~([\\']) | ESCAPE_CHAR_COMMON | BACKSLASH QUOTE_SINGLE | (QUOTE_SINGLE QUOTE_SINGLE) )* QUOTE_SINGLE; + + +// Alphabet and allowed symbols + +fragment A: [aA]; +fragment B: [bB]; +fragment C: [cC]; +fragment D: [dD]; +fragment E: [eE]; +fragment F: [fF]; +fragment G: [gG]; +fragment H: [hH]; +fragment I: [iI]; +fragment J: [jJ]; +fragment K: [kK]; +fragment L: [lL]; +fragment M: [mM]; +fragment N: [nN]; +fragment O: [oO]; +fragment P: [pP]; +fragment Q: [qQ]; +fragment R: [rR]; +fragment S: [sS]; +fragment T: [tT]; +fragment U: [uU]; +fragment V: [vV]; +fragment W: [wW]; +fragment X: [xX]; +fragment Y: [yY]; +fragment Z: [zZ]; + +fragment LETTER: [a-zA-Z]; +fragment BIN_DIGIT: [01]; +fragment OCT_DIGIT: [0-7]; +fragment DEC_DIGIT: [0-9]; +fragment HEX_DIGIT: [0-9a-fA-F]; + +ARROW: '->'; +ASTERISK: '*'; +BACKQUOTE: '`'; +BACKSLASH: '\\'; +DOUBLECOLON: '::'; +COLONEQUALS: ':='; +COLON: ':'; +COMMA: ','; +CONCAT: '||'; +DASH: '-'; +DOLLAR: '$'; +DOT: '.'; +EQ_DOUBLE: '=='; +EQ_SINGLE: '='; +GT_EQ: '>='; +GT: '>'; +HASH: '#'; +IREGEX_SINGLE: '~*'; +IREGEX_DOUBLE: '=~*'; +LBRACE: '{' -> pushMode(DEFAULT_MODE); +LBRACKET: '['; +LPAREN: '('; +NULL_SAFE_EQ: '<=>'; +LT_EQ: '<='; +TAG_LT_SLASH: ' type(LT_SLASH), pushMode(HOGQLX_TAG_CLOSE); +TAG_LT_OPEN: '<' {isOpeningTag()}? -> type(LT), pushMode(HOGQLX_TAG_OPEN); +LT: '<'; +LT_SLASH: ''; +NOT_IREGEX: '!~*'; +NOT_REGEX: '!~'; +NULL_PROPERTY: '?.'; +NULLISH: '??'; +PERCENT: '%'; +PLUS: '+'; +QUERY: '?'; +QUOTE_DOUBLE: '"'; +QUOTE_SINGLE_TEMPLATE: 'f\'' -> pushMode(IN_TEMPLATE_STRING); // start of regular f'' template strings +QUOTE_SINGLE_TEMPLATE_FULL: 'F\'' -> pushMode(IN_FULL_TEMPLATE_STRING); // magic F' symbol used to parse "full text" templates +QUOTE_SINGLE: '\''; +REGEX_SINGLE: '~'; +REGEX_DOUBLE: '=~'; +RBRACE: '}' -> popMode; +RBRACKET: ']'; +RPAREN: ')'; +SEMICOLON: ';'; +SLASH: '/'; +SLASH_GT: '/>'; +UNDERSCORE: '_'; + +// Comments and whitespace +MULTI_LINE_COMMENT: '/*' .*? '*/' -> skip; +SINGLE_LINE_COMMENT: ('--' | '//') ~('\n'|'\r')* ('\n' | '\r' | EOF) -> skip; +// MySQL-style `#` comments. `#` is excluded so positional references (`#1`) keep +// working — a `#` comment whose text starts with a digit is the one MySQL-ism this rejects. +HASH_COMMENT: '#' (~[0-9\n\r] ~[\n\r]*)? ('\n' | '\r' | EOF) -> skip; +// whitespace is hidden and not skipped so that it's preserved in ANTLR errors like "no viable alternative" +// The class is the full Unicode `White_Space` set, not just ASCII: a +// NO-BREAK SPACE or other Unicode space (often pasted in from rich +// editors or docs) is genuine whitespace and must keep separating +// tokens. Recognising it here keeps such programs valid — otherwise it +// would fall through to UNEXPECTED_CHARACTER below and fail the whole +// parse. U+FEFF (BOM) is included too, so a file saved with a +// byte-order mark still parses. +WHITESPACE: [ \t\r\n\u000B\u000C\u0085\u00A0\u1680\u2000-\u200A\u2028\u2029\u202F\u205F\u3000\uFEFF] -> channel(HIDDEN); + +// Catch-all for any character no rule above matched. Without this the +// lexer raises a recoverable token-recognition error and DROPS the +// character — so stray input (a JavaScript `!`, `&&`, …) silently +// vanishes and the surrounding text parses as a different, valid-looking +// program. Emitting an explicit token instead means the parser has no +// rule for it and fails loudly with a SyntaxError. Listed last so it +// only ever fires as a true fallback (maximal munch keeps `!=`, `!~`, +// multi-character operators, comments, etc. intact). +UNEXPECTED_CHARACTER: . ; + +// ───────── f' TEMPLATE STRING MODE ───────── +mode IN_TEMPLATE_STRING; +STRING_TEXT: ((~([\\'{])) | ESCAPE_CHAR_COMMON | BACKSLASH QUOTE_SINGLE | (BACKSLASH LBRACE) | (QUOTE_SINGLE QUOTE_SINGLE))+; +STRING_ESCAPE_TRIGGER: LBRACE -> pushMode(DEFAULT_MODE); +STRING_QUOTE_SINGLE: QUOTE_SINGLE -> type(QUOTE_SINGLE), popMode; + +// ───────── F' FULL TEMPLATE STRING MODE ───────── +// a magic F' takes us to "full template strings" mode, where we don't need to escape single quotes and parse until EOF +// this can't be used within a normal columnExpr, but has to be parsed for separately +mode IN_FULL_TEMPLATE_STRING; +FULL_STRING_TEXT: ((~([{])) | ESCAPE_CHAR_COMMON | (BACKSLASH LBRACE))+; +FULL_STRING_ESCAPE_TRIGGER: LBRACE -> pushMode(DEFAULT_MODE); + +// ───────── HOGQLX TAG MODE for opening/self-closing tags ───────── +mode HOGQLX_TAG_OPEN; + +TAG_SELF_CLOSE_GT : '/>' -> type(SLASH_GT), popMode; // +TAG_OPEN_GT : '>' -> type(GT), popMode, pushMode(HOGQLX_TEXT); // + +// Skip comments between attributes — without these, the recoverable lexer error drops the delimiters and re-tokenises the body as phantom attributes. +TAG_MULTI_LINE_COMMENT : '/*' .*? '*/' -> skip; +TAG_SINGLE_LINE_COMMENT : ('--' | '//') ~('\n'|'\r')* ('\n' | '\r' | EOF) -> skip; + +// minimal token set; map everything back to the default token types +TAG_IDENT : [a-zA-Z_][a-zA-Z0-9_-]* -> type(IDENTIFIER); +TAG_EQ : '=' -> type(EQ_SINGLE); +TAG_STRING : STRING_LITERAL -> type(STRING_LITERAL); +TAG_WS : [ \t\r\n]+ -> channel(HIDDEN); +TAG_LBRACE : '{' -> type(LBRACE), pushMode(DEFAULT_MODE); +// Catch-all for unmatched bytes (e.g. `#`, `&`, `@`) so the parser fails loudly instead of silently re-tokenising the surrounding text. +TAG_UNEXPECTED : . -> type(UNEXPECTED_CHARACTER); + + +// ───────── HOGQLX TAG MODE for closing tags ───────── +mode HOGQLX_TAG_CLOSE; + +TAGC_GT : '>' -> type(GT), popMode; // *** no TEXT push *** +TAGC_MULTI_LINE_COMMENT : '/*' .*? '*/' -> skip; +TAGC_SINGLE_LINE_COMMENT : ('--' | '//') ~('\n'|'\r')* ('\n' | '\r' | EOF) -> skip; +TAGC_IDENT : [a-zA-Z_][a-zA-Z0-9_-]* -> type(IDENTIFIER); +TAGC_WS : [ \t\r\n]+ -> channel(HIDDEN); +TAGC_UNEXPECTED : . -> type(UNEXPECTED_CHARACTER); + + +// ───────── HOGQLX TEXT MODE ───────── +mode HOGQLX_TEXT; + +HOGQLX_TEXT_TEXT + : ~[<{]+ ; // everything except “{” or “<” + +HOGQLX_TEXT_LBRACE + : '{' -> type(LBRACE), pushMode(DEFAULT_MODE); + +HOGQLX_TEXT_LT_SLASH + : ' type(LT_SLASH), popMode, pushMode(HOGQLX_TAG_CLOSE); + +HOGQLX_TEXT_LT + : '<' -> type(LT), pushMode(HOGQLX_TAG_OPEN); + +HOGQLX_TEXT_WS + : [ \t\r\n]+ -> channel(HIDDEN); diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar/HogQLLexer.java.g4 b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar/HogQLLexer.java.g4 new file mode 100644 index 000000000000..7afce8bfce2a --- /dev/null +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar/HogQLLexer.java.g4 @@ -0,0 +1,81 @@ +lexer grammar HogQLLexer; + +@members { + +private static boolean isAsciiAlpha(int character) { + return character >= 0 && character < 128 && + ((character >= 'a' && character <= 'z') || (character >= 'A' && character <= 'Z')); +} + +private static boolean isAsciiAlphanumeric(int character) { + return isAsciiAlpha(character) || (character >= '0' && character <= '9'); +} + +private static boolean isAsciiWhitespace(int character) { + return character == ' ' || character == '\t' || character == '\n' || + character == '\u000B' || character == '\f' || character == '\r'; +} + +private int skipWhitespaceAndComments(int index) { + while (true) { + int character = _input.LA(index); + if (isAsciiWhitespace(character)) { + index++; + continue; + } + + if (character == '/' && _input.LA(index + 1) == '/') { + index += 2; + } + else if (character == '-' && _input.LA(index + 1) == '-') { + index += 2; + } + else if (character == '#') { + index++; + } + else { + return index; + } + + while (true) { + character = _input.LA(index); + if (character <= 0 || character == '\n' || character == '\r') { + break; + } + index++; + } + } +} + +private boolean isOpeningTag() { + int firstCharacter = _input.LA(1); + if (!isAsciiAlpha(firstCharacter) && firstCharacter != '_') { + return false; + } + + int index = 2; + while (true) { + int character = _input.LA(index); + if (isAsciiAlphanumeric(character) || character == '_' || character == '-') { + index++; + } + else { + break; + } + } + + int character = _input.LA(index); + if (character == '>' || character == '/') { + return true; + } + + if (isAsciiWhitespace(character)) { + index = skipWhitespaceAndComments(index + 1); + character = _input.LA(index); + return isAsciiAlphanumeric(character) || character == '_' || character == '>' || character == '/'; + } + + return false; +} + +} diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar/HogQLParser.g4 b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar/HogQLParser.g4 new file mode 100644 index 000000000000..6fe244c0a1f3 --- /dev/null +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/grammar/HogQLParser.g4 @@ -0,0 +1,445 @@ +parser grammar HogQLParser; + +options { + tokenVocab = HogQLLexer; +} + + +program: declaration* EOF; + +declaration: varDecl | statement ; + +expression: columnExpr; + +varDecl: LET identifier ( COLONEQUALS expression )? ; +identifierList: nestedIdentifier (COMMA nestedIdentifier)* COMMA?; + +statement : returnStmt + | throwStmt + | tryCatchStmt + | ifStmt + | whileStmt + | forInStmt + | forStmt + | funcStmt + | block + | exprStmt + | emptyStmt + ; + +returnStmt : RETURN expression? SEMICOLON?; +throwStmt : THROW expression SEMICOLON?; +catchBlock : CATCH (LPAREN catchVar=identifier (COLON catchType=identifier)? RPAREN)? catchStmt=block; +tryCatchStmt : TRY tryStmt=block catchBlock* (FINALLY finallyStmt=block)?; +ifStmt : IF LPAREN expression RPAREN statement ( ELSE statement )? ; +whileStmt : WHILE LPAREN expression RPAREN statement SEMICOLON?; +forStmt : FOR LPAREN + (initializerVarDeclr=varDecl | initializerVarAssignment=varAssignment | initializerExpression=expression)? SEMICOLON + condition=expression? SEMICOLON + (incrementVarDeclr=varDecl | incrementVarAssignment=varAssignment | incrementExpression=expression)? + RPAREN statement SEMICOLON?; +forInStmt : FOR LPAREN LET identifier (COMMA identifier)? IN expression RPAREN statement SEMICOLON?; +funcStmt : (FN | FUN) identifier LPAREN identifierList? RPAREN block; +varAssignment : expression COLONEQUALS expression ; +// Assignment folded in as an optional suffix: one expression-leading alternative in +// `statement` means `declaration*` parses without unbounded lookahead. `varAssignment` is forStmt-only. +exprStmt : expression (COLONEQUALS expression)? SEMICOLON?; +emptyStmt : SEMICOLON ; +block : LBRACE declaration* RBRACE ; + +kvPair: expression ':' expression ; +kvPairList: kvPair (COMMA kvPair)* COMMA?; + + +// SELECT statement +select: (selectSetStmt | selectStmt | hogqlxTagElement) SEMICOLON? EOF; + +selectStmtWithParens: selectStmt | withClause LPAREN selectSetStmt RPAREN | LPAREN selectSetStmt RPAREN | placeholder; + +subsequentSelectSetClause: (EXCEPT ALL (BY NAME)? | EXCEPT (BY NAME)? | UNION ALL (BY NAME)? | UNION DISTINCT (BY NAME)? | UNION (BY NAME)? | INTERSECT ALL (BY NAME)? | INTERSECT DISTINCT (BY NAME)? | INTERSECT (BY NAME)?) selectStmtWithParens; +selectSetStmt: selectStmtWithParens (subsequentSelectSetClause)* orderByClause? limitAndOffsetClauseOptional?; +limitAndOffsetClauseOptional + : LIMIT columnExpr PERCENT? (COMMA columnExpr)? (WITH TIES)? + | LIMIT columnExpr PERCENT? (WITH TIES)? OFFSET columnExpr + | OFFSET columnExpr + ; + +selectStmt: + with=withClause? + SELECT DISTINCT? topClause? + columns=selectColumnExprListBeforeFrom + from=fromClause? + arrayJoinClause? + prewhereClause? + where=whereClause? + (USING? sampleClause)? + groupByClause? (WITH (CUBE | ROLLUP))? (WITH TOTALS)? + havingClause? + qualifyClause? + (USING sampleClause)? + windowClause? + orderByClause? + limitByClause? + (limitAndOffsetClause | offsetOnlyClause)? + settingsClause? + ; + +withClause: WITH RECURSIVE? withExprList; +topClause: TOP DECIMAL_LITERAL (WITH TIES)?; +fromClause: FROM joinExpr; +arrayJoinClause: (LEFT | INNER)? ARRAY JOIN columnExprList; +windowClause: WINDOW identifier AS LPAREN windowExpr RPAREN (COMMA identifier AS LPAREN windowExpr RPAREN)*; +prewhereClause: PREWHERE columnExpr; +whereClause: WHERE columnExpr; +groupByClause: GROUP BY ( + ALL + | (CUBE | ROLLUP) LPAREN columnExprList RPAREN + | GROUPING SETS LPAREN groupingSetList RPAREN + | columnExprList + ); +groupingSetList: groupingSet (COMMA groupingSet)*; +groupingSet: LPAREN columnExprList? RPAREN; +havingClause: HAVING columnExpr; +qualifyClause: QUALIFY columnExpr; +orderByClause: ORDER BY orderExprList interpolateClause?; +interpolateClause: INTERPOLATE (LPAREN interpolateExpr (COMMA interpolateExpr)* RPAREN)?; +projectionOrderByClause: ORDER BY columnExprList; +limitByClause: LIMIT limitExpr BY columnExprList; +limitAndOffsetClause + : LIMIT columnExpr PERCENT? (COMMA columnExpr)? (WITH TIES)? // compact OFFSET-optional form + | LIMIT columnExpr PERCENT? (WITH TIES)? OFFSET columnExpr // verbose OFFSET-included form with WITH TIES + ; +offsetOnlyClause: OFFSET columnExpr; +settingsClause: SETTINGS settingExprList; + +valuesClause: VALUES valuesRow (COMMA valuesRow)*; +valuesRow: LPAREN columnExpr (COMMA columnExpr)* RPAREN; + +joinExpr + : joinExpr NATURAL? joinOp? JOIN joinExpr joinConstraintClause? # JoinExprOp + | joinExpr POSITIONAL JOIN joinExpr joinConstraintClause? # JoinExprPositional + | joinExpr joinOpCross joinExpr # JoinExprCrossOp + | joinExpr PIVOT LPAREN columnExprList pivotColumnList (GROUP BY columnExprList)? RPAREN # JoinExprPivot + | joinExpr UNPIVOT (INCLUDE NULLS)? LPAREN unpivotColumnList RPAREN # JoinExprUnpivot + | tableExpr FINAL? sampleClause? # JoinExprTable + | LPAREN joinExpr RPAREN # JoinExprParens + ; +joinOp + : ((ALL | ANY | ASOF)? INNER | INNER (ALL | ANY | ASOF)? | (ALL | ANY | ASOF) | ANTI | SEMI | ASOF (ANTI | SEMI)) # JoinOpInner + | ( (SEMI | ALL | ANTI | ANY | ASOF)? (LEFT | RIGHT) OUTER? + | (LEFT | RIGHT) OUTER? (SEMI | ALL | ANTI | ANY | ASOF)? + | ASOF (ANTI | SEMI) (LEFT | RIGHT) OUTER? + ) # JoinOpLeftRight + | ((ALL | ANY | ASOF)? FULL OUTER? | FULL OUTER? (ALL | ANY | ASOF)?) # JoinOpFull + ; +joinOpCross + : CROSS JOIN + | COMMA + ; +joinConstraintClause + : ON columnExprList + | USING LPAREN columnExprList RPAREN + | USING columnExprList + ; + +sampleClause: SAMPLE ratioExpr PERCENT? (OFFSET ratioExpr)? (LPAREN identifier RPAREN)?; +limitExpr: columnExpr ((COMMA | OFFSET) columnExpr)?; +orderExprList: orderExpr (COMMA orderExpr)*; +orderExpr: columnExpr (ASCENDING | DESCENDING | DESC)? (NULLS (FIRST | LAST))? (COLLATE STRING_LITERAL)? withFillClause?; +withFillClause: WITH FILL (FROM columnExpr)? (TO columnExpr)? (STEP columnExpr)?; +interpolateExpr: columnExpr (AS columnExpr)?; +ratioExpr: placeholder | numberLiteral (SLASH numberLiteral)?; +settingExprList: settingExpr (COMMA settingExpr)*; +settingExpr: identifier EQ_SINGLE literal; + +windowExpr: winPartitionByClause? winOrderByClause? winFrameClause?; +winPartitionByClause: PARTITION BY columnExprList; +winOrderByClause: ORDER BY orderExprList; +withinGroupClause: WITHIN GROUP LPAREN orderByClause RPAREN; +winFrameClause: (ROWS | RANGE) winFrameExtend; +winFrameExtend + : winFrameBound # frameStart + | BETWEEN winFrameBound AND winFrameBound # frameBetween + ; +winFrameBound: (CURRENT ROW | UNBOUNDED PRECEDING | UNBOUNDED FOLLOWING | columnExpr PRECEDING | columnExpr FOLLOWING); +//rangeClause: RANGE LPAREN (MIN identifier MAX identifier | MAX identifier MIN identifier) RPAREN; + +// Columns +expr: columnExpr EOF; +columnTypeExpr + : columnTypeExpr LBRACKET DECIMAL_LITERAL? RBRACKET # ColumnTypeExprArray // INTEGER[], VARCHAR[3] + | identifier LPAREN identifier columnTypeExpr (COMMA identifier columnTypeExpr)* COMMA? RPAREN # ColumnTypeExprNested // Nested + | identifier LPAREN enumValue (COMMA enumValue)* COMMA? RPAREN # ColumnTypeExprEnum // Enum + | identifier LPAREN columnTypeExpr (COMMA columnTypeExpr)* COMMA? RPAREN # ColumnTypeExprComplex // Array, Tuple + | identifier LPAREN columnExprList? RPAREN # ColumnTypeExprParam // FixedString(N) + | identifier identifier+ # ColumnTypeExprCompound // TIME WITH TIME ZONE + | identifier # ColumnTypeExprSimple // UInt64 + ; +// Restricted type expr for :: casts — no parenthesized variants to avoid ambiguity with function calls +columnTypeCastExpr + : columnTypeCastIdentifier WITH LOCAL? TIME ZONE # ColumnTypeCastExprWithTimeZone + | columnTypeCastIdentifier # ColumnTypeCastExprSimple + ; +columnTypeCastIdentifier + : IDENTIFIER + | QUOTED_IDENTIFIER + | interval + | keywordForTypeCast + ; +keywordForTypeCast + : DATE + | TIME + | TIMESTAMP + | INTERVAL + ; +columnExprList: columnExpr (COMMA columnExpr)* COMMA?; +selectColumnExprListBeforeFrom + : selectColumnExpr (COMMA selectColumnExpr)* COMMA # SelectColumnExprListBeforeFromTrailingComma + | selectColumnExprList # SelectColumnExprListBeforeFromPlain + ; +selectColumnExprList: selectColumnExpr (COMMA selectColumnExpr)* COMMA?; +selectColumnExpr + : identifier COLON columnExpr # ColumnExprAliasBefore + | FROM implicitAlias # ColumnExprInvalidFromImplicitAlias + | columnExpr # ColumnExprSelectValue + | columnExpr implicitAlias # ColumnExprAliasImplicit + ; +// Two precedence layers. `columnExpr` is the outer boolean/ternary/alias tier (loosest +// binding). `columnExprValue` holds everything tighter — arithmetic, comparisons, NOT, +// BETWEEN, and the primary/leaf productions. BETWEEN lives in the value tier at the comparison +// level, so its tested expression and both bounds bind tighter than AND/OR/NOT (matching +// ClickHouse and SQL); this is what fixes `a BETWEEN low AND high AND rest` grouping as +// `(a BETWEEN low AND high) AND rest` rather than letting the bounds swallow the AND chain. +// Splitting into two rules is the only way to express this in ANTLR4: the interior operand of +// a left-recursive alternative always parses at precedence 0, so a flat rule cannot stop the +// bounds from consuming AND (the reason for ilezhankin's original TODO). NOT stays in +// `columnExprValue` (looser than every value operator, tighter than AND/OR) so it keeps sitting +// *after* `ColumnExprFunction` in the same rule — ANTLR then still prefers the function form for +// `not(args)` (a `not` function call) over the `NOT (args)` operator, matching the old grammar. +columnExpr + : columnExpr AND columnExpr # ColumnExprAnd + | columnExpr OR columnExpr # ColumnExprOr + | columnExpr QUERY columnExpr COLON columnExpr # ColumnExprTernaryOp + | columnExpr AS (identifier | STRING_LITERAL) # ColumnExprAlias + | columnExprValue # ColumnExprValuePassthrough + ; + +columnExprValue + : CASE caseExpr=columnExpr? (WHEN whenExpr=columnExpr THEN thenExpr=columnExpr)+ (ELSE elseExpr=columnExpr)? END # ColumnExprCase + | CAST LPAREN columnExpr AS columnTypeExpr RPAREN # ColumnExprCast + | TRY_CAST LPAREN columnExpr AS columnTypeExpr RPAREN # ColumnExprTryCast + | DATE STRING_LITERAL # ColumnExprDate +// | EXTRACT LPAREN interval FROM columnExpr RPAREN # ColumnExprExtract // Interferes with a function call + | INTERVAL columnExpr interval # ColumnExprInterval + | INTERVAL STRING_LITERAL # ColumnExprIntervalString + | SUBSTRING LPAREN columnExpr FROM columnExpr (FOR columnExpr)? RPAREN # ColumnExprSubstring + | TIMESTAMP STRING_LITERAL # ColumnExprTimestamp + | TRIM LPAREN (BOTH | LEADING | TRAILING) string FROM columnExpr RPAREN # ColumnExprTrim + | COLUMNS LPAREN STRING_LITERAL RPAREN # ColumnExprColumnsRegex + | COLUMNS LPAREN columnExprList RPAREN # ColumnExprColumnsList + | (COLUMNS LPAREN ASTERISK EXCLUDE LPAREN identifierList RPAREN REPLACE LPAREN columnsReplaceList RPAREN RPAREN + | LPAREN ASTERISK EXCLUDE LPAREN identifierList RPAREN REPLACE LPAREN columnsReplaceList RPAREN RPAREN + ) # ColumnExprColumnsExcludeReplace + | COLUMNS LPAREN ASTERISK EXCLUDE LPAREN identifierList RPAREN RPAREN # ColumnExprColumnsExclude + | (COLUMNS LPAREN ASTERISK REPLACE LPAREN columnsReplaceList RPAREN RPAREN + | LPAREN ASTERISK REPLACE LPAREN columnsReplaceList RPAREN RPAREN + ) # ColumnExprColumnsReplace + | COLUMNS LPAREN ASTERISK RPAREN # ColumnExprColumnsAll + | COLUMNS LPAREN identifier DOT ASTERISK EXCLUDE LPAREN identifierList RPAREN REPLACE LPAREN columnsReplaceList RPAREN RPAREN # ColumnExprColumnsQualifiedExcludeReplace + | COLUMNS LPAREN identifier DOT ASTERISK EXCLUDE LPAREN identifierList RPAREN RPAREN # ColumnExprColumnsQualifiedExclude + | COLUMNS LPAREN identifier DOT ASTERISK REPLACE LPAREN columnsReplaceList RPAREN RPAREN # ColumnExprColumnsQualifiedReplace + | COLUMNS LPAREN identifier DOT ASTERISK RPAREN # ColumnExprColumnsQualifiedAll + | ASTERISK COLUMNS LPAREN STRING_LITERAL RPAREN # ColumnExprSpreadColumnsRegex + | ASTERISK COLUMNS LPAREN columnExprList RPAREN # ColumnExprSpreadColumnsList + | identifier LPAREN columnExprs=columnExprList? RPAREN withinGroupClause # ColumnExprFunctionWithinGroup + | identifier (LPAREN columnExprs=columnExprList? RPAREN) (LPAREN DISTINCT? columnArgList=columnExprList? RPAREN)? (FILTER LPAREN WHERE filterExpr=columnExpr RPAREN)? OVER LPAREN windowExpr RPAREN # ColumnExprWinFunction + | identifier (LPAREN columnExprs=columnExprList? RPAREN) (LPAREN DISTINCT? columnArgList=columnExprList? RPAREN)? (FILTER LPAREN WHERE filterExpr=columnExpr RPAREN)? OVER identifier # ColumnExprWinFunctionTarget + | identifier (LPAREN columnExprs=columnExprList? RPAREN)? LPAREN DISTINCT? columnArgList=columnExprList? (ORDER BY orderExprList)? RPAREN (FILTER LPAREN WHERE filterExpr=columnExpr RPAREN)? # ColumnExprFunction + | columnExprValue LPAREN selectSetStmt RPAREN # ColumnExprCallSelect + | columnExprValue LPAREN columnExprList? RPAREN # ColumnExprCall + | hogqlxTagElement # ColumnExprTagElement + | templateString # ColumnExprTemplateString + | literal # ColumnExprLiteral + + // FIXME(ilezhankin): this part looks very ugly, maybe there is another way to express it + | columnExprValue LBRACKET columnExpr RBRACKET # ColumnExprArrayAccess + | columnExprValue LBRACKET columnExpr? COLON columnExpr? RBRACKET # ColumnExprArraySlice + | columnExprValue DOT DECIMAL_LITERAL # ColumnExprTupleAccess + | columnExprValue DOT identifier # ColumnExprPropertyAccess + | columnExprValue NULL_PROPERTY LBRACKET columnExpr RBRACKET # ColumnExprNullArrayAccess + | columnExprValue NULL_PROPERTY DECIMAL_LITERAL # ColumnExprNullTupleAccess + | columnExprValue NULL_PROPERTY identifier # ColumnExprNullPropertyAccess + | columnExprValue DOUBLECOLON columnTypeCastExpr # ColumnExprTypeCast + | DASH columnExprValue # ColumnExprNegate + | left=columnExprValue ( operator=ASTERISK // * + | operator=SLASH // / + | operator=PERCENT // % + ) right=columnExprValue # ColumnExprPrecedence1 + | left=columnExprValue ( operator=PLUS // + + | operator=DASH // - + | operator=CONCAT // || + ) right=columnExprValue # ColumnExprPrecedence2 + | left=columnExprValue ( operator=EQ_DOUBLE // = + | operator=EQ_SINGLE // == + | operator=NOT_EQ // != + | operator=LT_EQ // <= + | operator=LT // < + | operator=GT_EQ // >= + | operator=GT // > + | operator=NOT? IN COHORT? // in, not in; in cohort; not in cohort + | operator=NOT? (LIKE | ILIKE) // like, not like, ilike, not ilike + | operator=REGEX_SINGLE // ~ + | operator=REGEX_DOUBLE // =~ + | operator=NOT_REGEX // !~ + | operator=IREGEX_SINGLE // ~* + | operator=IREGEX_DOUBLE // =~* + | operator=NOT_IREGEX // !~* + ) right=columnExprValue # ColumnExprPrecedence3 + | columnExprValue IGNORE NULLS # ColumnExprIgnoreNulls + | columnExprValue IS NOT? NULL_SQL # ColumnExprIsNull + | columnExprValue IS NOT? DISTINCT FROM columnExprValue # ColumnExprIsDistinctFrom + | columnExprValue NULL_SAFE_EQ columnExprValue # ColumnExprNullSafeEq // MySQL `a <=> b` ≡ `a IS NOT DISTINCT FROM b` + | columnExprValue NULLISH columnExprValue # ColumnExprNullish + | columnExprValue NOT? BETWEEN columnExprValue AND columnExprValue # ColumnExprBetween + | NOT columnExprValue # ColumnExprNot + | (tableIdentifier DOT)? ASTERISK (EXCLUDE LPAREN identifierList RPAREN)? # ColumnExprAsterisk // single-column only + | LAMBDA identifier (COMMA identifier)* COMMA? COLON columnExpr # ColumnExprColonLambda + | LPAREN selectSetStmt RPAREN # ColumnExprSubquery // single-column only + | LPAREN columnExpr RPAREN # ColumnExprParens // single-column only + | LPAREN columnExprList RPAREN # ColumnExprTuple + | ARRAY? LBRACKET columnExprList? RBRACKET # ColumnExprArray + | LBRACE (kvPairList)? RBRACE # ColumnExprDict + | columnLambdaExpr # ColumnExprLambda + | identifier COLONEQUALS columnExpr # ColumnExprNamedArg + | HASH DECIMAL_LITERAL # ColumnExprPositional + | columnIdentifier # ColumnExprIdentifier + ; + +columnLambdaExpr: + ( LPAREN identifier (COMMA identifier)* COMMA? RPAREN + | identifier (COMMA identifier)* COMMA? + | LPAREN RPAREN + ) + ARROW (columnExpr | block) # ArrowLambda + | LAMBDA identifier (COMMA identifier)* COMMA? COLON columnExpr # ColonLambda + ; + +columnsReplaceList: columnsReplaceItem (COMMA columnsReplaceItem)*; +columnsReplaceItem: columnExpr AS identifier; + +hogqlxChildElement + : hogqlxTagElement + | hogqlxText + | LBRACE columnExpr RBRACE; + +hogqlxText : HOGQLX_TEXT_TEXT ; + +hogqlxTagElement + : LT identifier hogqlxTagAttribute* SLASH_GT # HogqlxTagElementClosed + | LT identifier hogqlxTagAttribute* GT hogqlxChildElement* LT_SLASH identifier GT # HogqlxTagElementNested + ; +hogqlxTagAttribute + : identifier EQ_SINGLE string + | identifier EQ_SINGLE LBRACE columnExpr RBRACE + | identifier + ; + +withExprList: withExpr (COMMA withExpr)* COMMA?; +withExpr + : identifier withExprColumnNameList? (USING KEY withExprColumnNameList)? AS (NOT? MATERIALIZED)? LPAREN selectSetStmt RPAREN # WithExprSubquery + // NOTE: asterisk and subquery goes before |columnExpr| so that we can mark them as multi-column expressions. + | columnExpr AS identifier # WithExprColumn + ; + +withExprColumnNameList: LPAREN identifier (COMMA identifier)* RPAREN; + + +// This is slightly different in HogQL compared to ClickHouse SQL +// HogQL allows unlimited ("*") nestedIdentifier-s "properties.b.a.a.w.a.s". +// We parse and convert "databaseIdentifier.tableIdentifier.columnIdentifier.nestedIdentifier.*" +// to just one ast.Field(chain=['a','b','columnIdentifier','on','and','on']). +columnIdentifier: placeholder | ((tableIdentifier DOT)? nestedIdentifier); +nestedIdentifier: identifier (DOT identifier)*; +tableExpr + : tableIdentifier # TableExprIdentifier + | tableFunctionExpr # TableExprFunction + | LPAREN selectSetStmt RPAREN # TableExprSubquery + | LPAREN valuesClause RPAREN # TableExprValues + | tableExpr PIVOT LPAREN columnExprList pivotColumnList (GROUP BY columnExprList)? RPAREN # TableExprPivot + | tableExpr UNPIVOT (INCLUDE NULLS)? LPAREN unpivotColumnList RPAREN # TableExprUnpivot + | tableExpr (alias | AS identifier) columnAliases? # TableExprAlias + | hogqlxTagElement # TableExprTag + | placeholder # TableExprPlaceholder + ; + +pivotColumnList: FOR pivotColumn+; +pivotColumn: columnExprTupleOrSingle IN LPAREN columnExprList RPAREN; +unpivotColumnList: unpivotColumn (COMMA unpivotColumn)* COMMA?; +unpivotColumn: columnExprTupleOrSingle FOR columnExprTupleOrSingle IN LPAREN columnExprList RPAREN (columnExprTupleOrSingle IN LPAREN columnExprList RPAREN)*; +columnExprTupleOrSingle: LPAREN columnExprList RPAREN | columnExpr; +columnAliases: LPAREN identifier (COMMA identifier)* RPAREN; +tableFunctionExpr: identifier LPAREN tableArgList? RPAREN; +tableIdentifier: (databaseIdentifier DOT)? nestedIdentifier; +tableArgList: columnExpr (COMMA columnExpr)* COMMA?; + +// Databases + +databaseIdentifier: identifier; + +// Basics + +floatingLiteral + : FLOATING_LITERAL + | DOT (DECIMAL_LITERAL | OCTAL_LITERAL) + | DECIMAL_LITERAL DOT (DECIMAL_LITERAL | OCTAL_LITERAL)? // can't move this to the lexer or it will break nested tuple access: t.1.2 + ; +numberLiteral: (PLUS | DASH)? (floatingLiteral | BINARY_LITERAL | OCTAL_LITERAL | OCTAL_PREFIX_LITERAL | DECIMAL_LITERAL | HEXADECIMAL_LITERAL | INF | NAN_SQL); +literal + : numberLiteral + | STRING_LITERAL + | NULL_SQL + ; +interval: SECOND | MINUTE | HOUR | DAY | WEEK | MONTH | QUARTER | YEAR; +keyword + // except NULL_SQL, INF, NAN_SQL + : ALL | AND | ANTI | ANY | ARRAY | AS | ASCENDING | ASOF | BETWEEN | BOTH | BY | CASE + | CAST | COHORT | COLLATE | COLUMNS | CROSS | CUBE | CURRENT | DATE | DESC | DESCENDING + | DISTINCT | ELSE | END | EXCLUDE | EXTRACT | FILL | FILTER | FINAL | FIRST + | FOR | FOLLOWING | FROM | FULL | GROUP | HAVING | ID | INTERPOLATE | IS + | GROUPING | IF | IGNORE | ILIKE | INCLUDE | IN | INNER | INTERVAL | JOIN | KEY + | LAMBDA | LAST | LEADING | LEFT | LIKE | LIMIT + | LOCAL | NAME | NATURAL | NOT | NULLS | OFFSET | ON | OR | ORDER | OUTER | OVER | PARTITION + | PIVOT | POSITIONAL | PRECEDING | PREWHERE | QUALIFY | RANGE | RECURSIVE | REPLACE | RETURN | RIGHT | ROLLUP | ROW + | ROWS | SAMPLE | SELECT | SEMI | SETS | SETTINGS | STEP | SUBSTRING + | THEN | TIES | TIME | TIMESTAMP | TOTALS | TRAILING | TRIM | TRUNCATE | TRY_CAST | TO | TOP + | UNBOUNDED | UNION | UNPIVOT | USING | VALUES | WHEN | WHERE | WINDOW | WITH + | ZONE + ; +keywordForAlias + : DATE | FIRST | ID | KEY + ; +keywordForImplicitAlias + : ASCENDING + | COHORT + | DATE + | DESCENDING + | FINAL + | ID + | RETURN + | TOP + | TOTALS + ; +alias: IDENTIFIER | QUOTED_IDENTIFIER | keywordForAlias; // |interval| can't be an alias, otherwise 'INTERVAL 1 SOMETHING' becomes ambiguous. +implicitAlias: IDENTIFIER | QUOTED_IDENTIFIER | keywordForImplicitAlias; +identifier: IDENTIFIER | QUOTED_IDENTIFIER | interval | keyword; +enumValue: string EQ_SINGLE numberLiteral; +placeholder: LBRACE columnExpr RBRACE; + +string: STRING_LITERAL | templateString; +templateString : QUOTE_SINGLE_TEMPLATE stringContents* QUOTE_SINGLE ; +stringContents : STRING_ESCAPE_TRIGGER columnExpr RBRACE | STRING_TEXT; + +// These are magic "full template strings", which are used to parse "full text field" templates without the surrounding SQL. +// We will need to add F' to the start of the string to change the lexer's mode. +fullTemplateString: QUOTE_SINGLE_TEMPLATE_FULL stringContentsFull* EOF ; +stringContentsFull : FULL_STRING_ESCAPE_TRIGGER columnExpr RBRACE | FULL_STRING_TEXT; diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/language.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/language.json new file mode 100644 index 000000000000..eac6113fbd90 --- /dev/null +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/language.json @@ -0,0 +1,40 @@ +{ + "antlrVersion": "4.13.2", + "canonicalParser": "cpp-antlr", + "corpora": [ + { + "manifestPath": "corpus/expr_select_cases.json", + "manifestSha256": "100c08ef64c9a02b956f5bbaca265c3b47993acf26e000e1812f110ae5112a3d", + "oraclePath": "corpus/expr_select_cpp_oracle.jsonl", + "oracleSha256": "c45fa533a9fb9cb80b7a4009fc468b6e1a76ef1ac347220a0feb5a91453c500b", + "schemaVersion": 1, + "slice": "expression-and-plain-select" + } + ], + "entryPoints": { + "expression": "expr", + "query": "select" + }, + "files": [ + { + "path": "HogQLLexer.common.g4", + "sha256": "f5a92ca0c19267ee8ded72fe4f5ca05939aea4bb4e1d6ee3b6bf095622adf268" + }, + { + "path": "HogQLLexer.java.g4", + "sha256": "04f799086dfa92aacf7caa30c2e1bf75032118c61fe7eb1d80b8dddba7ad3afc" + }, + { + "path": "HogQLParser.g4", + "sha256": "4cc3cf6f6f8d2584df7cae12a31d546c6bffef3e0e85d7101e7749d6bd0f5430" + } + ], + "grammarFeatureManifest": { + "path": "grammar-features.json", + "schemaVersion": 1, + "sha256": "fbf7236643b43d929e9ef1c8fd01a04b746736dca2f965cc2e78ad62f1a98b10" + }, + "grammarSha256": "c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242", + "languageVersion": "1.0.0", + "schemaVersion": 1 +} diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/source-lock.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/source-lock.json new file mode 100644 index 000000000000..f223a0d77807 --- /dev/null +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/source-lock.json @@ -0,0 +1,6 @@ +{ + "descriptorSha256": "4abfb6ced4fcdeb77aac3e2b5694913ad123a2a7cbc8d8f18a9e586b038c9de3", + "schemaVersion": 1, + "sourceRepository": "https://github.com/PostHog/posthog", + "sourceRevision": "f55d075d51ca2cc756ef2000699ac37dffa2c25c" +} diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json new file mode 100644 index 000000000000..c4868978b0ad --- /dev/null +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json @@ -0,0 +1,954 @@ +{ + "corpusCases": [ + { + "id": "expr.accept.arithmetic", + "status": "supported" + }, + { + "id": "expr.accept.array", + "status": "supported" + }, + { + "id": "expr.accept.between", + "status": "supported" + }, + { + "id": "expr.accept.boolean", + "status": "supported" + }, + { + "id": "expr.accept.case", + "status": "supported" + }, + { + "id": "expr.accept.cast", + "status": "supported" + }, + { + "id": "expr.accept.dotted-field", + "status": "supported" + }, + { + "id": "expr.accept.function", + "status": "supported" + }, + { + "id": "expr.accept.in-list", + "status": "supported" + }, + { + "id": "expr.accept.literal-boolean", + "status": "supported" + }, + { + "id": "expr.accept.literal-null", + "status": "supported" + }, + { + "id": "expr.accept.literal-number", + "status": "supported" + }, + { + "id": "expr.accept.literal-string", + "status": "supported" + }, + { + "id": "expr.accept.null-predicate", + "status": "supported" + }, + { + "id": "expr.accept.tuple", + "status": "supported" + }, + { + "id": "expr.reject.incomplete-binary", + "status": "canonicalRejection" + }, + { + "id": "expr.reject.unclosed-tuple", + "status": "canonicalRejection" + }, + { + "id": "expr.reject.unexpected-character", + "status": "canonicalRejection" + }, + { + "id": "query.accept.aggregate", + "status": "supported" + }, + { + "id": "query.accept.constant", + "status": "supported" + }, + { + "id": "query.accept.multiline", + "status": "supported" + }, + { + "id": "query.accept.order-limit-offset", + "status": "supported" + }, + { + "id": "query.accept.projection-from", + "status": "supported" + }, + { + "id": "query.accept.where", + "status": "supported" + }, + { + "id": "query.reject.missing-projection", + "status": "canonicalRejection" + }, + { + "id": "query.reject.missing-source", + "status": "canonicalRejection" + }, + { + "id": "query.reject.missing-where-expression", + "status": "canonicalRejection" + }, + { + "id": "query.reject.trailing-comparison", + "status": "canonicalRejection" + } + ], + "features": [ + { + "id": "alternative:alias:keyword", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:alias:quoted-identifier", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:alias:unquoted-identifier", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:columnAliases", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, + { + "id": "alternative:columnExpr:ColumnExprAnd", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlParser#testUsesCanonicalGrammarForExpressionsAndWhereClause", + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:columnExpr:ColumnExprOr", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.boolean" + ] + }, + { + "id": "alternative:columnExpr:ColumnExprValuePassthrough", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:columnExprList", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpressions", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.array", + "TestHogQlCompatibilityCorpus#expr.accept.tuple" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprArray", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.array", + "TestHogQlCompiler#testLowersCollectionAndPredicateExpressions" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprArrayAccess", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlParser#testBuildsCollectionAccessWithSourceSpans", + "TestHogQlCompiler#testLowersCollectionAndPredicateExpressions" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprBetween", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.between", + "TestHogQlCompiler#testLowersCollectionAndPredicateExpressions" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprCase", + "loweringHandler": "TrinoAstFactory#createCaseExpression", + "parseHandler": "AstBuilder#buildCaseExpression", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.case", + "TestHogQlCompiler#testLowersCaseAndCastExpressions" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprCast", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.cast", + "TestHogQlCompiler#testLowersCaseAndCastExpressions" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprIsNull", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.null-predicate", + "TestHogQlCompiler#testLowersCollectionAndPredicateExpressions" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprNegate", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprNot", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlParser#testUsesCanonicalGrammarForExpressionsAndWhereClause", + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprParens", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.boolean" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprPropertyAccess", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlParser#testBuildsCollectionAccessWithSourceSpans", + "TestHogQlCompiler#testLowersCollectionAndPredicateExpressions" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprPrecedence1", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlParser#testUsesCanonicalGrammarForExpressionsAndWhereClause", + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprTryCast", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCaseAndCastExpressions" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprTuple", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.tuple", + "TestHogQlCompiler#testLowersCollectionAndPredicateExpressions" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprTupleAccess", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlParser#testBuildsCollectionAccessWithSourceSpans", + "TestHogQlCompiler#testLowersCollectionAndPredicateExpressions" + ] + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprSimple", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildSimpleType", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.cast", + "TestHogQlCompiler#testLowersCaseAndCastExpressions" + ] + }, + { + "id": "alternative:databaseIdentifier", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifiers", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "alternative:fromClause", + "loweringHandler": "TrinoAstFactory#createTable", + "parseHandler": "AstBuilder#buildTableReference", + "testCaseIds": [ + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:identifier:interval-keyword", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifier", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "alternative:identifier:keyword", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifier", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "alternative:identifier:quoted", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifier", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "alternative:identifier:unquoted", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifier", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "alternative:joinConstraintClause:on", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildJoinCriteria", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:joinConstraintClause:using", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildJoinCriteria", + "testCaseIds": [ + "TestHogQlCompiler#testLowersUnparenthesizedUsing" + ] + }, + { + "id": "alternative:joinConstraintClause:using-parenthesized", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildJoinCriteria", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:joinExpr:JoinExprCrossOp", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildRelation", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:joinExpr:JoinExprOp", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildRelation", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:joinExpr:JoinExprParens", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildRelation", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:joinExpr:JoinExprTable", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildRelation", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:joinOp:JoinOpFull", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildJoinType", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:joinOp:JoinOpInner", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildJoinType", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:joinOp:JoinOpLeftRight", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildJoinType", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:joinOpCross:cross-join", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildRelation", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:keywordForAlias", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:nestedIdentifier", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifiers", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "alternative:selectColumnExpr:ColumnExprSelectValue", + "loweringHandler": "TrinoAstFactory#createSelectItem", + "parseHandler": "AstBuilder#buildProjection", + "testCaseIds": [ + "TestHogQlCompiler#testReturnsOnlyStockTrinoTreeNodes" + ] + }, + { + "id": "alternative:selectColumnExprList", + "loweringHandler": "TrinoAstFactory#createSelectItem", + "parseHandler": "AstBuilder#selectColumns", + "testCaseIds": [ + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:selectColumnExprListBeforeFrom:SelectColumnExprListBeforeFromPlain", + "loweringHandler": "TrinoAstFactory#createSelectItem", + "parseHandler": "AstBuilder#selectColumns", + "testCaseIds": [ + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:selectSetStmt", + "loweringHandler": "TrinoAstFactory#createQueryBody", + "parseHandler": "AstBuilder#buildSetQuery", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, + { + "id": "alternative:selectStmtWithParens:direct", + "loweringHandler": "TrinoAstFactory#createQuery", + "parseHandler": "AstBuilder#extractPlainSelect", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCtesAndDerivedTables" + ] + }, + { + "id": "alternative:selectStmtWithParens:parenthesized", + "loweringHandler": "TrinoAstFactory#createQuery", + "parseHandler": "AstBuilder#extractPlainSelect", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCtesAndDerivedTables" + ] + }, + { + "id": "alternative:subsequentSelectSetClause", + "loweringHandler": "TrinoAstFactory#createQueryBody", + "parseHandler": "AstBuilder#buildSetQuery", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, + { + "id": "alternative:tableExpr:TableExprAlias", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins", + "TestHogQlCompiler#testLowersCtesAndDerivedTables" + ] + }, + { + "id": "alternative:tableExpr:TableExprIdentifier", + "loweringHandler": "TrinoAstFactory#createTable", + "parseHandler": "AstBuilder#buildTableExpression", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "alternative:tableExpr:TableExprSubquery", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCtesAndDerivedTables" + ] + }, + { + "id": "alternative:tableExpr:TableExprValues", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, + { + "id": "alternative:tableIdentifier", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifiers", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "alternative:valuesClause", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, + { + "id": "alternative:valuesRow", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, + { + "id": "alternative:whereClause", + "loweringHandler": "TrinoAstFactory#createStatement", + "parseHandler": "AstBuilder#build", + "testCaseIds": [ + "TestHogQlParser#testUsesCanonicalGrammarForExpressionsAndWhereClause" + ] + }, + { + "id": "alternative:withClause", + "loweringHandler": "TrinoAstFactory#createWith", + "parseHandler": "AstBuilder#buildCommonTables", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCtesAndDerivedTables" + ] + }, + { + "id": "alternative:withExpr:WithExprSubquery", + "loweringHandler": "TrinoAstFactory#createWithQuery", + "parseHandler": "AstBuilder#buildCommonTables", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCtesAndDerivedTables" + ] + }, + { + "id": "alternative:withExprColumnNameList", + "loweringHandler": "TrinoAstFactory#createWithQuery", + "parseHandler": "AstBuilder#buildCommonTables", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCtesAndDerivedTables" + ] + }, + { + "id": "alternative:withExprList", + "loweringHandler": "TrinoAstFactory#createWith", + "parseHandler": "AstBuilder#buildCommonTables", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCtesAndDerivedTables" + ] + }, + { + "id": "rule:alias", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "rule:columnAliases", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, + { + "id": "rule:columnExprList", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpressions", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.array", + "TestHogQlCompatibilityCorpus#expr.accept.tuple" + ] + }, + { + "id": "rule:databaseIdentifier", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifiers", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "rule:fromClause", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildRelation", + "testCaseIds": [ + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "rule:identifier", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifier", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "rule:joinConstraintClause", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildJoinCriteria", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins", + "TestHogQlCompiler#testLowersUnparenthesizedUsing" + ] + }, + { + "id": "rule:joinOp", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildJoinType", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "rule:keywordForAlias", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "rule:nestedIdentifier", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifiers", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "rule:selectColumnExprList", + "loweringHandler": "TrinoAstFactory#createSelectItem", + "parseHandler": "AstBuilder#selectColumns", + "testCaseIds": [ + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "rule:selectColumnExprListBeforeFrom", + "loweringHandler": "TrinoAstFactory#createSelectItem", + "parseHandler": "AstBuilder#selectColumns", + "testCaseIds": [ + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "rule:selectSetStmt", + "loweringHandler": "TrinoAstFactory#createQueryBody", + "parseHandler": "AstBuilder#buildSetQuery", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, + { + "id": "rule:subsequentSelectSetClause", + "loweringHandler": "TrinoAstFactory#createQueryBody", + "parseHandler": "AstBuilder#buildSetQuery", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, + { + "id": "rule:tableIdentifier", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifiers", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "rule:valuesClause", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, + { + "id": "rule:valuesRow", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, + { + "id": "rule:whereClause", + "loweringHandler": "TrinoAstFactory#createStatement", + "parseHandler": "AstBuilder#build", + "testCaseIds": [ + "TestHogQlParser#testUsesCanonicalGrammarForExpressionsAndWhereClause" + ] + }, + { + "id": "rule:withClause", + "loweringHandler": "TrinoAstFactory#createWith", + "parseHandler": "AstBuilder#buildCommonTables", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCtesAndDerivedTables" + ] + }, + { + "id": "rule:withExprColumnNameList", + "loweringHandler": "TrinoAstFactory#createWithQuery", + "parseHandler": "AstBuilder#buildCommonTables", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCtesAndDerivedTables" + ] + }, + { + "id": "rule:withExprList", + "loweringHandler": "TrinoAstFactory#createWith", + "parseHandler": "AstBuilder#buildCommonTables", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCtesAndDerivedTables" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprWinFunction", + "loweringHandler": "TrinoAstFactory#createFunctionCall", + "parseHandler": "AstBuilder#buildWindowFunction", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprWinFunctionTarget", + "loweringHandler": "TrinoAstFactory#createFunctionCall", + "parseHandler": "AstBuilder#buildWindowFunction", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:winFrameBound", + "loweringHandler": "TrinoAstFactory#createFrameBound", + "parseHandler": "AstBuilder#buildFrameBound", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersEveryCanonicalFrameBound" + ] + }, + { + "id": "alternative:winFrameClause", + "loweringHandler": "TrinoAstFactory#createWindowFrame", + "parseHandler": "AstBuilder#buildWindowFrame", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersEveryCanonicalFrameBound" + ] + }, + { + "id": "alternative:winFrameExtend:frameBetween", + "loweringHandler": "TrinoAstFactory#createWindowFrame", + "parseHandler": "AstBuilder#buildWindowFrame", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersEveryCanonicalFrameBound" + ] + }, + { + "id": "alternative:winFrameExtend:frameStart", + "loweringHandler": "TrinoAstFactory#createWindowFrame", + "parseHandler": "AstBuilder#buildWindowFrame", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersEveryCanonicalFrameBound" + ] + }, + { + "id": "alternative:winOrderByClause", + "loweringHandler": "TrinoAstFactory#createWindow", + "parseHandler": "AstBuilder#buildWindowSpecification", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:winPartitionByClause", + "loweringHandler": "TrinoAstFactory#createWindow", + "parseHandler": "AstBuilder#buildWindowSpecification", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:windowClause", + "loweringHandler": "TrinoAstFactory#createWindowDefinition", + "parseHandler": "AstBuilder#buildWindowDefinitions", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:windowExpr", + "loweringHandler": "TrinoAstFactory#createWindow", + "parseHandler": "AstBuilder#buildWindowSpecification", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" + ] + }, + { + "id": "rule:winFrameBound", + "loweringHandler": "TrinoAstFactory#createFrameBound", + "parseHandler": "AstBuilder#buildFrameBound", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersEveryCanonicalFrameBound" + ] + }, + { + "id": "rule:winFrameClause", + "loweringHandler": "TrinoAstFactory#createWindowFrame", + "parseHandler": "AstBuilder#buildWindowFrame", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersEveryCanonicalFrameBound" + ] + }, + { + "id": "rule:winFrameExtend", + "loweringHandler": "TrinoAstFactory#createWindowFrame", + "parseHandler": "AstBuilder#buildWindowFrame", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersEveryCanonicalFrameBound" + ] + }, + { + "id": "rule:winOrderByClause", + "loweringHandler": "TrinoAstFactory#createWindow", + "parseHandler": "AstBuilder#buildWindowSpecification", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" + ] + }, + { + "id": "rule:winPartitionByClause", + "loweringHandler": "TrinoAstFactory#createWindow", + "parseHandler": "AstBuilder#buildWindowSpecification", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" + ] + }, + { + "id": "rule:windowClause", + "loweringHandler": "TrinoAstFactory#createWindowDefinition", + "parseHandler": "AstBuilder#buildWindowDefinitions", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" + ] + }, + { + "id": "rule:windowExpr", + "loweringHandler": "TrinoAstFactory#createWindow", + "parseHandler": "AstBuilder#buildWindowSpecification", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprIgnoreNulls", + "loweringHandler": "TrinoAstFactory#createFunctionCall", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersCanonicalWindowNullTreatmentOrder" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprInterval", + "loweringHandler": "TrinoAstFactory#createIntervalExpression", + "parseHandler": "AstBuilder#buildExpression", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCanonicalIntervals", + "TestHogQlParser#testBuildsCanonicalIntervalExpressions" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprIntervalString", + "loweringHandler": "TrinoAstFactory#createIntervalExpression", + "parseHandler": "AstBuilder#buildStringInterval", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCanonicalIntervals", + "TestHogQlParser#testRejectsInvalidCombinedStringIntervals" + ] + } + ], + "schemaVersion": 1 +} diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json new file mode 100644 index 000000000000..a62ebf43b88c --- /dev/null +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-compatibility.json @@ -0,0 +1,6569 @@ +{ + "features": [ + { + "id": "alternative:alias:keyword", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:alias:quoted-identifier", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:alias:unquoted-identifier", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:arrayJoinClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:block", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:catchBlock", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnAliases", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, + { + "id": "alternative:columnExpr:ColumnExprAlias", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExpr:ColumnExprAnd", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlParser#testUsesCanonicalGrammarForExpressionsAndWhereClause", + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:columnExpr:ColumnExprOr", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.boolean" + ] + }, + { + "id": "alternative:columnExpr:ColumnExprTernaryOp", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExpr:ColumnExprValuePassthrough", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:columnExprList", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpressions", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.array", + "TestHogQlCompatibilityCorpus#expr.accept.tuple" + ] + }, + { + "id": "alternative:columnExprTupleOrSingle:scalar", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprTupleOrSingle:tuple", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprArray", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.array", + "TestHogQlCompiler#testLowersCollectionAndPredicateExpressions" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprArrayAccess", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlParser#testBuildsCollectionAccessWithSourceSpans", + "TestHogQlCompiler#testLowersCollectionAndPredicateExpressions" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprArraySlice", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprAsterisk", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprBetween", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.between", + "TestHogQlCompiler#testLowersCollectionAndPredicateExpressions" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprCall", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprCallSelect", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprCase", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createCaseExpression", + "parseHandler": "AstBuilder#buildCaseExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.case", + "TestHogQlCompiler#testLowersCaseAndCastExpressions" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprCast", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.cast", + "TestHogQlCompiler#testLowersCaseAndCastExpressions" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprColonLambda", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsAll", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsExclude", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsExcludeReplace", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsList", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsQualifiedAll", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsQualifiedExclude", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsQualifiedExcludeReplace", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsQualifiedReplace", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsRegex", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsReplace", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprDate", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprDict", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprFunction", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprFunctionWithinGroup", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprIdentifier", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprIgnoreNulls", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createFunctionCall", + "parseHandler": "AstBuilder#buildExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersCanonicalWindowNullTreatmentOrder" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprInterval", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createIntervalExpression", + "parseHandler": "AstBuilder#buildExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCanonicalIntervals", + "TestHogQlParser#testBuildsCanonicalIntervalExpressions" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprIntervalString", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createIntervalExpression", + "parseHandler": "AstBuilder#buildStringInterval", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCanonicalIntervals", + "TestHogQlParser#testRejectsInvalidCombinedStringIntervals" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprIsDistinctFrom", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprIsNull", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.null-predicate", + "TestHogQlCompiler#testLowersCollectionAndPredicateExpressions" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprLambda", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprLiteral", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprNamedArg", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprNegate", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprNot", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlParser#testUsesCanonicalGrammarForExpressionsAndWhereClause", + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprNullArrayAccess", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprNullPropertyAccess", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprNullSafeEq", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprNullTupleAccess", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprNullish", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprParens", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.boolean" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprPositional", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprPrecedence1", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlParser#testUsesCanonicalGrammarForExpressionsAndWhereClause", + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprPrecedence2", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprPrecedence3", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprPropertyAccess", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlParser#testBuildsCollectionAccessWithSourceSpans", + "TestHogQlCompiler#testLowersCollectionAndPredicateExpressions" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprSpreadColumnsList", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprSpreadColumnsRegex", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprSubquery", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprSubstring", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprTagElement", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprTemplateString", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprTimestamp", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprTrim", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprTryCast", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCaseAndCastExpressions" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprTuple", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.tuple", + "TestHogQlCompiler#testLowersCollectionAndPredicateExpressions" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprTupleAccess", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlParser#testBuildsCollectionAccessWithSourceSpans", + "TestHogQlCompiler#testLowersCollectionAndPredicateExpressions" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprTypeCast", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprWinFunction", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createFunctionCall", + "parseHandler": "AstBuilder#buildWindowFunction", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:columnExprValue:ColumnExprWinFunctionTarget", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createFunctionCall", + "parseHandler": "AstBuilder#buildWindowFunction", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:columnIdentifier:field-path", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnIdentifier:placeholder", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnLambdaExpr:ArrowLambda", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnLambdaExpr:ColonLambda", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnTypeCastExpr:ColumnTypeCastExprSimple", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnTypeCastExpr:ColumnTypeCastExprWithTimeZone", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnTypeCastIdentifier:interval-keyword", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnTypeCastIdentifier:quoted-identifier", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnTypeCastIdentifier:type-keyword", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnTypeCastIdentifier:unquoted-identifier", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprArray", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprComplex", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprCompound", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprEnum", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprNested", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprParam", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprSimple", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildSimpleType", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.cast", + "TestHogQlCompiler#testLowersCaseAndCastExpressions" + ] + }, + { + "id": "alternative:columnsReplaceItem", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:columnsReplaceList", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:databaseIdentifier", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifiers", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "alternative:declaration:statement", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:declaration:variable", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:emptyStmt", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:enumValue", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:expr", + "kind": "parserAlternative", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "alternative:exprStmt", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:expression", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:floatingLiteral:floating-token", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:floatingLiteral:leading-decimal-point", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:floatingLiteral:trailing-decimal-point", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:forInStmt", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:forStmt", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:fromClause", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createTable", + "parseHandler": "AstBuilder#buildTableReference", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:fullTemplateString", + "kind": "parserAlternative", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "alternative:funcStmt", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:groupByClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:groupingSet", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:groupingSetList", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:havingClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:hogqlxChildElement:expression", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:hogqlxChildElement:tag", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:hogqlxChildElement:text", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:hogqlxTagAttribute:boolean", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:hogqlxTagAttribute:expression", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:hogqlxTagAttribute:string", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:hogqlxTagElement:HogqlxTagElementClosed", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:hogqlxTagElement:HogqlxTagElementNested", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:hogqlxText", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:identifier:interval-keyword", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifier", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "alternative:identifier:keyword", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifier", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "alternative:identifier:quoted", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifier", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "alternative:identifier:unquoted", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifier", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "alternative:identifierList", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:ifStmt", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:implicitAlias:keyword", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:implicitAlias:quoted-identifier", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:implicitAlias:unquoted-identifier", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:interpolateClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:interpolateExpr", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:interval", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:joinConstraintClause:on", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildJoinCriteria", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:joinConstraintClause:using", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildJoinCriteria", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersUnparenthesizedUsing" + ] + }, + { + "id": "alternative:joinConstraintClause:using-parenthesized", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildJoinCriteria", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:joinExpr:JoinExprCrossOp", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildRelation", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:joinExpr:JoinExprOp", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildRelation", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:joinExpr:JoinExprParens", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildRelation", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:joinExpr:JoinExprPivot", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:joinExpr:JoinExprPositional", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:joinExpr:JoinExprTable", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildRelation", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:joinExpr:JoinExprUnpivot", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:joinOp:JoinOpFull", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildJoinType", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:joinOp:JoinOpInner", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildJoinType", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:joinOp:JoinOpLeftRight", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildJoinType", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:joinOpCross:comma", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:joinOpCross:cross-join", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildRelation", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:keyword", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:keywordForAlias", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "alternative:keywordForImplicitAlias", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:keywordForTypeCast", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:kvPair", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:kvPairList", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:limitAndOffsetClause:compact", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:limitAndOffsetClause:with-offset", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:limitAndOffsetClauseOptional:limit", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:limitAndOffsetClauseOptional:limit-with-offset", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:limitAndOffsetClauseOptional:offset-only", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:limitByClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:limitExpr", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:literal:null", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:literal:number", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:literal:string", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:nestedIdentifier", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifiers", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "alternative:numberLiteral", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:offsetOnlyClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:orderByClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:orderExpr", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:orderExprList", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:pivotColumn", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:pivotColumnList", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:placeholder", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:prewhereClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:program", + "kind": "parserAlternative", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "alternative:projectionOrderByClause", + "kind": "parserAlternative", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "alternative:qualifyClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:ratioExpr:numeric", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:ratioExpr:placeholder", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:returnStmt", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:sampleClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:select", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:selectColumnExpr:ColumnExprAliasBefore", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:selectColumnExpr:ColumnExprAliasImplicit", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:selectColumnExpr:ColumnExprInvalidFromImplicitAlias", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:selectColumnExpr:ColumnExprSelectValue", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createSelectItem", + "parseHandler": "AstBuilder#buildProjection", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testReturnsOnlyStockTrinoTreeNodes" + ] + }, + { + "id": "alternative:selectColumnExprList", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createSelectItem", + "parseHandler": "AstBuilder#selectColumns", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:selectColumnExprListBeforeFrom:SelectColumnExprListBeforeFromPlain", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createSelectItem", + "parseHandler": "AstBuilder#selectColumns", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:selectColumnExprListBeforeFrom:SelectColumnExprListBeforeFromTrailingComma", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:selectSetStmt", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createQueryBody", + "parseHandler": "AstBuilder#buildSetQuery", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, + { + "id": "alternative:selectStmt", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:selectStmtWithParens:direct", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createQuery", + "parseHandler": "AstBuilder#extractPlainSelect", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCtesAndDerivedTables" + ] + }, + { + "id": "alternative:selectStmtWithParens:parenthesized", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createQuery", + "parseHandler": "AstBuilder#extractPlainSelect", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCtesAndDerivedTables" + ] + }, + { + "id": "alternative:selectStmtWithParens:placeholder", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:selectStmtWithParens:with-parenthesized", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:settingExpr", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:settingExprList", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:settingsClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:statement:block", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:statement:empty", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:statement:expression", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:statement:for", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:statement:for-in", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:statement:function", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:statement:if", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:statement:return", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:statement:throw", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:statement:try-catch", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:statement:while", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:string:literal", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:string:template", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:stringContents:interpolation", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:stringContents:text", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:stringContentsFull:interpolation", + "kind": "parserAlternative", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "alternative:stringContentsFull:text", + "kind": "parserAlternative", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "alternative:subsequentSelectSetClause", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createQueryBody", + "parseHandler": "AstBuilder#buildSetQuery", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, + { + "id": "alternative:tableArgList", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:tableExpr:TableExprAlias", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins", + "TestHogQlCompiler#testLowersCtesAndDerivedTables" + ] + }, + { + "id": "alternative:tableExpr:TableExprFunction", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:tableExpr:TableExprIdentifier", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createTable", + "parseHandler": "AstBuilder#buildTableExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "alternative:tableExpr:TableExprPivot", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:tableExpr:TableExprPlaceholder", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:tableExpr:TableExprSubquery", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCtesAndDerivedTables" + ] + }, + { + "id": "alternative:tableExpr:TableExprTag", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:tableExpr:TableExprUnpivot", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:tableExpr:TableExprValues", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, + { + "id": "alternative:tableFunctionExpr", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:tableIdentifier", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifiers", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "alternative:templateString", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:throwStmt", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:topClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:tryCatchStmt", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:unpivotColumn", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:unpivotColumnList", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:valuesClause", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, + { + "id": "alternative:valuesRow", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, + { + "id": "alternative:varAssignment", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:varDecl", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:whereClause", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createStatement", + "parseHandler": "AstBuilder#build", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlParser#testUsesCanonicalGrammarForExpressionsAndWhereClause" + ] + }, + { + "id": "alternative:whileStmt", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:winFrameBound", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createFrameBound", + "parseHandler": "AstBuilder#buildFrameBound", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersEveryCanonicalFrameBound" + ] + }, + { + "id": "alternative:winFrameClause", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createWindowFrame", + "parseHandler": "AstBuilder#buildWindowFrame", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersEveryCanonicalFrameBound" + ] + }, + { + "id": "alternative:winFrameExtend:frameBetween", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createWindowFrame", + "parseHandler": "AstBuilder#buildWindowFrame", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersEveryCanonicalFrameBound" + ] + }, + { + "id": "alternative:winFrameExtend:frameStart", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createWindowFrame", + "parseHandler": "AstBuilder#buildWindowFrame", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersEveryCanonicalFrameBound" + ] + }, + { + "id": "alternative:winOrderByClause", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createWindow", + "parseHandler": "AstBuilder#buildWindowSpecification", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:winPartitionByClause", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createWindow", + "parseHandler": "AstBuilder#buildWindowSpecification", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:windowClause", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createWindowDefinition", + "parseHandler": "AstBuilder#buildWindowDefinitions", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:windowExpr", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createWindow", + "parseHandler": "AstBuilder#buildWindowSpecification", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" + ] + }, + { + "id": "alternative:withClause", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createWith", + "parseHandler": "AstBuilder#buildCommonTables", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCtesAndDerivedTables" + ] + }, + { + "id": "alternative:withExpr:WithExprColumn", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:withExpr:WithExprSubquery", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createWithQuery", + "parseHandler": "AstBuilder#buildCommonTables", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCtesAndDerivedTables" + ] + }, + { + "id": "alternative:withExprColumnNameList", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createWithQuery", + "parseHandler": "AstBuilder#buildCommonTables", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCtesAndDerivedTables" + ] + }, + { + "id": "alternative:withExprList", + "kind": "parserAlternative", + "loweringHandler": "TrinoAstFactory#createWith", + "parseHandler": "AstBuilder#buildCommonTables", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCtesAndDerivedTables" + ] + }, + { + "id": "alternative:withFillClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "alternative:withinGroupClause", + "kind": "parserAlternative", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:alias", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "rule:arrayJoinClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:block", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:catchBlock", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:columnAliases", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, + { + "id": "rule:columnExpr", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:columnExprList", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createExpression", + "parseHandler": "AstBuilder#buildExpressions", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompatibilityCorpus#expr.accept.array", + "TestHogQlCompatibilityCorpus#expr.accept.tuple" + ] + }, + { + "id": "rule:columnExprTupleOrSingle", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:columnExprValue", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:columnIdentifier", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:columnLambdaExpr", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:columnTypeCastExpr", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:columnTypeCastIdentifier", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:columnTypeExpr", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:columnsReplaceItem", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:columnsReplaceList", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:databaseIdentifier", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifiers", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "rule:declaration", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:emptyStmt", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:enumValue", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:expr", + "kind": "parserRule", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "rule:exprStmt", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:expression", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:floatingLiteral", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:forInStmt", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:forStmt", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:fromClause", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildRelation", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "rule:fullTemplateString", + "kind": "parserRule", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "rule:funcStmt", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:groupByClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:groupingSet", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:groupingSetList", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:havingClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:hogqlxChildElement", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:hogqlxTagAttribute", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:hogqlxTagElement", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:hogqlxText", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:identifier", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifier", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "rule:identifierList", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:ifStmt", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:implicitAlias", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:interpolateClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:interpolateExpr", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:interval", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:joinConstraintClause", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildJoinCriteria", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins", + "TestHogQlCompiler#testLowersUnparenthesizedUsing" + ] + }, + { + "id": "rule:joinExpr", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:joinOp", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildJoinType", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "rule:joinOpCross", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:keyword", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:keywordForAlias", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersAliasesAndStockJoins" + ] + }, + { + "id": "rule:keywordForImplicitAlias", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:keywordForTypeCast", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:kvPair", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:kvPairList", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:limitAndOffsetClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:limitAndOffsetClauseOptional", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:limitByClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:limitExpr", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:literal", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:nestedIdentifier", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifiers", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "rule:numberLiteral", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:offsetOnlyClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:orderByClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:orderExpr", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:orderExprList", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:pivotColumn", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:pivotColumnList", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:placeholder", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:prewhereClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:program", + "kind": "parserRule", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "rule:projectionOrderByClause", + "kind": "parserRule", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "rule:qualifyClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:ratioExpr", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:returnStmt", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:sampleClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:select", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:selectColumnExpr", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:selectColumnExprList", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createSelectItem", + "parseHandler": "AstBuilder#selectColumns", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "rule:selectColumnExprListBeforeFrom", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createSelectItem", + "parseHandler": "AstBuilder#selectColumns", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersToEquivalentStockTrinoAst" + ] + }, + { + "id": "rule:selectSetStmt", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createQueryBody", + "parseHandler": "AstBuilder#buildSetQuery", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, + { + "id": "rule:selectStmt", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:selectStmtWithParens", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:settingExpr", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:settingExprList", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:settingsClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:statement", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:string", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:stringContents", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:stringContentsFull", + "kind": "parserRule", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "rule:subsequentSelectSetClause", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createQueryBody", + "parseHandler": "AstBuilder#buildSetQuery", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, + { + "id": "rule:tableArgList", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:tableExpr", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:tableFunctionExpr", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:tableIdentifier", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createIdentifier", + "parseHandler": "AstBuilder#buildIdentifiers", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testPreservesQuotedIdentifiersAndSourceLocations" + ] + }, + { + "id": "rule:templateString", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:throwStmt", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:topClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:tryCatchStmt", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:unpivotColumn", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:unpivotColumnList", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:valuesClause", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, + { + "id": "rule:valuesRow", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createRelation", + "parseHandler": "AstBuilder#buildTableExpression", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersSetOperationsAndValuesToStockTrinoAst" + ] + }, + { + "id": "rule:varAssignment", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:varDecl", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:whereClause", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createStatement", + "parseHandler": "AstBuilder#build", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlParser#testUsesCanonicalGrammarForExpressionsAndWhereClause" + ] + }, + { + "id": "rule:whileStmt", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:winFrameBound", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createFrameBound", + "parseHandler": "AstBuilder#buildFrameBound", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersEveryCanonicalFrameBound" + ] + }, + { + "id": "rule:winFrameClause", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createWindowFrame", + "parseHandler": "AstBuilder#buildWindowFrame", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersEveryCanonicalFrameBound" + ] + }, + { + "id": "rule:winFrameExtend", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createWindowFrame", + "parseHandler": "AstBuilder#buildWindowFrame", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersEveryCanonicalFrameBound" + ] + }, + { + "id": "rule:winOrderByClause", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createWindow", + "parseHandler": "AstBuilder#buildWindowSpecification", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" + ] + }, + { + "id": "rule:winPartitionByClause", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createWindow", + "parseHandler": "AstBuilder#buildWindowSpecification", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" + ] + }, + { + "id": "rule:windowClause", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createWindowDefinition", + "parseHandler": "AstBuilder#buildWindowDefinitions", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" + ] + }, + { + "id": "rule:windowExpr", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createWindow", + "parseHandler": "AstBuilder#buildWindowSpecification", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlWindowCompiler#testLowersWindowsToEquivalentStockTrinoAst" + ] + }, + { + "id": "rule:withClause", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createWith", + "parseHandler": "AstBuilder#buildCommonTables", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCtesAndDerivedTables" + ] + }, + { + "id": "rule:withExpr", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:withExprColumnNameList", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createWithQuery", + "parseHandler": "AstBuilder#buildCommonTables", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCtesAndDerivedTables" + ] + }, + { + "id": "rule:withExprList", + "kind": "parserRule", + "loweringHandler": "TrinoAstFactory#createWith", + "parseHandler": "AstBuilder#buildCommonTables", + "queryReachable": true, + "status": "supported", + "testCaseIds": [ + "TestHogQlCompiler#testLowersCtesAndDerivedTables" + ] + }, + { + "id": "rule:withFillClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "rule:withinGroupClause", + "kind": "parserRule", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ALL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:AND", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ANTI", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ANY", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ARRAY", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ARROW", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:AS", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ASCENDING", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ASOF", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ASTERISK", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:BACKQUOTE", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:BACKSLASH", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:BETWEEN", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:BINARY_LITERAL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:BOTH", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:BY", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:CASE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:CAST", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:CATCH", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:COHORT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:COLLATE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:COLON", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:COLONEQUALS", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:COLUMNS", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:COMMA", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:CONCAT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:CROSS", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:CUBE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:CURRENT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:DASH", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:DATE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:DAY", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:DECIMAL_LITERAL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:DESC", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:DESCENDING", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:DISTINCT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:DOLLAR", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:DOT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:DOUBLECOLON", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ELSE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:END", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:EQ_DOUBLE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:EQ_SINGLE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ESCAPE_CHAR_COMMON", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:EXCEPT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:EXCLUDE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:EXTRACT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:FILL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:FILTER", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:FINAL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:FINALLY", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:FIRST", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:FLOATING_LITERAL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:FN", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:FOLLOWING", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:FOR", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:FROM", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:FULL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:FULL_STRING_ESCAPE_TRIGGER", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:FULL_STRING_TEXT", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:FUN", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:GROUP", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:GROUPING", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:GT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:GT_EQ", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:HASH", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:HASH_COMMENT", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:HAVING", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:HEXADECIMAL_LITERAL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:HOGQLX_TEXT_TEXT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:HOGQLX_TEXT_WS", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:HOUR", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ID", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:IDENTIFIER", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:IF", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:IGNORE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ILIKE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:IN", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:INCLUDE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:INF", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:INNER", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:INTERPOLATE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:INTERSECT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:INTERVAL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:IREGEX_DOUBLE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:IREGEX_SINGLE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:IS", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:JOIN", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:KEY", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:LAMBDA", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:LAST", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:LBRACE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:LBRACKET", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:LEADING", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:LEFT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:LET", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:LIKE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:LIMIT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:LOCAL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:LPAREN", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:LT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:LT_EQ", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:LT_SLASH", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:MALFORMED_BINARY_LITERAL", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:MATERIALIZED", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:MINUTE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:MONTH", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:MULTI_LINE_COMMENT", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:NAME", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:NAN_SQL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:NATURAL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:NOT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:NOT_EQ", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:NOT_IREGEX", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:NOT_REGEX", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:NULLISH", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:NULLS", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:NULL_PROPERTY", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:NULL_SAFE_EQ", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:NULL_SQL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:OCTAL_LITERAL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:OCTAL_PREFIX_LITERAL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:OFFSET", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ON", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:OR", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ORDER", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:OUTER", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:OVER", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:PARTITION", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:PERCENT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:PIVOT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:PLUS", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:POSITIONAL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:PRECEDING", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:PREWHERE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:QUALIFY", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:QUARTER", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:QUERY", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:QUOTED_IDENTIFIER", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:QUOTE_DOUBLE", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:QUOTE_SINGLE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:QUOTE_SINGLE_TEMPLATE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:QUOTE_SINGLE_TEMPLATE_FULL", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:RANGE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:RBRACE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:RBRACKET", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:RECURSIVE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:REGEX_DOUBLE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:REGEX_SINGLE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:REPLACE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:RETURN", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:RIGHT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ROLLUP", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ROW", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ROWS", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:RPAREN", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:SAMPLE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:SECOND", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:SELECT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:SEMI", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:SEMICOLON", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:SETS", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:SETTINGS", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:SINGLE_LINE_COMMENT", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:SLASH", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:SLASH_GT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:STEP", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:STRING_ESCAPE_TRIGGER", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:STRING_LITERAL", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:STRING_TEXT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:SUBSTRING", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:TAGC_MULTI_LINE_COMMENT", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:TAGC_SINGLE_LINE_COMMENT", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:TAGC_WS", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:TAG_MULTI_LINE_COMMENT", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:TAG_SINGLE_LINE_COMMENT", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:TAG_WS", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:THEN", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:THROW", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:TIES", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:TIME", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:TIMESTAMP", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:TO", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:TOP", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:TOTALS", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:TRAILING", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:TRIM", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:TRUNCATE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:TRY", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:TRY_CAST", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:UNBOUNDED", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:UNDERSCORE", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:UNEXPECTED_CHARACTER", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:UNION", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:UNPIVOT", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:USING", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:VALUES", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:WEEK", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:WHEN", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:WHERE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:WHILE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:WHITESPACE", + "kind": "token", + "loweringHandler": "NotQueryLanguage", + "parseHandler": "NotQueryLanguage", + "queryReachable": false, + "status": "notQueryLanguage", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testAccountsForEveryPublishedGrammarFeature" + ] + }, + { + "id": "token:WINDOW", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:WITH", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:WITHIN", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:YEAR", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + }, + { + "id": "token:ZONE", + "kind": "token", + "loweringHandler": "UnsupportedFeature", + "parseHandler": "CanonicalAntlrParser", + "queryReachable": true, + "status": "explicitCurrentError", + "testCaseIds": [ + "TestHogQlCompatibilityManifest#testUsesOnlyExplicitDispositionsForUnsupportedFeatures" + ] + } + ], + "grammarAlternativeIdentityManifestSha256": "55e33d04906b058bdbab00507b402766d386e4f771b60e4f5106fef63e18383d", + "grammarFeatureManifestSha256": "fbf7236643b43d929e9ef1c8fd01a04b746736dca2f965cc2e78ad62f1a98b10", + "grammarSha256": "c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242", + "languageVersion": "1.0.0", + "schemaVersion": 1, + "sourceUnlabeledAlternativeRules": [ + { + "alternativeCount": 3, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "alias" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "columnExprTupleOrSingle" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "columnIdentifier" + }, + { + "alternativeCount": 4, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "columnTypeCastIdentifier" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "declaration" + }, + { + "alternativeCount": 3, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "floatingLiteral" + }, + { + "alternativeCount": 3, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "hogqlxChildElement" + }, + { + "alternativeCount": 3, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "hogqlxTagAttribute" + }, + { + "alternativeCount": 4, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "identifier" + }, + { + "alternativeCount": 3, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "implicitAlias" + }, + { + "alternativeCount": 3, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "joinConstraintClause" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "joinOpCross" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "limitAndOffsetClause" + }, + { + "alternativeCount": 3, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "limitAndOffsetClauseOptional" + }, + { + "alternativeCount": 3, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "literal" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "ratioExpr" + }, + { + "alternativeCount": 4, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "selectStmtWithParens" + }, + { + "alternativeCount": 11, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "statement" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "string" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": true, + "rule": "stringContents" + }, + { + "alternativeCount": 2, + "code": "UNLABELED_MULTI_ALTERNATIVE_RULE", + "queryReachable": false, + "rule": "stringContentsFull" + } + ] +} diff --git a/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-syntax-ast.json b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-syntax-ast.json new file mode 100644 index 000000000000..ef67c6be9f79 --- /dev/null +++ b/core/trino-hogql-parser/src/main/resources/io/trino/hogql/parser/language/1.0.0/trino-syntax-ast.json @@ -0,0 +1,2343 @@ +{ + "schemaVersion": 1, + "languageVersion": "1.0.0", + "grammarSha256": "c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242", + "grammarFeatureManifestSha256": "fbf7236643b43d929e9ef1c8fd01a04b746736dca2f965cc2e78ad62f1a98b10", + "grammarAlternativeIdentityManifestSha256": "55e33d04906b058bdbab00507b402766d386e4f771b60e4f5106fef63e18383d", + "syntaxTreeKind": "tree", + "sourceSpanGuarantee": "codePointOffsetsEndExclusiveOneBasedLineColumns", + "features": [ + { + "id": "alternative:alias:keyword", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:alias:quoted-identifier", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:alias:unquoted-identifier", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:arrayJoinClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:block", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:catchBlock", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnAliases", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExpr:ColumnExprAlias", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExpr:ColumnExprAnd", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExpr:ColumnExprOr", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExpr:ColumnExprTernaryOp", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExpr:ColumnExprValuePassthrough", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprList", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprTupleOrSingle:scalar", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprTupleOrSingle:tuple", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprArray", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprArrayAccess", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprArraySlice", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprAsterisk", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprBetween", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprCall", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprCallSelect", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprCase", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprCast", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprColonLambda", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsAll", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsExclude", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsExcludeReplace", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsList", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsQualifiedAll", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsQualifiedExclude", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsQualifiedExcludeReplace", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsQualifiedReplace", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsRegex", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprColumnsReplace", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprDate", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprDict", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprFunction", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprFunctionWithinGroup", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprIdentifier", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprIgnoreNulls", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprInterval", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprIntervalString", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprIsDistinctFrom", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprIsNull", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprLambda", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprLiteral", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprNamedArg", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprNegate", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprNot", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprNullArrayAccess", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprNullPropertyAccess", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprNullSafeEq", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprNullTupleAccess", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprNullish", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprParens", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprPositional", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprPrecedence1", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprPrecedence2", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprPrecedence3", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprPropertyAccess", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprSpreadColumnsList", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprSpreadColumnsRegex", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprSubquery", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprSubstring", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprTagElement", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprTemplateString", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprTimestamp", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprTrim", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprTryCast", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprTuple", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprTupleAccess", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprTypeCast", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprWinFunction", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnExprValue:ColumnExprWinFunctionTarget", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnIdentifier:field-path", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnIdentifier:placeholder", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnLambdaExpr:ArrowLambda", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnLambdaExpr:ColonLambda", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnTypeCastExpr:ColumnTypeCastExprSimple", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnTypeCastExpr:ColumnTypeCastExprWithTimeZone", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnTypeCastIdentifier:interval-keyword", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnTypeCastIdentifier:quoted-identifier", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnTypeCastIdentifier:type-keyword", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnTypeCastIdentifier:unquoted-identifier", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprArray", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprComplex", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprCompound", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprEnum", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprNested", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprParam", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnTypeExpr:ColumnTypeExprSimple", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnsReplaceItem", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:columnsReplaceList", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:databaseIdentifier", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:declaration:statement", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:declaration:variable", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:emptyStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:enumValue", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:expr", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:exprStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:expression", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:floatingLiteral:floating-token", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:floatingLiteral:leading-decimal-point", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:floatingLiteral:trailing-decimal-point", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:forInStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:forStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:fromClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:fullTemplateString", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:funcStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:groupByClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:groupingSet", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:groupingSetList", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:havingClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:hogqlxChildElement:expression", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:hogqlxChildElement:tag", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:hogqlxChildElement:text", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:hogqlxTagAttribute:boolean", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:hogqlxTagAttribute:expression", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:hogqlxTagAttribute:string", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:hogqlxTagElement:HogqlxTagElementClosed", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:hogqlxTagElement:HogqlxTagElementNested", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:hogqlxText", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:identifier:interval-keyword", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:identifier:keyword", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:identifier:quoted", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:identifier:unquoted", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:identifierList", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:ifStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:implicitAlias:keyword", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:implicitAlias:quoted-identifier", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:implicitAlias:unquoted-identifier", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:interpolateClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:interpolateExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:interval", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:joinConstraintClause:on", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:joinConstraintClause:using", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:joinConstraintClause:using-parenthesized", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:joinExpr:JoinExprCrossOp", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:joinExpr:JoinExprOp", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:joinExpr:JoinExprParens", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:joinExpr:JoinExprPivot", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:joinExpr:JoinExprPositional", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:joinExpr:JoinExprTable", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:joinExpr:JoinExprUnpivot", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:joinOp:JoinOpFull", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:joinOp:JoinOpInner", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:joinOp:JoinOpLeftRight", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:joinOpCross:comma", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:joinOpCross:cross-join", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:keyword", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:keywordForAlias", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:keywordForImplicitAlias", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:keywordForTypeCast", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:kvPair", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:kvPairList", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:limitAndOffsetClause:compact", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:limitAndOffsetClause:with-offset", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:limitAndOffsetClauseOptional:limit", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:limitAndOffsetClauseOptional:limit-with-offset", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:limitAndOffsetClauseOptional:offset-only", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:limitByClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:limitExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:literal:null", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:literal:number", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:literal:string", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:nestedIdentifier", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:numberLiteral", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:offsetOnlyClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:orderByClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:orderExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:orderExprList", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:pivotColumn", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:pivotColumnList", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:placeholder", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:prewhereClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:program", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:projectionOrderByClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:qualifyClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:ratioExpr:numeric", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:ratioExpr:placeholder", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:returnStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:sampleClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:select", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:selectColumnExpr:ColumnExprAliasBefore", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:selectColumnExpr:ColumnExprAliasImplicit", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:selectColumnExpr:ColumnExprInvalidFromImplicitAlias", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:selectColumnExpr:ColumnExprSelectValue", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:selectColumnExprList", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:selectColumnExprListBeforeFrom:SelectColumnExprListBeforeFromPlain", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:selectColumnExprListBeforeFrom:SelectColumnExprListBeforeFromTrailingComma", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:selectSetStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:selectStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:selectStmtWithParens:direct", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:selectStmtWithParens:parenthesized", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:selectStmtWithParens:placeholder", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:selectStmtWithParens:with-parenthesized", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:settingExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:settingExprList", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:settingsClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:statement:block", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:statement:empty", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:statement:expression", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:statement:for", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:statement:for-in", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:statement:function", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:statement:if", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:statement:return", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:statement:throw", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:statement:try-catch", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:statement:while", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:string:literal", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:string:template", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:stringContents:interpolation", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:stringContents:text", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:stringContentsFull:interpolation", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:stringContentsFull:text", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:subsequentSelectSetClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:tableArgList", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:tableExpr:TableExprAlias", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:tableExpr:TableExprFunction", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:tableExpr:TableExprIdentifier", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:tableExpr:TableExprPivot", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:tableExpr:TableExprPlaceholder", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:tableExpr:TableExprSubquery", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:tableExpr:TableExprTag", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:tableExpr:TableExprUnpivot", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:tableExpr:TableExprValues", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:tableFunctionExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:tableIdentifier", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:templateString", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:throwStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:topClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:tryCatchStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:unpivotColumn", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:unpivotColumnList", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:valuesClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:valuesRow", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:varAssignment", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:varDecl", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:whereClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:whileStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:winFrameBound", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:winFrameClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:winFrameExtend:frameBetween", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:winFrameExtend:frameStart", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:winOrderByClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:winPartitionByClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:windowClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:windowExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:withClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:withExpr:WithExprColumn", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:withExpr:WithExprSubquery", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:withExprColumnNameList", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:withExprList", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:withFillClause", + "syntaxNodeKind": "rule" + }, + { + "id": "alternative:withinGroupClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:alias", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:arrayJoinClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:block", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:catchBlock", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:columnAliases", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:columnExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:columnExprList", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:columnExprTupleOrSingle", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:columnExprValue", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:columnIdentifier", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:columnLambdaExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:columnTypeCastExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:columnTypeCastIdentifier", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:columnTypeExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:columnsReplaceItem", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:columnsReplaceList", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:databaseIdentifier", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:declaration", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:emptyStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:enumValue", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:expr", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:exprStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:expression", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:floatingLiteral", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:forInStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:forStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:fromClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:fullTemplateString", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:funcStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:groupByClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:groupingSet", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:groupingSetList", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:havingClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:hogqlxChildElement", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:hogqlxTagAttribute", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:hogqlxTagElement", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:hogqlxText", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:identifier", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:identifierList", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:ifStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:implicitAlias", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:interpolateClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:interpolateExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:interval", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:joinConstraintClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:joinExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:joinOp", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:joinOpCross", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:keyword", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:keywordForAlias", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:keywordForImplicitAlias", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:keywordForTypeCast", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:kvPair", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:kvPairList", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:limitAndOffsetClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:limitAndOffsetClauseOptional", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:limitByClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:limitExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:literal", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:nestedIdentifier", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:numberLiteral", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:offsetOnlyClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:orderByClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:orderExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:orderExprList", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:pivotColumn", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:pivotColumnList", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:placeholder", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:prewhereClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:program", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:projectionOrderByClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:qualifyClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:ratioExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:returnStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:sampleClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:select", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:selectColumnExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:selectColumnExprList", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:selectColumnExprListBeforeFrom", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:selectSetStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:selectStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:selectStmtWithParens", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:settingExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:settingExprList", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:settingsClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:statement", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:string", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:stringContents", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:stringContentsFull", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:subsequentSelectSetClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:tableArgList", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:tableExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:tableFunctionExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:tableIdentifier", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:templateString", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:throwStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:topClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:tryCatchStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:unpivotColumn", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:unpivotColumnList", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:valuesClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:valuesRow", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:varAssignment", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:varDecl", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:whereClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:whileStmt", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:winFrameBound", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:winFrameClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:winFrameExtend", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:winOrderByClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:winPartitionByClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:windowClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:windowExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:withClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:withExpr", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:withExprColumnNameList", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:withExprList", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:withFillClause", + "syntaxNodeKind": "rule" + }, + { + "id": "rule:withinGroupClause", + "syntaxNodeKind": "rule" + }, + { + "id": "token:ALL", + "syntaxNodeKind": "token" + }, + { + "id": "token:AND", + "syntaxNodeKind": "token" + }, + { + "id": "token:ANTI", + "syntaxNodeKind": "token" + }, + { + "id": "token:ANY", + "syntaxNodeKind": "token" + }, + { + "id": "token:ARRAY", + "syntaxNodeKind": "token" + }, + { + "id": "token:ARROW", + "syntaxNodeKind": "token" + }, + { + "id": "token:AS", + "syntaxNodeKind": "token" + }, + { + "id": "token:ASCENDING", + "syntaxNodeKind": "token" + }, + { + "id": "token:ASOF", + "syntaxNodeKind": "token" + }, + { + "id": "token:ASTERISK", + "syntaxNodeKind": "token" + }, + { + "id": "token:BACKQUOTE", + "syntaxNodeKind": "token" + }, + { + "id": "token:BACKSLASH", + "syntaxNodeKind": "token" + }, + { + "id": "token:BETWEEN", + "syntaxNodeKind": "token" + }, + { + "id": "token:BINARY_LITERAL", + "syntaxNodeKind": "token" + }, + { + "id": "token:BOTH", + "syntaxNodeKind": "token" + }, + { + "id": "token:BY", + "syntaxNodeKind": "token" + }, + { + "id": "token:CASE", + "syntaxNodeKind": "token" + }, + { + "id": "token:CAST", + "syntaxNodeKind": "token" + }, + { + "id": "token:CATCH", + "syntaxNodeKind": "token" + }, + { + "id": "token:COHORT", + "syntaxNodeKind": "token" + }, + { + "id": "token:COLLATE", + "syntaxNodeKind": "token" + }, + { + "id": "token:COLON", + "syntaxNodeKind": "token" + }, + { + "id": "token:COLONEQUALS", + "syntaxNodeKind": "token" + }, + { + "id": "token:COLUMNS", + "syntaxNodeKind": "token" + }, + { + "id": "token:COMMA", + "syntaxNodeKind": "token" + }, + { + "id": "token:CONCAT", + "syntaxNodeKind": "token" + }, + { + "id": "token:CROSS", + "syntaxNodeKind": "token" + }, + { + "id": "token:CUBE", + "syntaxNodeKind": "token" + }, + { + "id": "token:CURRENT", + "syntaxNodeKind": "token" + }, + { + "id": "token:DASH", + "syntaxNodeKind": "token" + }, + { + "id": "token:DATE", + "syntaxNodeKind": "token" + }, + { + "id": "token:DAY", + "syntaxNodeKind": "token" + }, + { + "id": "token:DECIMAL_LITERAL", + "syntaxNodeKind": "token" + }, + { + "id": "token:DESC", + "syntaxNodeKind": "token" + }, + { + "id": "token:DESCENDING", + "syntaxNodeKind": "token" + }, + { + "id": "token:DISTINCT", + "syntaxNodeKind": "token" + }, + { + "id": "token:DOLLAR", + "syntaxNodeKind": "token" + }, + { + "id": "token:DOT", + "syntaxNodeKind": "token" + }, + { + "id": "token:DOUBLECOLON", + "syntaxNodeKind": "token" + }, + { + "id": "token:ELSE", + "syntaxNodeKind": "token" + }, + { + "id": "token:END", + "syntaxNodeKind": "token" + }, + { + "id": "token:EQ_DOUBLE", + "syntaxNodeKind": "token" + }, + { + "id": "token:EQ_SINGLE", + "syntaxNodeKind": "token" + }, + { + "id": "token:ESCAPE_CHAR_COMMON", + "syntaxNodeKind": "token" + }, + { + "id": "token:EXCEPT", + "syntaxNodeKind": "token" + }, + { + "id": "token:EXCLUDE", + "syntaxNodeKind": "token" + }, + { + "id": "token:EXTRACT", + "syntaxNodeKind": "token" + }, + { + "id": "token:FILL", + "syntaxNodeKind": "token" + }, + { + "id": "token:FILTER", + "syntaxNodeKind": "token" + }, + { + "id": "token:FINAL", + "syntaxNodeKind": "token" + }, + { + "id": "token:FINALLY", + "syntaxNodeKind": "token" + }, + { + "id": "token:FIRST", + "syntaxNodeKind": "token" + }, + { + "id": "token:FLOATING_LITERAL", + "syntaxNodeKind": "token" + }, + { + "id": "token:FN", + "syntaxNodeKind": "token" + }, + { + "id": "token:FOLLOWING", + "syntaxNodeKind": "token" + }, + { + "id": "token:FOR", + "syntaxNodeKind": "token" + }, + { + "id": "token:FROM", + "syntaxNodeKind": "token" + }, + { + "id": "token:FULL", + "syntaxNodeKind": "token" + }, + { + "id": "token:FULL_STRING_ESCAPE_TRIGGER", + "syntaxNodeKind": "token" + }, + { + "id": "token:FULL_STRING_TEXT", + "syntaxNodeKind": "token" + }, + { + "id": "token:FUN", + "syntaxNodeKind": "token" + }, + { + "id": "token:GROUP", + "syntaxNodeKind": "token" + }, + { + "id": "token:GROUPING", + "syntaxNodeKind": "token" + }, + { + "id": "token:GT", + "syntaxNodeKind": "token" + }, + { + "id": "token:GT_EQ", + "syntaxNodeKind": "token" + }, + { + "id": "token:HASH", + "syntaxNodeKind": "token" + }, + { + "id": "token:HASH_COMMENT", + "syntaxNodeKind": "token" + }, + { + "id": "token:HAVING", + "syntaxNodeKind": "token" + }, + { + "id": "token:HEXADECIMAL_LITERAL", + "syntaxNodeKind": "token" + }, + { + "id": "token:HOGQLX_TEXT_TEXT", + "syntaxNodeKind": "token" + }, + { + "id": "token:HOGQLX_TEXT_WS", + "syntaxNodeKind": "token" + }, + { + "id": "token:HOUR", + "syntaxNodeKind": "token" + }, + { + "id": "token:ID", + "syntaxNodeKind": "token" + }, + { + "id": "token:IDENTIFIER", + "syntaxNodeKind": "token" + }, + { + "id": "token:IF", + "syntaxNodeKind": "token" + }, + { + "id": "token:IGNORE", + "syntaxNodeKind": "token" + }, + { + "id": "token:ILIKE", + "syntaxNodeKind": "token" + }, + { + "id": "token:IN", + "syntaxNodeKind": "token" + }, + { + "id": "token:INCLUDE", + "syntaxNodeKind": "token" + }, + { + "id": "token:INF", + "syntaxNodeKind": "token" + }, + { + "id": "token:INNER", + "syntaxNodeKind": "token" + }, + { + "id": "token:INTERPOLATE", + "syntaxNodeKind": "token" + }, + { + "id": "token:INTERSECT", + "syntaxNodeKind": "token" + }, + { + "id": "token:INTERVAL", + "syntaxNodeKind": "token" + }, + { + "id": "token:IREGEX_DOUBLE", + "syntaxNodeKind": "token" + }, + { + "id": "token:IREGEX_SINGLE", + "syntaxNodeKind": "token" + }, + { + "id": "token:IS", + "syntaxNodeKind": "token" + }, + { + "id": "token:JOIN", + "syntaxNodeKind": "token" + }, + { + "id": "token:KEY", + "syntaxNodeKind": "token" + }, + { + "id": "token:LAMBDA", + "syntaxNodeKind": "token" + }, + { + "id": "token:LAST", + "syntaxNodeKind": "token" + }, + { + "id": "token:LBRACE", + "syntaxNodeKind": "token" + }, + { + "id": "token:LBRACKET", + "syntaxNodeKind": "token" + }, + { + "id": "token:LEADING", + "syntaxNodeKind": "token" + }, + { + "id": "token:LEFT", + "syntaxNodeKind": "token" + }, + { + "id": "token:LET", + "syntaxNodeKind": "token" + }, + { + "id": "token:LIKE", + "syntaxNodeKind": "token" + }, + { + "id": "token:LIMIT", + "syntaxNodeKind": "token" + }, + { + "id": "token:LOCAL", + "syntaxNodeKind": "token" + }, + { + "id": "token:LPAREN", + "syntaxNodeKind": "token" + }, + { + "id": "token:LT", + "syntaxNodeKind": "token" + }, + { + "id": "token:LT_EQ", + "syntaxNodeKind": "token" + }, + { + "id": "token:LT_SLASH", + "syntaxNodeKind": "token" + }, + { + "id": "token:MALFORMED_BINARY_LITERAL", + "syntaxNodeKind": "token" + }, + { + "id": "token:MATERIALIZED", + "syntaxNodeKind": "token" + }, + { + "id": "token:MINUTE", + "syntaxNodeKind": "token" + }, + { + "id": "token:MONTH", + "syntaxNodeKind": "token" + }, + { + "id": "token:MULTI_LINE_COMMENT", + "syntaxNodeKind": "token" + }, + { + "id": "token:NAME", + "syntaxNodeKind": "token" + }, + { + "id": "token:NAN_SQL", + "syntaxNodeKind": "token" + }, + { + "id": "token:NATURAL", + "syntaxNodeKind": "token" + }, + { + "id": "token:NOT", + "syntaxNodeKind": "token" + }, + { + "id": "token:NOT_EQ", + "syntaxNodeKind": "token" + }, + { + "id": "token:NOT_IREGEX", + "syntaxNodeKind": "token" + }, + { + "id": "token:NOT_REGEX", + "syntaxNodeKind": "token" + }, + { + "id": "token:NULLISH", + "syntaxNodeKind": "token" + }, + { + "id": "token:NULLS", + "syntaxNodeKind": "token" + }, + { + "id": "token:NULL_PROPERTY", + "syntaxNodeKind": "token" + }, + { + "id": "token:NULL_SAFE_EQ", + "syntaxNodeKind": "token" + }, + { + "id": "token:NULL_SQL", + "syntaxNodeKind": "token" + }, + { + "id": "token:OCTAL_LITERAL", + "syntaxNodeKind": "token" + }, + { + "id": "token:OCTAL_PREFIX_LITERAL", + "syntaxNodeKind": "token" + }, + { + "id": "token:OFFSET", + "syntaxNodeKind": "token" + }, + { + "id": "token:ON", + "syntaxNodeKind": "token" + }, + { + "id": "token:OR", + "syntaxNodeKind": "token" + }, + { + "id": "token:ORDER", + "syntaxNodeKind": "token" + }, + { + "id": "token:OUTER", + "syntaxNodeKind": "token" + }, + { + "id": "token:OVER", + "syntaxNodeKind": "token" + }, + { + "id": "token:PARTITION", + "syntaxNodeKind": "token" + }, + { + "id": "token:PERCENT", + "syntaxNodeKind": "token" + }, + { + "id": "token:PIVOT", + "syntaxNodeKind": "token" + }, + { + "id": "token:PLUS", + "syntaxNodeKind": "token" + }, + { + "id": "token:POSITIONAL", + "syntaxNodeKind": "token" + }, + { + "id": "token:PRECEDING", + "syntaxNodeKind": "token" + }, + { + "id": "token:PREWHERE", + "syntaxNodeKind": "token" + }, + { + "id": "token:QUALIFY", + "syntaxNodeKind": "token" + }, + { + "id": "token:QUARTER", + "syntaxNodeKind": "token" + }, + { + "id": "token:QUERY", + "syntaxNodeKind": "token" + }, + { + "id": "token:QUOTED_IDENTIFIER", + "syntaxNodeKind": "token" + }, + { + "id": "token:QUOTE_DOUBLE", + "syntaxNodeKind": "token" + }, + { + "id": "token:QUOTE_SINGLE", + "syntaxNodeKind": "token" + }, + { + "id": "token:QUOTE_SINGLE_TEMPLATE", + "syntaxNodeKind": "token" + }, + { + "id": "token:QUOTE_SINGLE_TEMPLATE_FULL", + "syntaxNodeKind": "token" + }, + { + "id": "token:RANGE", + "syntaxNodeKind": "token" + }, + { + "id": "token:RBRACE", + "syntaxNodeKind": "token" + }, + { + "id": "token:RBRACKET", + "syntaxNodeKind": "token" + }, + { + "id": "token:RECURSIVE", + "syntaxNodeKind": "token" + }, + { + "id": "token:REGEX_DOUBLE", + "syntaxNodeKind": "token" + }, + { + "id": "token:REGEX_SINGLE", + "syntaxNodeKind": "token" + }, + { + "id": "token:REPLACE", + "syntaxNodeKind": "token" + }, + { + "id": "token:RETURN", + "syntaxNodeKind": "token" + }, + { + "id": "token:RIGHT", + "syntaxNodeKind": "token" + }, + { + "id": "token:ROLLUP", + "syntaxNodeKind": "token" + }, + { + "id": "token:ROW", + "syntaxNodeKind": "token" + }, + { + "id": "token:ROWS", + "syntaxNodeKind": "token" + }, + { + "id": "token:RPAREN", + "syntaxNodeKind": "token" + }, + { + "id": "token:SAMPLE", + "syntaxNodeKind": "token" + }, + { + "id": "token:SECOND", + "syntaxNodeKind": "token" + }, + { + "id": "token:SELECT", + "syntaxNodeKind": "token" + }, + { + "id": "token:SEMI", + "syntaxNodeKind": "token" + }, + { + "id": "token:SEMICOLON", + "syntaxNodeKind": "token" + }, + { + "id": "token:SETS", + "syntaxNodeKind": "token" + }, + { + "id": "token:SETTINGS", + "syntaxNodeKind": "token" + }, + { + "id": "token:SINGLE_LINE_COMMENT", + "syntaxNodeKind": "token" + }, + { + "id": "token:SLASH", + "syntaxNodeKind": "token" + }, + { + "id": "token:SLASH_GT", + "syntaxNodeKind": "token" + }, + { + "id": "token:STEP", + "syntaxNodeKind": "token" + }, + { + "id": "token:STRING_ESCAPE_TRIGGER", + "syntaxNodeKind": "token" + }, + { + "id": "token:STRING_LITERAL", + "syntaxNodeKind": "token" + }, + { + "id": "token:STRING_TEXT", + "syntaxNodeKind": "token" + }, + { + "id": "token:SUBSTRING", + "syntaxNodeKind": "token" + }, + { + "id": "token:TAGC_MULTI_LINE_COMMENT", + "syntaxNodeKind": "token" + }, + { + "id": "token:TAGC_SINGLE_LINE_COMMENT", + "syntaxNodeKind": "token" + }, + { + "id": "token:TAGC_WS", + "syntaxNodeKind": "token" + }, + { + "id": "token:TAG_MULTI_LINE_COMMENT", + "syntaxNodeKind": "token" + }, + { + "id": "token:TAG_SINGLE_LINE_COMMENT", + "syntaxNodeKind": "token" + }, + { + "id": "token:TAG_WS", + "syntaxNodeKind": "token" + }, + { + "id": "token:THEN", + "syntaxNodeKind": "token" + }, + { + "id": "token:THROW", + "syntaxNodeKind": "token" + }, + { + "id": "token:TIES", + "syntaxNodeKind": "token" + }, + { + "id": "token:TIME", + "syntaxNodeKind": "token" + }, + { + "id": "token:TIMESTAMP", + "syntaxNodeKind": "token" + }, + { + "id": "token:TO", + "syntaxNodeKind": "token" + }, + { + "id": "token:TOP", + "syntaxNodeKind": "token" + }, + { + "id": "token:TOTALS", + "syntaxNodeKind": "token" + }, + { + "id": "token:TRAILING", + "syntaxNodeKind": "token" + }, + { + "id": "token:TRIM", + "syntaxNodeKind": "token" + }, + { + "id": "token:TRUNCATE", + "syntaxNodeKind": "token" + }, + { + "id": "token:TRY", + "syntaxNodeKind": "token" + }, + { + "id": "token:TRY_CAST", + "syntaxNodeKind": "token" + }, + { + "id": "token:UNBOUNDED", + "syntaxNodeKind": "token" + }, + { + "id": "token:UNDERSCORE", + "syntaxNodeKind": "token" + }, + { + "id": "token:UNEXPECTED_CHARACTER", + "syntaxNodeKind": "token" + }, + { + "id": "token:UNION", + "syntaxNodeKind": "token" + }, + { + "id": "token:UNPIVOT", + "syntaxNodeKind": "token" + }, + { + "id": "token:USING", + "syntaxNodeKind": "token" + }, + { + "id": "token:VALUES", + "syntaxNodeKind": "token" + }, + { + "id": "token:WEEK", + "syntaxNodeKind": "token" + }, + { + "id": "token:WHEN", + "syntaxNodeKind": "token" + }, + { + "id": "token:WHERE", + "syntaxNodeKind": "token" + }, + { + "id": "token:WHILE", + "syntaxNodeKind": "token" + }, + { + "id": "token:WHITESPACE", + "syntaxNodeKind": "token" + }, + { + "id": "token:WINDOW", + "syntaxNodeKind": "token" + }, + { + "id": "token:WITH", + "syntaxNodeKind": "token" + }, + { + "id": "token:WITHIN", + "syntaxNodeKind": "token" + }, + { + "id": "token:YEAR", + "syntaxNodeKind": "token" + }, + { + "id": "token:ZONE", + "syntaxNodeKind": "token" + } + ] +} diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlCompatibilityCorpus.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlCompatibilityCorpus.java new file mode 100644 index 000000000000..9d6811bab4f9 --- /dev/null +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlCompatibilityCorpus.java @@ -0,0 +1,348 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.airlift.json.JsonCodec; +import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; +import io.trino.hogql.parser.tree.HogQlSyntaxTree; +import io.trino.hogql.parser.tree.HogQlSyntaxTree.Element; +import io.trino.hogql.parser.tree.HogQlSyntaxTree.Node; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestFactory; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import static io.airlift.json.JsonCodec.jsonCodec; +import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.toMap; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.fail; +import static org.junit.jupiter.api.DynamicTest.dynamicTest; + +public class TestHogQlCompatibilityCorpus +{ + private static final String RESOURCE_ROOT = "/io/trino/hogql/parser/language/1.0.0/"; + private static final JsonCodec CORPUS_MANIFEST_CODEC = jsonCodec(CorpusManifest.class); + private static final JsonCodec ORACLE_CASE_CODEC = jsonCodec(OracleCase.class); + private static final JsonCodec OVERRIDES_CODEC = jsonCodec(CompatibilityOverrides.class); + + private final HogQlParser parser = new HogQlParser(); + + @Test + public void testOracleMetadataMatchesThePublishedCorpus() + throws IOException + { + CorpusManifest manifest = loadCorpusManifest(); + Map oracleById = loadOracleCases().stream() + .collect(toMap(OracleCase::id, oracleCase -> oracleCase)); + + assertThat(oracleById).hasSize(manifest.cases().size()); + for (CorpusCase corpusCase : manifest.cases()) { + OracleCase oracleCase = oracleById.remove(corpusCase.id()); + assertThat(oracleCase).isNotNull(); + assertThat(oracleCase.source()).isEqualTo(corpusCase.source()); + assertThat(oracleCase.entryPoint()).isEqualTo(corpusCase.entryPoint()); + assertThat(oracleCase.accepted()).isEqualTo(corpusCase.accepted()); + assertThat(oracleCase.languageVersion()).isEqualTo(HogQlLanguageContract.current().languageVersion()); + assertThat(oracleCase.grammarSha256()).isEqualTo(HogQlLanguageContract.current().grammarSha256()); + assertThat(oracleCase.span()).isNotNull(); + if (oracleCase.accepted()) { + assertThat(oracleCase.ast()).isNotNull(); + assertThat(oracleCase.errorCategory()).isNull(); + } + else { + assertThat(oracleCase.ast()).isNull(); + assertThat(oracleCase.errorCategory()).isNotBlank(); + } + } + assertThat(oracleById).isEmpty(); + } + + @TestFactory + public Stream testTrinoDispositionForEveryCorpusCase() + throws IOException + { + CorpusManifest manifest = loadCorpusManifest(); + Map dispositionById = loadCompatibilityOverrides().corpusCases().stream() + .collect(toMap(CorpusDisposition::id, disposition -> disposition)); + + assertThat(dispositionById.keySet()) + .containsExactlyInAnyOrderElementsOf(manifest.cases().stream().map(CorpusCase::id).toList()); + return manifest.cases().stream() + .map(corpusCase -> dynamicTest(corpusCase.id(), () -> assertDisposition(corpusCase, dispositionById.get(corpusCase.id())))); + } + + @TestFactory + public Stream testSyntaxTreeDispositionForEveryCorpusCase() + throws IOException + { + Map oracleById = loadOracleCases().stream() + .collect(toMap(OracleCase::id, oracleCase -> oracleCase)); + return loadCorpusManifest().cases().stream() + .map(corpusCase -> dynamicTest(corpusCase.id(), () -> assertSyntaxDisposition(corpusCase, oracleById.get(corpusCase.id())))); + } + + private void assertDisposition(CorpusCase corpusCase, CorpusDisposition disposition) + { + String query = switch (corpusCase.entryPoint()) { + case "expression" -> "SELECT " + corpusCase.source(); + case "query" -> corpusCase.source(); + default -> throw new IllegalArgumentException("unknown corpus entry point: " + corpusCase.entryPoint()); + }; + switch (disposition.status()) { + case "supported" -> assertThatCode(() -> parser.parseStatement(query)).doesNotThrowAnyException(); + case "explicitCurrentError" -> assertThatThrownBy(() -> parser.parseStatement(query)) + .isInstanceOf(HogQlParsingException.class) + .hasMessageContaining("HogQL feature is not lowered yet"); + case "canonicalRejection" -> assertThatThrownBy(() -> parser.parseStatement(query)) + .isInstanceOf(HogQlParsingException.class) + .hasMessageStartingWith("line "); + default -> throw new IllegalArgumentException("unknown corpus disposition: " + disposition.status()); + } + } + + private void assertSyntaxDisposition(CorpusCase corpusCase, OracleCase oracleCase) + { + if (!corpusCase.accepted()) { + assertThatThrownBy(() -> parseSyntax(corpusCase)) + .isInstanceOf(HogQlParsingException.class) + .satisfies(error -> { + HogQlParsingException parsingException = (HogQlParsingException) error; + assertThat(parsingException.getLineNumber()).isEqualTo(oracleCase.span().start().line()); + if (oracleCase.errorCategory().equals("unexpected-character")) { + assertThat(parsingException.getColumnNumber()).isEqualTo(oracleCase.span().start().column() + 1); + } + else { + assertThat(parsingException.getColumnNumber()).isBetween(1, oracleCase.span().start().column() + 1); + } + }); + return; + } + + HogQlSyntaxTree syntaxTree = parseSyntax(corpusCase); + String source = corpusCase.source(); + int sourceLength = source.codePointCount(0, source.length()); + SourceSpan oracleSpan = toSourceSpan(oracleCase.span()); + assertThat(syntaxTree.root().span().startOffset()).isEqualTo(oracleSpan.startOffset()); + assertThat(syntaxTree.root().span().endOffset()).isGreaterThanOrEqualTo(oracleSpan.endOffset()); + if (corpusCase.entryPoint().equals("expression")) { + assertThat(syntaxTree.root().span()).isEqualTo(oracleSpan); + } + assertThat(syntaxTree.root().span()).isEqualTo(new SourceSpan(0, sourceLength, 1, 1, lineCount(source), finalColumn(source))); + assertContainedSpans(syntaxTree.root(), sourceLength); + } + + private static SourceSpan toSourceSpan(OracleSpan span) + { + return new SourceSpan( + span.start().offset(), + span.end().offset(), + span.start().line(), + span.start().column() + 1, + span.end().line(), + span.end().column() + 1); + } + + private HogQlSyntaxTree parseSyntax(CorpusCase corpusCase) + { + return switch (corpusCase.entryPoint()) { + case "expression" -> parser.parseExpressionSyntax(corpusCase.source()); + case "query" -> parser.parseSyntax(corpusCase.source()); + default -> throw new IllegalArgumentException("unknown corpus entry point: " + corpusCase.entryPoint()); + }; + } + + private static void assertContainedSpans(Element element, int sourceLength) + { + SourceSpan span = element.span(); + assertThat(span.startOffset()).isBetween(0, sourceLength); + assertThat(span.endOffset()).isBetween(span.startOffset(), sourceLength); + if (element instanceof Node node) { + for (Element child : node.children()) { + assertThat(child.span().startOffset()).isGreaterThanOrEqualTo(span.startOffset()); + assertThat(child.span().endOffset()).isLessThanOrEqualTo(span.endOffset()); + assertContainedSpans(child, sourceLength); + } + } + } + + private static int lineCount(String source) + { + int lines = 1; + boolean previousWasCarriageReturn = false; + for (int offset = 0; offset < source.length(); ) { + int codePoint = source.codePointAt(offset); + offset += Character.charCount(codePoint); + if (codePoint == '\r') { + lines++; + previousWasCarriageReturn = true; + } + else if (codePoint == '\n') { + if (!previousWasCarriageReturn) { + lines++; + } + previousWasCarriageReturn = false; + } + else { + previousWasCarriageReturn = false; + } + } + return lines; + } + + private static int finalColumn(String source) + { + int lastNewline = Math.max(source.lastIndexOf('\n'), source.lastIndexOf('\r')); + return source.codePointCount(lastNewline + 1, source.length()) + 1; + } + + private static CorpusManifest loadCorpusManifest() + throws IOException + { + return CORPUS_MANIFEST_CODEC.fromJson(readResource("corpus/expr_select_cases.json")); + } + + private static List loadOracleCases() + throws IOException + { + return Arrays.stream(readResource("corpus/expr_select_cpp_oracle.jsonl").split("\\R")) + .filter(line -> !line.isBlank()) + .map(ORACLE_CASE_CODEC::fromJson) + .toList(); + } + + private static CompatibilityOverrides loadCompatibilityOverrides() + throws IOException + { + return OVERRIDES_CODEC.fromJson(readResource("trino-compatibility-overrides.json")); + } + + private static String readResource(String relativePath) + throws IOException + { + String path = RESOURCE_ROOT + relativePath; + try (InputStream input = TestHogQlCompatibilityCorpus.class.getResourceAsStream(path)) { + if (input == null) { + fail("missing test resource: " + path); + } + return new String(input.readAllBytes(), StandardCharsets.UTF_8); + } + } + + public record CorpusManifest(List cases) + { + @JsonCreator + public CorpusManifest(@JsonProperty("cases") List cases) + { + this.cases = List.copyOf(requireNonNull(cases, "cases is null")); + } + } + + public record CorpusCase(String id, String source, String entryPoint, boolean accepted) + { + @JsonCreator + public CorpusCase( + @JsonProperty("id") String id, + @JsonProperty("source") String source, + @JsonProperty("entryPoint") String entryPoint, + @JsonProperty("accepted") boolean accepted) + { + this.id = requireNonNull(id, "id is null"); + this.source = requireNonNull(source, "source is null"); + this.entryPoint = requireNonNull(entryPoint, "entryPoint is null"); + this.accepted = accepted; + } + } + + public record OracleCase( + String id, + String source, + String entryPoint, + boolean accepted, + HogQlLanguageVersion languageVersion, + String grammarSha256, + Map ast, + OracleSpan span, + String errorCategory) + { + @JsonCreator + public OracleCase( + @JsonProperty("id") String id, + @JsonProperty("source") String source, + @JsonProperty("entryPoint") String entryPoint, + @JsonProperty("accepted") boolean accepted, + @JsonProperty("languageVersion") HogQlLanguageVersion languageVersion, + @JsonProperty("grammarSha256") String grammarSha256, + @JsonProperty("ast") Map ast, + @JsonProperty("span") OracleSpan span, + @JsonProperty("errorCategory") String errorCategory) + { + this.id = requireNonNull(id, "id is null"); + this.source = requireNonNull(source, "source is null"); + this.entryPoint = requireNonNull(entryPoint, "entryPoint is null"); + this.accepted = accepted; + this.languageVersion = requireNonNull(languageVersion, "languageVersion is null"); + this.grammarSha256 = requireNonNull(grammarSha256, "grammarSha256 is null"); + this.ast = ast; + this.span = requireNonNull(span, "span is null"); + this.errorCategory = errorCategory; + } + } + + public record OracleSpan(OraclePoint start, OraclePoint end) + { + @JsonCreator + public OracleSpan( + @JsonProperty("start") OraclePoint start, + @JsonProperty("end") OraclePoint end) + { + this.start = requireNonNull(start, "start is null"); + this.end = requireNonNull(end, "end is null"); + } + } + + public record OraclePoint(int line, int column, int offset) {} + + public record CompatibilityOverrides(List corpusCases) + { + @JsonCreator + public CompatibilityOverrides(@JsonProperty("corpusCases") List corpusCases) + { + this.corpusCases = List.copyOf(requireNonNull(corpusCases, "corpusCases is null")); + } + } + + public record CorpusDisposition(String id, String status) + { + @JsonCreator + public CorpusDisposition( + @JsonProperty("id") String id, + @JsonProperty("status") String status) + { + this.id = requireNonNull(id, "id is null"); + this.status = requireNonNull(status, "status is null"); + } + } +} diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlCompatibilityManifest.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlCompatibilityManifest.java new file mode 100644 index 000000000000..c2f25df777be --- /dev/null +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlCompatibilityManifest.java @@ -0,0 +1,110 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.airlift.json.JsonCodec; +import io.trino.hogql.parser.HogQlCompatibilityManifest.Status; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Map; + +import static io.airlift.json.JsonCodec.jsonCodec; +import static java.util.Objects.requireNonNull; +import static java.util.stream.Collectors.toMap; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; + +public class TestHogQlCompatibilityManifest +{ + private static final String OVERRIDES_RESOURCE = "/io/trino/hogql/parser/language/1.0.0/trino-compatibility-overrides.json"; + private static final JsonCodec OVERRIDES_CODEC = jsonCodec(CompatibilityOverrides.class); + + @Test + public void testAccountsForEveryPublishedGrammarFeature() + { + HogQlCompatibilityManifest manifest = HogQlCompatibilityManifest.current(); + + assertThat(manifest.schemaVersion()).isEqualTo(1); + assertThat(manifest.languageVersion()).isEqualTo(HogQlLanguageContract.current().languageVersion()); + assertThat(manifest.features()).hasSize(583); + assertThat(manifest.sourceUnlabeledAlternativeRules()).hasSize(21); + assertThat(manifest.features()) + .allSatisfy(feature -> assertThat(feature.testCaseIds()).isNotEmpty()); + } + + @Test + public void testUsesOnlyExplicitDispositionsForUnsupportedFeatures() + { + HogQlCompatibilityManifest manifest = HogQlCompatibilityManifest.current(); + + assertThat(manifest.features()) + .filteredOn(feature -> !feature.queryReachable()) + .allSatisfy(feature -> assertThat(feature.status()).isEqualTo(Status.NOT_QUERY_LANGUAGE)); + assertThat(manifest.features()) + .filteredOn(feature -> feature.status() == Status.EXPLICIT_CURRENT_ERROR) + .allSatisfy(feature -> assertThat(feature.loweringHandler()).isEqualTo("UnsupportedFeature")); + } + + @Test + public void testSupportedFeaturesMatchTheSparseOverrides() + throws IOException + { + CompatibilityOverrides overrides; + try (InputStream input = TestHogQlCompatibilityManifest.class.getResourceAsStream(OVERRIDES_RESOURCE)) { + if (input == null) { + fail("missing test resource: " + OVERRIDES_RESOURCE); + } + overrides = OVERRIDES_CODEC.fromJson(new String(input.readAllBytes(), StandardCharsets.UTF_8)); + } + + Map overridesById = overrides.features().stream() + .collect(toMap(CompatibilityOverride::id, feature -> feature)); + assertThat(HogQlCompatibilityManifest.current().features()) + .filteredOn(feature -> feature.status() == Status.SUPPORTED) + .allSatisfy(feature -> assertThat(overridesById.remove(feature.id())) + .isEqualTo(new CompatibilityOverride(feature.id(), feature.parseHandler(), feature.loweringHandler(), feature.testCaseIds()))); + assertThat(overridesById).isEmpty(); + } + + public record CompatibilityOverrides(List features) + { + @JsonCreator + public CompatibilityOverrides(@JsonProperty("features") List features) + { + this.features = List.copyOf(requireNonNull(features, "features is null")); + } + } + + public record CompatibilityOverride(String id, String parseHandler, String loweringHandler, List testCaseIds) + { + @JsonCreator + public CompatibilityOverride( + @JsonProperty("id") String id, + @JsonProperty("parseHandler") String parseHandler, + @JsonProperty("loweringHandler") String loweringHandler, + @JsonProperty("testCaseIds") List testCaseIds) + { + this.id = requireNonNull(id, "id is null"); + this.parseHandler = requireNonNull(parseHandler, "parseHandler is null"); + this.loweringHandler = requireNonNull(loweringHandler, "loweringHandler is null"); + this.testCaseIds = List.copyOf(requireNonNull(testCaseIds, "testCaseIds is null")); + } + } +} diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlGrammarAlternativeIdentities.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlGrammarAlternativeIdentities.java new file mode 100644 index 000000000000..888b9bee0ef1 --- /dev/null +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlGrammarAlternativeIdentities.java @@ -0,0 +1,106 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import io.trino.hogql.parser.HogQlGrammarAlternativeIdentities.AlternativeRule; +import org.antlr.v4.Tool; +import org.antlr.v4.tool.Grammar; +import org.antlr.v4.tool.LexerGrammar; +import org.antlr.v4.tool.Rule; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.HexFormat; +import java.util.Map; +import java.util.Set; +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TestHogQlGrammarAlternativeIdentities +{ + private static final String GRAMMAR_RESOURCE_ROOT = "/io/trino/hogql/parser/language/1.0.0/grammar/"; + private static final Map> CANONICAL_FINGERPRINTS = loadCanonicalFingerprints(); + + @ParameterizedTest(name = "{0}") + @MethodSource("alternativeRules") + public void testSemanticIdentitiesMatchCanonicalAlternativeStructure(String ruleName, AlternativeRule expected) + { + assertThat(CANONICAL_FINGERPRINTS.get(ruleName)) + .containsExactlyInAnyOrderElementsOf(expected.alternatives().stream() + .map(HogQlGrammarAlternativeIdentities.Alternative::structuralFingerprint) + .toList()); + } + + private static Stream alternativeRules() + { + return HogQlGrammarAlternativeIdentities.current().rules().stream() + .map(rule -> Arguments.of(rule.rule(), rule)); + } + + private static Map> loadCanonicalFingerprints() + { + try { + assertThat(Tool.VERSION).isEqualTo(HogQlLanguageContract.current().antlrVersion()); + String javaLexer = readResource("HogQLLexer.java.g4"); + String commonLexer = readResource("HogQLLexer.common.g4"); + String completeLexer = javaLexer + commonLexer.substring(commonLexer.indexOf('\n') + 1); + LexerGrammar lexer = new LexerGrammar(completeLexer); + Grammar parser = new Grammar(readResource("HogQLParser.g4"), lexer); + + Map> fingerprints = new HashMap<>(); + for (AlternativeRule expected : HogQlGrammarAlternativeIdentities.current().rules()) { + Rule rule = parser.getRule(expected.rule()); + Set alternatives = new HashSet<>(); + for (int alternative = 1; alternative <= rule.numberOfAlts; alternative++) { + alternatives.add(sha256(rule.alt[alternative].ast.toStringTree())); + } + fingerprints.put(expected.rule(), Set.copyOf(alternatives)); + } + return Map.copyOf(fingerprints); + } + catch (Exception e) { + throw new AssertionError("failed to inspect canonical HogQL grammar", e); + } + } + + private static String readResource(String name) + throws IOException + { + try (InputStream input = TestHogQlGrammarAlternativeIdentities.class.getResourceAsStream(GRAMMAR_RESOURCE_ROOT + name)) { + if (input == null) { + throw new IOException("missing canonical HogQL grammar resource: " + name); + } + return new String(input.readAllBytes(), StandardCharsets.UTF_8); + } + } + + private static String sha256(String value) + { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8))); + } + catch (NoSuchAlgorithmException e) { + throw new AssertionError(e); + } + } +} diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlLanguageContract.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlLanguageContract.java new file mode 100644 index 000000000000..758e5b33a95f --- /dev/null +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlLanguageContract.java @@ -0,0 +1,129 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.HexFormat; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.fail; + +public class TestHogQlLanguageContract +{ + private static final String LANGUAGE_RESOURCE_ROOT = "/io/trino/hogql/parser/language/1.0.0/"; + + @Test + public void testLoadsCurrentLanguageContract() + { + HogQlLanguageContract contract = HogQlLanguageContract.current(); + + assertThat(contract.schemaVersion()).isEqualTo(1); + assertThat(contract.languageVersion()).isEqualTo(new HogQlLanguageVersion(1, 0, 0)); + assertThat(contract.antlrVersion()).isEqualTo("4.13.2"); + assertThat(contract.canonicalParser()).isEqualTo("cpp-antlr"); + assertThat(contract.entryPoints()).containsEntry("query", "select"); + assertThat(contract.grammarFeatureManifest().schemaVersion()).isEqualTo(1); + assertThat(contract.corpora()).singleElement() + .satisfies(corpus -> assertThat(corpus.slice()).isEqualTo("expression-and-plain-select")); + assertThat(contract.grammarSha256()).isEqualTo("c255b1c828ea34b4eb5373dd1a153bbcf69f4f9eb58a900d353972ce07724242"); + } + + @ParameterizedTest + @ValueSource(strings = {"", "1", "1.0", "01.0.0", "1.0.0-beta", "-1.0.0"}) + public void testRejectsInvalidLanguageVersions(String value) + { + assertThatThrownBy(() -> HogQlLanguageVersion.valueOf(value)) + .isInstanceOf(IllegalArgumentException.class) + .hasMessageContaining("invalid HogQL language version"); + } + + @Test + public void testRejectsUnsupportedLanguageVersion() + { + assertThatThrownBy(() -> new HogQlParser().parseStatement("SELECT 1", new HogQlLanguageVersion(1, 1, 0))) + .isInstanceOf(IllegalArgumentException.class) + .hasMessage("unsupported HogQL language version: 1.1.0"); + } + + @Test + public void testGrammarFilesMatchLanguageDescriptor() + throws IOException + { + HogQlLanguageContract contract = HogQlLanguageContract.current(); + MessageDigest aggregate = sha256Digest(); + + for (HogQlLanguageContract.GrammarFile file : contract.files()) { + byte[] content = readResource(LANGUAGE_RESOURCE_ROOT + "grammar/" + file.path()); + assertThat(sha256(content)).isEqualTo(file.sha256()); + aggregate.update(file.path().getBytes(StandardCharsets.UTF_8)); + aggregate.update((byte) 0); + aggregate.update(content); + aggregate.update((byte) 0); + } + + assertThat(HexFormat.of().formatHex(aggregate.digest())).isEqualTo(contract.grammarSha256()); + } + + @Test + public void testPublishedManifestsMatchLanguageDescriptor() + throws IOException + { + HogQlLanguageContract contract = HogQlLanguageContract.current(); + HogQlLanguageContract.GrammarFeatureManifest featureManifest = contract.grammarFeatureManifest(); + assertThat(sha256(readResource(LANGUAGE_RESOURCE_ROOT + featureManifest.path()))) + .isEqualTo(featureManifest.sha256()); + + for (HogQlLanguageContract.Corpus corpus : contract.corpora()) { + assertThat(sha256(readResource(LANGUAGE_RESOURCE_ROOT + corpus.manifestPath()))) + .isEqualTo(corpus.manifestSha256()); + assertThat(sha256(readResource(LANGUAGE_RESOURCE_ROOT + corpus.oraclePath()))) + .isEqualTo(corpus.oracleSha256()); + } + } + + private static byte[] readResource(String path) + throws IOException + { + try (InputStream input = TestHogQlLanguageContract.class.getResourceAsStream(path)) { + if (input == null) { + fail("missing test resource: " + path); + } + return input.readAllBytes(); + } + } + + private static String sha256(byte[] content) + { + return HexFormat.of().formatHex(sha256Digest().digest(content)); + } + + private static MessageDigest sha256Digest() + { + try { + return MessageDigest.getInstance("SHA-256"); + } + catch (NoSuchAlgorithmException e) { + throw new AssertionError(e); + } + } +} diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java new file mode 100644 index 000000000000..f96e144d3951 --- /dev/null +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParser.java @@ -0,0 +1,606 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.AliasedRelation; +import io.trino.hogql.parser.tree.HogQlQuery.BinaryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.CastExpression; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnReference; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsList; +import io.trino.hogql.parser.tree.HogQlQuery.ColumnsRegex; +import io.trino.hogql.parser.tree.HogQlQuery.CommonTableReference; +import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; +import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; +import io.trino.hogql.parser.tree.HogQlQuery.InSubqueryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IntervalExpression; +import io.trino.hogql.parser.tree.HogQlQuery.IntervalUnit; +import io.trino.hogql.parser.tree.HogQlQuery.JoinOn; +import io.trino.hogql.parser.tree.HogQlQuery.JoinRelation; +import io.trino.hogql.parser.tree.HogQlQuery.JoinType; +import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.MemberAccessExpression; +import io.trino.hogql.parser.tree.HogQlQuery.NullPlacement; +import io.trino.hogql.parser.tree.HogQlQuery.PivotRelation; +import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; +import io.trino.hogql.parser.tree.HogQlQuery.ScalarSubqueryExpression; +import io.trino.hogql.parser.tree.HogQlQuery.SetOperation; +import io.trino.hogql.parser.tree.HogQlQuery.SetOperationType; +import io.trino.hogql.parser.tree.HogQlQuery.SortDirection; +import io.trino.hogql.parser.tree.HogQlQuery.Star; +import io.trino.hogql.parser.tree.HogQlQuery.StarReplacement; +import io.trino.hogql.parser.tree.HogQlQuery.SubqueryRelation; +import io.trino.hogql.parser.tree.HogQlQuery.SubscriptExpression; +import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; +import io.trino.hogql.parser.tree.HogQlQuery.ValuesRelation; +import io.trino.hogql.parser.tree.HogQlSyntaxTree; +import io.trino.hogql.parser.tree.HogQlSyntaxTree.Element; +import io.trino.hogql.parser.tree.HogQlSyntaxTree.Node; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.stream.Stream; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlParser +{ + private final HogQlParser parser = new HogQlParser(); + + @ParameterizedTest + @ValueSource(strings = { + "SELECT", + "SELECT * FROM first JOIN second", + "SELECT * FROM first NATURAL JOIN second", + "SELECT * FROM first LEFT SEMI JOIN second ON first.id = second.id", + "SELECT * FROM first ASOF JOIN second ON first.id = second.id", + "SELECT * FROM first POSITIONAL JOIN second", + "SELECT * FROM first, second", + "SELECT function(1)(value)", + "SELECT tuple_value.0", + "SELECT array_value[1:2]", + "SELECT array_value?.[1]", + "SELECT tuple_value?.1", + "DROP TABLE events", + "SELECT 1; SELECT 2", + "SELECT * FROM", + "SELECT {1 + 2}", + "SELECT 1 GROUP BY ALL", + "WITH RECURSIVE x AS (SELECT 1) SELECT * FROM x", + "WITH x AS MATERIALIZED (SELECT 1) SELECT * FROM x", + "WITH x AS NOT MATERIALIZED (SELECT 1) SELECT * FROM x", + "WITH x USING KEY (id) AS (SELECT 1) SELECT * FROM x", + "WITH x AS (SELECT * FROM x) SELECT * FROM x", + "WITH x AS (SELECT 1), y AS (WITH x AS (SELECT * FROM x) SELECT * FROM x) SELECT * FROM y", + "WITH x AS (SELECT 1), x AS (SELECT 2) SELECT * FROM x", + "SELECT * FROM events e JOIN (SELECT * FROM persons WHERE personId = e.person_id) p ON true", + "SELECT * FROM events e JOIN (SELECT person_id) p ON true", + "SELECT 1 UNION BY NAME SELECT 1", + "SELECT 1 UNION ALL BY NAME SELECT 1", + "SELECT 1 INTERSECT BY NAME SELECT 1", + "SELECT 1 EXCEPT BY NAME SELECT 1", + "SELECT * FROM (VALUES (1, 2), (3)) AS data(first, second)", + "SELECT * FROM (VALUES (1, 2)) AS data(first)", + }) + public void testRejectsSyntaxWithoutAnAstMapping(String hogql) + { + assertThatThrownBy(() -> parser.parseStatement(hogql)) + .isInstanceOf(HogQlParsingException.class) + .hasMessageStartingWith("line "); + } + + @Test + public void testUsesCanonicalGrammarForExpressionsAndWhereClause() + { + HogQlQuery query = parser.parseStatement("SELECT event + 1 * 2 FROM events WHERE event >= 3 AND NOT false"); + + assertThat(query.projections()).hasSize(1); + assertThat(query.from()).isPresent(); + assertThat(query.where()).isPresent(); + } + + @Test + public void testBuildsCanonicalIntervalExpressions() + { + HogQlQuery query = parser.parseStatement("SELECT INTERVAL 1 WEEK, INTERVAL event QUARTER, INTERVAL '5 months'"); + + assertThat(query.projections()).extracting(projection -> ((IntervalExpression) ((ExpressionProjection) projection).expression()).unit()) + .containsExactly(IntervalUnit.WEEK, IntervalUnit.QUARTER, IntervalUnit.MONTH); + IntervalExpression stringInterval = (IntervalExpression) ((ExpressionProjection) query.projections().get(2)).expression(); + assertThat(stringInterval.value()).isInstanceOfSatisfying(Literal.class, value -> { + assertThat(value.kind()).isEqualTo(HogQlQuery.LiteralKind.INTEGER); + assertThat(value.value()).isEqualTo("5"); + }); + } + + @Test + public void testRejectsInvalidCombinedStringIntervals() + { + assertThatThrownBy(() -> parser.parseStatement("SELECT INTERVAL 'twenty days'")) + .isInstanceOf(HogQlParsingException.class) + .hasMessageContaining("Unsupported interval count: 'twenty' is not a valid integer"); + assertThatThrownBy(() -> parser.parseStatement("SELECT INTERVAL '9223372036854775808 day'")) + .isInstanceOf(HogQlParsingException.class) + .hasMessageContaining("Unsupported interval count: '9223372036854775808' is too large"); + assertThatThrownBy(() -> parser.parseStatement("SELECT INTERVAL '1 SECOND'")) + .isInstanceOf(HogQlParsingException.class) + .hasMessageContaining("Unsupported interval unit: SECOND"); + assertThatThrownBy(() -> parser.parseStatement("SELECT INTERVAL '1 dayss'")) + .isInstanceOf(HogQlParsingException.class) + .hasMessageContaining("Unsupported interval unit: dayss"); + assertThatThrownBy(() -> parser.parseStatement("SELECT INTERVAL 'x'")) + .isInstanceOf(HogQlParsingException.class) + .hasMessageContaining("Unsupported interval type: must be in the format ' '"); + } + + @Test + public void testPreservesNamedPlaceholderNamesAndSourceSpans() + { + HogQlQuery query = parser.parseStatement("SELECT {later},\n {first} + {later}"); + Placeholder later = (Placeholder) ((ExpressionProjection) query.projections().getFirst()).expression(); + BinaryExpression addition = (BinaryExpression) ((ExpressionProjection) query.projections().get(1)).expression(); + Placeholder first = (Placeholder) addition.left(); + Placeholder repeated = (Placeholder) addition.right(); + + assertThat(later.name()).isEqualTo("later"); + assertThat(later.span()).isEqualTo(new HogQlQuery.SourceSpan(7, 14, 1, 8, 1, 15)); + assertThat(first.name()).isEqualTo("first"); + assertThat(first.span()).isEqualTo(new HogQlQuery.SourceSpan(17, 24, 2, 2, 2, 9)); + assertThat(repeated.name()).isEqualTo("later"); + assertThat(repeated.span()).isEqualTo(new HogQlQuery.SourceSpan(27, 34, 2, 12, 2, 19)); + } + + @Test + public void testBuildsQualifiedStarWithExclusionsAndSourceSpans() + { + HogQlQuery query = parser.parseStatement("SELECT \"Event Alias\".* EXCLUDE (\"event\", personId) FROM events AS \"Event Alias\""); + + Star star = (Star) query.projections().getFirst(); + assertThat(star.qualifier()).singleElement().satisfies(qualifier -> { + assertThat(qualifier.value()).isEqualTo("Event Alias"); + assertThat(qualifier.delimited()).isTrue(); + assertThat(qualifier.span()).isEqualTo(new HogQlQuery.SourceSpan(7, 20, 1, 8, 1, 21)); + }); + assertThat(star.exclusions()).extracting(exclusion -> exclusion.parts().getFirst().value()).containsExactly("event", "personId"); + assertThat(star.exclusions()).extracting(exclusion -> exclusion.parts().getFirst().delimited()).containsExactly(true, false); + assertThat(star.exclusions().getFirst().span()).isEqualTo(new HogQlQuery.SourceSpan(32, 39, 1, 33, 1, 40)); + assertThat(star.exclusions().get(1).span()).isEqualTo(new HogQlQuery.SourceSpan(41, 49, 1, 42, 1, 50)); + assertThat(star.span()).isEqualTo(new HogQlQuery.SourceSpan(7, 50, 1, 8, 1, 51)); + } + + @Test + public void testPreservesQualifiedStarExclusionPath() + { + Star star = (Star) parser.parseStatement("SELECT * EXCLUDE (analytics.events.event) FROM events").projections().getFirst(); + + assertThat(star.exclusions()).singleElement().satisfies(exclusion -> + assertThat(exclusion.parts()).extracting(HogQlQuery.Identifier::value).containsExactly("analytics", "events", "event")); + } + + @Test + public void testBuildsColumnsSelectorsAndReplacementAst() + { + HogQlQuery query = parser.parseStatement( + "SELECT COLUMNS('^(event|person)'), COLUMNS(event, personId + 1), " + + "COLUMNS(events.* REPLACE (personId AS event, event AS \"personId\")) FROM events"); + + ColumnsRegex regex = (ColumnsRegex) query.projections().getFirst(); + assertThat(regex.pattern()).isEqualTo("^(event|person)"); + assertThat(regex.patternSpan()).isEqualTo(new HogQlQuery.SourceSpan(15, 32, 1, 16, 1, 33)); + + ColumnsList explicit = (ColumnsList) query.projections().get(1); + assertThat(explicit.expressions()).hasSize(2); + assertThat(explicit.expressions().getFirst()).isInstanceOf(ColumnReference.class); + assertThat(explicit.expressions().get(1)).isInstanceOf(BinaryExpression.class); + + Star star = (Star) query.projections().get(2); + assertThat(star.qualifier()).extracting(HogQlQuery.Identifier::value).containsExactly("events"); + assertThat(star.replacements()).extracting(replacement -> replacement.target().value()).containsExactly("event", "personId"); + assertThat(star.replacements()).extracting(replacement -> replacement.target().delimited()).containsExactly(false, true); + assertThat(star.replacements()).extracting(StarReplacement::expression).allSatisfy(expression -> + assertThat(expression).isInstanceOf(ColumnReference.class)); + } + + @Test + public void testPreservesParameterizedAndNestedCastTypeSyntax() + { + HogQlQuery query = parser.parseStatement( + "SELECT CAST(value AS Nullable(Decimal(18, 4))), " + + "TRY_CAST(value AS Tuple(id Int64, payload Array(String)))"); + + CastExpression decimal = (CastExpression) ((ExpressionProjection) query.projections().getFirst()).expression(); + CastExpression tuple = (CastExpression) ((ExpressionProjection) query.projections().get(1)).expression(); + + assertThat(decimal.type().value()).isEqualTo("Nullable(Decimal(18, 4))"); + assertThat(decimal.type().span()).isEqualTo(new HogQlQuery.SourceSpan(21, 45, 1, 22, 1, 46)); + assertThat(decimal.safe()).isFalse(); + assertThat(decimal.typeDialect()).isEqualTo(HogQlQuery.CastTypeDialect.HOGQL); + assertThat(tuple.type().value()).isEqualTo("Tuple(id Int64, payload Array(String))"); + assertThat(tuple.safe()).isTrue(); + } + + @Test + public void testBuildsDistinctOrderingAndPaginationAst() + { + HogQlQuery query = parser.parseStatement("SELECT DISTINCT event FROM events ORDER BY event DESC NULLS FIRST LIMIT 10 OFFSET 2"); + + assertThat(query.distinct()).isTrue(); + assertThat(query.orderBy()).singleElement().satisfies(sortItem -> { + assertThat(sortItem.direction()).isEqualTo(SortDirection.DESCENDING); + assertThat(sortItem.nullPlacement()).isEqualTo(NullPlacement.FIRST); + }); + assertThat(query.limit()).get().extracting(HogQlQuery.Literal.class::cast).extracting(HogQlQuery.Literal::value).isEqualTo("10"); + assertThat(query.offset()).get().extracting(HogQlQuery.Literal.class::cast).extracting(HogQlQuery.Literal::value).isEqualTo("2"); + } + + @Test + public void testBuildsLimitByAst() + { + HogQlQuery query = parser.parseStatement("SELECT event FROM events LIMIT 2, 3 BY event, properties.plan LIMIT 10"); + + assertThat(query.limitBy()).get().satisfies(limitBy -> { + assertThat(limitBy.limit()).extracting(HogQlQuery.Literal.class::cast).extracting(HogQlQuery.Literal::value).isEqualTo("3"); + assertThat(limitBy.offset()).get().extracting(HogQlQuery.Literal.class::cast).extracting(HogQlQuery.Literal::value).isEqualTo("2"); + assertThat(limitBy.partitionBy()).hasSize(2); + }); + assertThat(query.limit()).get().extracting(HogQlQuery.Literal.class::cast).extracting(HogQlQuery.Literal::value).isEqualTo("10"); + + HogQlQuery offsetQuery = parser.parseStatement("SELECT event FROM events LIMIT 3 OFFSET 2 BY event"); + assertThat(offsetQuery.limitBy()).get().satisfies(limitBy -> { + assertThat(limitBy.limit()).extracting(HogQlQuery.Literal.class::cast).extracting(HogQlQuery.Literal::value).isEqualTo("3"); + assertThat(limitBy.offset()).get().extracting(HogQlQuery.Literal.class::cast).extracting(HogQlQuery.Literal::value).isEqualTo("2"); + }); + } + + @Test + public void testBuildsCollectionAccessWithSourceSpans() + { + HogQlQuery query = parser.parseStatement("SELECT [10,{element}][{index}], (1, 'two').2, {payload}.plan"); + + SubscriptExpression arrayAccess = (SubscriptExpression) ((ExpressionProjection) query.projections().getFirst()).expression(); + assertThat(arrayAccess.base()).isInstanceOf(HogQlQuery.ArrayExpression.class); + assertThat(arrayAccess.index()).isInstanceOf(Placeholder.class); + assertThat(arrayAccess.span()).isEqualTo(new HogQlQuery.SourceSpan(7, 30, 1, 8, 1, 31)); + assertThat(arrayAccess.index().span()).isEqualTo(new HogQlQuery.SourceSpan(22, 29, 1, 23, 1, 30)); + + SubscriptExpression tupleAccess = (SubscriptExpression) ((ExpressionProjection) query.projections().get(1)).expression(); + assertThat(tupleAccess.base()).isInstanceOf(TupleExpression.class); + assertThat(tupleAccess.index()).isInstanceOf(HogQlQuery.Literal.class); + assertThat(tupleAccess.span()).isEqualTo(new HogQlQuery.SourceSpan(32, 44, 1, 33, 1, 45)); + assertThat(tupleAccess.index().span()).isEqualTo(new HogQlQuery.SourceSpan(43, 44, 1, 44, 1, 45)); + + MemberAccessExpression memberAccess = (MemberAccessExpression) ((ExpressionProjection) query.projections().get(2)).expression(); + assertThat(memberAccess.base()).isInstanceOf(Placeholder.class); + assertThat(memberAccess.member().value()).isEqualTo("plan"); + assertThat(memberAccess.span()).isEqualTo(new HogQlQuery.SourceSpan(46, 60, 1, 47, 1, 61)); + assertThat(memberAccess.member().span()).isEqualTo(new HogQlQuery.SourceSpan(56, 60, 1, 57, 1, 61)); + } + + @Test + public void testBuildsPropertyAccessPaths() + { + HogQlQuery query = parser.parseStatement("SELECT properties.browser, e.properties.browser FROM events e"); + + ColumnReference unqualified = (ColumnReference) ((ExpressionProjection) query.projections().getFirst()).expression(); + assertThat(unqualified.parts()).extracting(HogQlQuery.Identifier::value).containsExactly("properties", "browser"); + assertThat(unqualified.span()).isEqualTo(new HogQlQuery.SourceSpan(7, 25, 1, 8, 1, 26)); + + ColumnReference qualified = (ColumnReference) ((ExpressionProjection) query.projections().get(1)).expression(); + assertThat(qualified.parts()).extracting(HogQlQuery.Identifier::value).containsExactly("e", "properties", "browser"); + assertThat(qualified.span()).isEqualTo(new HogQlQuery.SourceSpan(27, 47, 1, 28, 1, 48)); + } + + @Test + public void testBuildsOrdinaryGroupingHavingAndAggregateFunctionAst() + { + HogQlQuery query = parser.parseStatement("SELECT country, count(DISTINCT person_id) FROM events GROUP BY country, lower(source) HAVING count(*) > 1"); + + assertThat(query.groupBy()).hasSize(2); + assertThat(query.having()).isPresent(); + FunctionCall count = (FunctionCall) ((ExpressionProjection) query.projections().get(1)).expression(); + assertThat(count.name().value()).isEqualTo("count"); + assertThat(count.distinct()).isTrue(); + assertThat(count.arguments()).singleElement().isInstanceOf(ColumnReference.class); + } + + @Test + public void testBuildsAliasedJoinAstWithSourceSpans() + { + HogQlQuery query = parser.parseStatement("SELECT e.id\nFROM events AS e\nLEFT JOIN persons AS p ON e.person_id = p.id"); + + JoinRelation join = (JoinRelation) query.from().orElseThrow(); + assertThat(join.type()).isEqualTo(JoinType.LEFT); + assertThat(join.span()).isEqualTo(new HogQlQuery.SourceSpan(17, 73, 2, 6, 3, 45)); + AliasedRelation left = (AliasedRelation) join.left(); + assertThat(left.alias().value()).isEqualTo("e"); + assertThat(left.span()).isEqualTo(new HogQlQuery.SourceSpan(17, 28, 2, 6, 2, 17)); + JoinOn criteria = (JoinOn) join.criteria().orElseThrow(); + assertThat(criteria.span()).isEqualTo(new HogQlQuery.SourceSpan(52, 73, 3, 24, 3, 45)); + } + + @ParameterizedTest + @ValueSource(strings = {"LEFT ANY JOIN", "ANY LEFT JOIN"}) + public void testBuildsLeftAnyJoinAst(String joinOperator) + { + HogQlQuery query = parser.parseStatement( + "SELECT e.id FROM events e " + joinOperator + " persons p ON e.person_id = p.id"); + + JoinRelation join = (JoinRelation) query.from().orElseThrow(); + assertThat(join.type()).isEqualTo(JoinType.LEFT_ANY); + assertThat(join.criteria()).hasValueSatisfying(JoinOn.class::isInstance); + } + + @Test + public void testBuildsInnerAnyJoinAst() + { + HogQlQuery query = parser.parseStatement( + "SELECT e.id FROM events e ANY INNER JOIN persons p USING (person_id)"); + + JoinRelation join = (JoinRelation) query.from().orElseThrow(); + assertThat(join.type()).isEqualTo(JoinType.INNER_ANY); + } + + @Test + public void testUnwrapsNestedExpressionAlias() + { + HogQlQuery query = parser.parseStatement("SELECT md5((first || second) AS TEXT)"); + FunctionCall md5 = (FunctionCall) ((ExpressionProjection) query.projections().getFirst()).expression(); + + assertThat(md5.arguments()).hasSize(1); + assertThat(md5.arguments().getFirst()).isInstanceOf(BinaryExpression.class); + } + + @Test + public void testExpandsWithExpressionAliases() + { + HogQlQuery query = parser.parseStatement( + "WITH (SELECT max(date) FROM daily) AS last_day SELECT last_day, last_day + 1"); + + assertThat(query.with()).isEmpty(); + assertThat(((ExpressionProjection) query.projections().get(0)).expression()) + .isInstanceOf(ScalarSubqueryExpression.class); + assertThat(((ExpressionProjection) query.projections().get(1)).expression()) + .isInstanceOfSatisfying( + BinaryExpression.class, + addition -> assertThat(addition.left()).isInstanceOf(ScalarSubqueryExpression.class)); + } + + @Test + public void testBuildsPivotAstWithCompositeKeysAliasesAndSourceSpans() + { + String hogql = "SELECT * FROM orders PIVOT (sum(totalprice) AS total FOR (orderstatus, custkey) " + + "IN (('F', 1) AS filled, ('O', 2) AS open) GROUP BY clerk)"; + + PivotRelation pivot = (PivotRelation) parser.parseStatement(hogql).from().orElseThrow(); + + assertThat(pivot.input()).isInstanceOf(HogQlQuery.TableReference.class); + assertThat(pivot.span()).isEqualTo(span(hogql, hogql.substring(hogql.indexOf("orders")))); + assertThat(pivot.aggregations()).singleElement().satisfies(aggregation -> { + assertThat(aggregation.expression()).isInstanceOf(FunctionCall.class); + assertThat(aggregation.alias()).get().extracting(HogQlQuery.Identifier::value).isEqualTo("total"); + assertThat(aggregation.span()).isEqualTo(span(hogql, "sum(totalprice) AS total")); + }); + assertThat(pivot.pivotColumns()).extracting(expression -> ((ColumnReference) expression).parts().getLast().value()) + .containsExactly("orderstatus", "custkey"); + assertThat(pivot.pivotColumns().getFirst().span()).isEqualTo(span(hogql, "orderstatus")); + assertThat(pivot.valueGroups()).hasSize(2); + assertThat(pivot.valueGroups().getFirst().values()).hasSize(2); + assertThat(pivot.valueGroups().getFirst().alias()).get().extracting(HogQlQuery.Identifier::value).isEqualTo("filled"); + assertThat(pivot.valueGroups().getFirst().span()).isEqualTo(span(hogql, "('F', 1) AS filled")); + assertThat(pivot.groupBy()).singleElement().isInstanceOf(ColumnReference.class); + } + + @Test + public void testAttachesPivotToWholeJoin() + { + HogQlQuery query = parser.parseStatement( + "SELECT * FROM orders o JOIN customer c ON o.custkey = c.custkey " + + "PIVOT (count(*) FOR o.orderstatus IN ('F'))"); + + assertThat(query.from()).get().isInstanceOfSatisfying(PivotRelation.class, pivot -> + assertThat(pivot.input()).isInstanceOf(JoinRelation.class)); + } + + @Test + public void testRejectsPivotShapesWithoutAStockRepresentation() + { + assertThatThrownBy(() -> parser.parseStatement( + "SELECT * FROM orders PIVOT (sum(totalprice) FOR orderstatus IN ('F') custkey IN (1))")) + .isInstanceOf(HogQlParsingException.class) + .hasMessageContaining("multiple PIVOT column clauses"); + assertThatThrownBy(() -> parser.parseStatement( + "SELECT * FROM orders PIVOT (sum(totalprice) FOR orderstatus + 1 IN ('F'))")) + .isInstanceOf(HogQlParsingException.class) + .hasMessageContaining("non-column PIVOT key"); + } + + @Test + public void testBuildsCteAndDerivedTableAstWithSourceSpans() + { + String hogql = "WITH base(id) AS (SELECT {cte}), next AS (SELECT id FROM base)\n" + + "SELECT d.id FROM (SELECT id FROM next WHERE id = {inner}) AS d WHERE d.id = {outer}"; + + HogQlQuery query = parser.parseStatement(hogql); + + assertThat(query.with()).hasSize(2); + assertThat(query.with().getFirst().name().value()).isEqualTo("base"); + assertThat(query.with().getFirst().columnAliases()).singleElement().satisfies(alias -> assertThat(alias.value()).isEqualTo("id")); + assertThat(query.with().getFirst().span()).isEqualTo(new HogQlQuery.SourceSpan(5, 31, 1, 6, 1, 32)); + assertThat(query.with().get(1).query().from()).get().isInstanceOf(CommonTableReference.class); + AliasedRelation derived = (AliasedRelation) query.from().orElseThrow(); + assertThat(derived.alias().value()).isEqualTo("d"); + SubqueryRelation subquery = (SubqueryRelation) derived.relation(); + assertThat(subquery.query().from()).get().isInstanceOf(CommonTableReference.class); + assertThat(subquery.span()).isEqualTo(new HogQlQuery.SourceSpan(80, 120, 2, 18, 2, 58)); + } + + @Test + public void testAllowsUnqualifiedColumnsFromUnaliasedNestedDerivedTables() + { + HogQlQuery query = parser.parseStatement( + "SELECT left_source.id FROM first_table left_source " + + "JOIN (SELECT id FROM (SELECT id FROM second_table)) right_source " + + "ON left_source.id = right_source.id"); + + assertThat(query.from()).get().isInstanceOf(JoinRelation.class); + } + + @Test + public void testBuildsCorrelatedInSubqueryWithSourceSpans() + { + HogQlQuery query = parser.parseStatement( + """ + SELECT o.orderkey + FROM orders o + WHERE o.custkey IN ( + SELECT c.custkey + FROM customer c + WHERE c.custkey = o.custkey + ) + """); + + InSubqueryExpression predicate = (InSubqueryExpression) query.where().orElseThrow(); + BinaryExpression correlation = (BinaryExpression) predicate.query().where().orElseThrow(); + ColumnReference outerReference = (ColumnReference) correlation.right(); + + assertThat(predicate.predicateSpan()).isEqualTo(new HogQlQuery.SourceSpan(48, 127, 3, 17, 7, 2)); + assertThat(predicate.query().span()).isEqualTo(new HogQlQuery.SourceSpan(57, 125, 4, 5, 6, 32)); + assertThat(outerReference.parts()).extracting(HogQlQuery.Identifier::value).containsExactly("o", "custkey"); + assertThat(outerReference.span()).isEqualTo(new HogQlQuery.SourceSpan(116, 125, 6, 23, 6, 32)); + } + + @Test + public void testBuildsCorrelatedScalarSubqueryWithSourceSpans() + { + HogQlQuery query = parser.parseStatement( + "SELECT o.orderkey, (SELECT c.custkey FROM customer c WHERE c.custkey = o.custkey) AS matched FROM orders o"); + + ScalarSubqueryExpression subquery = (ScalarSubqueryExpression) ((ExpressionProjection) query.projections().get(1)).expression(); + BinaryExpression correlation = (BinaryExpression) subquery.query().where().orElseThrow(); + ColumnReference outerReference = (ColumnReference) correlation.right(); + + assertThat(subquery.span()).isEqualTo(new HogQlQuery.SourceSpan(19, 81, 1, 20, 1, 82)); + assertThat(subquery.query().span()).isEqualTo(new HogQlQuery.SourceSpan(20, 80, 1, 21, 1, 81)); + assertThat(outerReference.parts()).extracting(HogQlQuery.Identifier::value).containsExactly("o", "custkey"); + assertThat(outerReference.span()).isEqualTo(new HogQlQuery.SourceSpan(71, 80, 1, 72, 1, 81)); + } + + @Test + public void testBuildsSetOperationAstWithPrecedenceAndSourceSpans() + { + HogQlQuery query = parser.parseStatement("SELECT {left}\nUNION ALL\nSELECT {middle}\nINTERSECT\nSELECT {right}"); + + SetOperation union = (SetOperation) query.body(); + assertThat(union.type()).isEqualTo(SetOperationType.UNION); + assertThat(union.distinct()).isFalse(); + assertThat(union.operatorSpan()).isEqualTo(new HogQlQuery.SourceSpan(14, 23, 2, 1, 2, 10)); + SetOperation intersect = (SetOperation) union.right().body(); + assertThat(intersect.type()).isEqualTo(SetOperationType.INTERSECT); + assertThat(intersect.distinct()).isTrue(); + assertThat(intersect.operatorSpan()).isEqualTo(new HogQlQuery.SourceSpan(40, 49, 4, 1, 4, 10)); + } + + @Test + public void testBuildsValuesRelationWithColumnSchema() + { + HogQlQuery query = parser.parseStatement("SELECT *\nFROM (VALUES (1, {first}), (2, {second})) AS data(id, label)"); + + AliasedRelation alias = (AliasedRelation) query.from().orElseThrow(); + assertThat(alias.columnAliases()) + .extracting(HogQlQuery.Identifier::value) + .containsExactly("id", "label"); + ValuesRelation values = (ValuesRelation) alias.relation(); + assertThat(values.rows()).hasSize(2).allSatisfy(row -> assertThat(row).hasSize(2)); + assertThat(values.span().startLine()).isEqualTo(2); + assertThat(values.span().startColumn()).isEqualTo(6); + } + + @Test + public void testSourceSpansUseUnicodeCodePointOffsetsAndEndPositions() + { + HogQlQuery query = parser.parseStatement("SELECT '😀',\n event"); + ExpressionProjection projection = (ExpressionProjection) query.projections().get(1); + ColumnReference reference = (ColumnReference) projection.expression(); + + assertThat(query.span()).isEqualTo(new HogQlQuery.SourceSpan(0, 18, 1, 1, 2, 7)); + assertThat(reference.span()).isEqualTo(new HogQlQuery.SourceSpan(13, 18, 2, 2, 2, 7)); + } + + @Test + public void testSyntaxTreeRepresentsReadOnlyGrammarBeforeLowering() + { + String hogql = "WITH x AS (SELECT 1) SELECT * FROM x UNION ALL SELECT event FROM events ORDER BY event LIMIT 10 OFFSET 2;"; + + HogQlSyntaxTree syntaxTree = parser.parseSyntax(hogql); + + assertThat(syntaxTree.languageClass()).isEqualTo(HogQlSyntaxTree.LanguageClass.READ_ONLY_QUERY); + assertThat(syntaxTree.root().span()).isEqualTo(new HogQlQuery.SourceSpan(0, hogql.length(), 1, 1, 1, hogql.length() + 1)); + assertThat(nodes(syntaxTree.root())) + .extracting(Node::rule) + .contains("withClause", "selectSetStmt", "subsequentSelectSetClause", "orderByClause", "limitAndOffsetClauseOptional"); + assertThat(nodes(syntaxTree.root())) + .extracting(Node::alternative) + .contains(java.util.Optional.of("WithExprSubquery"), java.util.Optional.of("ColumnExprIdentifier")); + } + + @Test + public void testSyntaxTreeClassifiesHogQlXAsNonQuerySyntax() + { + HogQlSyntaxTree syntaxTree = parser.parseSyntax(""); + + assertThat(syntaxTree.languageClass()).isEqualTo(HogQlSyntaxTree.LanguageClass.HOGQLX); + assertThat(nodes(syntaxTree.root())) + .extracting(Node::rule) + .contains("hogqlxTagElement"); + } + + @Test + public void testSyntaxTreeClassifiesBlockLambdaAsProcedural() + { + HogQlSyntaxTree syntaxTree = parser.parseSyntax("SELECT value -> { RETURN value }"); + + assertThat(syntaxTree.languageClass()).isEqualTo(HogQlSyntaxTree.LanguageClass.PROCEDURAL); + assertThat(nodes(syntaxTree.root())) + .extracting(Node::rule) + .contains("block", "returnStmt"); + } + + @Test + public void testParsesCanonicalExpressionEntryPointWithoutAQueryWrapper() + { + HogQlSyntaxTree syntaxTree = parser.parseExpressionSyntax("value + 1"); + + assertThat(syntaxTree.root().rule()).isEqualTo("expression"); + assertThat(syntaxTree.root().span()).isEqualTo(new HogQlQuery.SourceSpan(0, 9, 1, 1, 1, 10)); + assertThatThrownBy(() -> parser.parseExpressionSyntax("value + 1 trailing")) + .isInstanceOf(HogQlParsingException.class) + .hasMessageContaining("unexpected trailing input"); + } + + private static Stream nodes(Element element) + { + if (!(element instanceof Node node)) { + return Stream.empty(); + } + return Stream.concat(Stream.of(node), node.children().stream().flatMap(TestHogQlParser::nodes)); + } + + private static HogQlQuery.SourceSpan span(String source, String value) + { + int start = source.indexOf(value); + return new HogQlQuery.SourceSpan(start, start + value.length(), 1, start + 1, 1, start + value.length() + 1); + } +} diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParserLimits.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParserLimits.java new file mode 100644 index 000000000000..2db41271590a --- /dev/null +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParserLimits.java @@ -0,0 +1,89 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlParserLimits +{ + private static final int LARGE_VALID_QUERY_TOKEN_COUNT = 160_000; + + @Test + public void testRejectsTokenBomb() + { + HogQlParser parser = new HogQlParser(new HogQlParserLimits(5, 100, 1_000)); + + assertThatThrownBy(() -> parser.parseSyntax("SELECT one, two, three")) + .isInstanceOf(HogQlParsingException.class) + .hasMessageContaining("token limit exceeded"); + } + + @Test + public void testRejectsDeeplyNestedExpression() + { + HogQlParser parser = new HogQlParser(new HogQlParserLimits(1_000, 32, 10_000)); + String query = "SELECT " + "(".repeat(64) + "1" + ")".repeat(64); + + assertThatThrownBy(() -> parser.parseSyntax(query)) + .isInstanceOf(HogQlParsingException.class) + .hasMessageContaining("parse depth limit exceeded"); + } + + @Test + public void testRejectsParseTreeNodeBomb() + { + HogQlParser parser = new HogQlParser(new HogQlParserLimits(1_000, 100, 25)); + + assertThatThrownBy(() -> parser.parseSyntax("SELECT one, two, three, four")) + .isInstanceOf(HogQlParsingException.class) + .hasMessageContaining("parse tree node limit exceeded"); + } + + @Test + public void testAcceptsInputWithinInjectedLimits() + { + HogQlParser parser = new HogQlParser(new HogQlParserLimits(100, 100, 1_000)); + + assertThatCode(() -> parser.parseSyntax("SELECT event FROM events WHERE event = 'signup'")) + .doesNotThrowAnyException(); + } + + @Test + public void testDefaultLimitsAcceptLargeValidQuery() + { + assertThatCode(() -> new HogQlParser().parseSyntax(queryWithTokenCount(LARGE_VALID_QUERY_TOKEN_COUNT))) + .doesNotThrowAnyException(); + } + + @Test + public void testDefaultTokenLimitRemainsBounded() + { + int tokenCount = HogQlParserLimits.defaults().maxTokens() + 1; + + assertThatThrownBy(() -> new HogQlParser().parseSyntax(queryWithTokenCount(tokenCount))) + .isInstanceOf(HogQlParsingException.class) + .hasMessageContaining("token limit exceeded"); + } + + private static String queryWithTokenCount(int tokenCount) + { + if (tokenCount < 3) { + throw new IllegalArgumentException("tokenCount must be at least 3"); + } + return "SELECT" + " /**/".repeat(tokenCount - 3) + " 1"; + } +} diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParserRobustness.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParserRobustness.java new file mode 100644 index 000000000000..23d626e3b914 --- /dev/null +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlParserRobustness.java @@ -0,0 +1,325 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import io.airlift.json.JsonCodec; +import io.trino.hogql.parser.tree.HogQlQuery.SourceSpan; +import io.trino.hogql.parser.tree.HogQlSyntaxTree; +import io.trino.hogql.parser.tree.HogQlSyntaxTree.Element; +import io.trino.hogql.parser.tree.HogQlSyntaxTree.Node; +import io.trino.hogql.parser.tree.HogQlSyntaxTree.Token; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayDeque; +import java.util.Arrays; +import java.util.Deque; +import java.util.List; +import java.util.Optional; +import java.util.Random; +import java.util.stream.IntStream; +import java.util.stream.Stream; + +import static io.airlift.json.JsonCodec.jsonCodec; +import static java.util.Objects.requireNonNull; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.fail; + +public class TestHogQlParserRobustness +{ + private static final long RANDOM_SEED = 0x484C0FFEEL; + private static final int GENERATED_ROUNDS = 64; + private static final int MUTATIONS_PER_INPUT = 2; + private static final String MALFORMED_RESOURCE = "/io/trino/hogql/parser/language/1.0.0/robustness/malformed-regressions.jsonl"; + private static final JsonCodec MALFORMED_REGRESSION_CODEC = jsonCodec(MalformedRegression.class); + + private static final List ATOMS = List.of( + "event", + "value", + "enabled", + "properties.plan", + "0", + "17", + "true", + "NULL", + "'example'", + "'😀'"); + private static final List INSERTIONS = List.of("(", ")", "]", "'", "@", ","); + private static final List APPENDAGES = List.of(" +", " )", " '", " FROM", " WHERE", " /*"); + + private final HogQlParser parser = new HogQlParser(); + + @Test + public void testGeneratedAndMutatedInputsHaveOnlyDocumentedOutcomes() + { + Random random = new Random(RANDOM_SEED); + List generated = IntStream.range(0, GENERATED_ROUNDS) + .boxed() + .flatMap(index -> generatedInputs(random, index)) + .toList(); + + assertThat(generated).hasSize(GENERATED_ROUNDS * 2); + for (GeneratedInput input : generated) { + Optional outcome = parseSafely(input.source(), input.id()); + assertThat(outcome) + .as("generated input %s must be valid: %s", input.id(), display(input.source())) + .isPresent(); + outcome.ifPresent(tree -> assertValidTree(input.source(), tree)); + + for (int mutation = 0; mutation < MUTATIONS_PER_INPUT; mutation++) { + String mutated = mutate(random, input.source()); + parseSafely(mutated, input.id() + "-mutation-" + mutation) + .ifPresent(tree -> assertValidTree(mutated, tree)); + } + } + } + + @Test + public void testMinimizedMalformedRegressionSeedsAreRejectedCleanly() + throws IOException + { + List regressions = loadMalformedRegressions(); + + assertThat(regressions) + .extracting(MalformedRegression::id) + .containsExactly( + "missing-projection", + "trailing-binary-operator", + "unclosed-tuple", + "unexpected-character", + "missing-source", + "missing-predicate", + "unclosed-string", + "second-statement"); + for (MalformedRegression regression : regressions) { + assertThat(regression.source().codePointCount(0, regression.source().length())) + .as("regression %s remains minimized", regression.id()) + .isLessThanOrEqualTo(32); + assertThat(parseSafely(regression.source(), regression.id())) + .as("malformed regression must be rejected: %s", regression.id()) + .isEmpty(); + } + } + + private Optional parseSafely(String source, String id) + { + try { + return Optional.of(parser.parseSyntax(source)); + } + catch (HogQlParsingException _) { + return Optional.empty(); + } + catch (RuntimeException | Error unexpected) { + throw new AssertionError("unexpected parser failure for " + id + ": " + display(source), unexpected); + } + } + + private static Stream generatedInputs(Random random, int index) + { + String expression = expression(random, 3); + return Stream.of( + new GeneratedInput("expression-" + index, "SELECT " + expression), + new GeneratedInput("query-" + index, query(random, expression))); + } + + private static String expression(Random random, int depth) + { + if (depth == 0) { + return ATOMS.get(random.nextInt(ATOMS.size())); + } + return switch (random.nextInt(8)) { + case 0 -> "(" + expression(random, depth - 1) + ")"; + case 1 -> "NOT (" + expression(random, depth - 1) + ")"; + case 2 -> "-(" + expression(random, depth - 1) + ")"; + case 3 -> "(" + expression(random, depth - 1) + " + " + expression(random, depth - 1) + ")"; + case 4 -> "(" + expression(random, depth - 1) + " = " + expression(random, depth - 1) + ")"; + case 5 -> "coalesce(" + expression(random, depth - 1) + ", " + expression(random, depth - 1) + ")"; + case 6 -> "[" + expression(random, depth - 1) + ", " + expression(random, depth - 1) + "]"; + default -> "CASE WHEN " + predicate(random, depth - 1) + " THEN " + expression(random, depth - 1) + " ELSE 0 END"; + }; + } + + private static String predicate(Random random, int depth) + { + if (depth == 0) { + return switch (random.nextInt(4)) { + case 0 -> "event = '$pageview'"; + case 1 -> "value >= 0"; + case 2 -> "enabled"; + default -> "properties.plan IS NOT NULL"; + }; + } + return switch (random.nextInt(4)) { + case 0 -> "NOT (" + predicate(random, depth - 1) + ")"; + case 1 -> "(" + predicate(random, depth - 1) + " AND " + predicate(random, depth - 1) + ")"; + case 2 -> "(" + predicate(random, depth - 1) + " OR " + predicate(random, depth - 1) + ")"; + default -> "(" + expression(random, depth - 1) + " = " + expression(random, depth - 1) + ")"; + }; + } + + private static String query(Random random, String projection) + { + return switch (random.nextInt(6)) { + case 0 -> "SELECT " + projection; + case 1 -> "SELECT " + projection + " AS result FROM events"; + case 2 -> "SELECT " + projection + ", " + expression(random, 2) + " FROM events WHERE " + predicate(random, 2); + case 3 -> "SELECT\n " + projection + "\r\nFROM events\nWHERE " + predicate(random, 2); + case 4 -> "WITH sample AS (SELECT " + projection + " AS result) SELECT result FROM sample"; + default -> "SELECT " + projection + " FROM events ORDER BY event LIMIT " + (random.nextInt(20) + 1); + }; + } + + private static String mutate(Random random, String source) + { + int codePointLength = source.codePointCount(0, source.length()); + return switch (random.nextInt(6)) { + case 0 -> source.substring(0, source.offsetByCodePoints(0, random.nextInt(codePointLength + 1))); + case 1 -> deleteCodePoint(source, random.nextInt(codePointLength)); + case 2 -> insertAtCodePoint(source, random.nextInt(codePointLength + 1), INSERTIONS.get(random.nextInt(INSERTIONS.size()))); + case 3 -> replaceCodePoint(source, random.nextInt(codePointLength), ')'); + case 4 -> duplicateCodePoint(source, random.nextInt(codePointLength)); + default -> source + APPENDAGES.get(random.nextInt(APPENDAGES.size())); + }; + } + + private static String deleteCodePoint(String source, int codePointOffset) + { + int start = source.offsetByCodePoints(0, codePointOffset); + int end = source.offsetByCodePoints(start, 1); + return source.substring(0, start) + source.substring(end); + } + + private static String insertAtCodePoint(String source, int codePointOffset, String insertion) + { + int offset = source.offsetByCodePoints(0, codePointOffset); + return source.substring(0, offset) + insertion + source.substring(offset); + } + + private static String replaceCodePoint(String source, int codePointOffset, int replacement) + { + int start = source.offsetByCodePoints(0, codePointOffset); + int end = source.offsetByCodePoints(start, 1); + return source.substring(0, start) + Character.toString(replacement) + source.substring(end); + } + + private static String duplicateCodePoint(String source, int codePointOffset) + { + int start = source.offsetByCodePoints(0, codePointOffset); + int end = source.offsetByCodePoints(start, 1); + return source.substring(0, end) + source.substring(start); + } + + private static void assertValidTree(String source, HogQlSyntaxTree tree) + { + int sourceLength = source.codePointCount(0, source.length()); + Deque pending = new ArrayDeque<>(); + pending.add(new PendingElement(tree.root(), Optional.empty())); + + while (!pending.isEmpty()) { + PendingElement current = pending.removeFirst(); + Element element = current.element(); + SourceSpan span = element.span(); + assertThat(span.startOffset()).isBetween(0, sourceLength); + assertThat(span.endOffset()).isBetween(span.startOffset(), sourceLength); + assertPosition(source, span.startOffset(), span.startLine(), span.startColumn()); + assertPosition(source, span.endOffset(), span.endLine(), span.endColumn()); + current.parentSpan().ifPresent(parent -> { + assertThat(span.startOffset()).isGreaterThanOrEqualTo(parent.startOffset()); + assertThat(span.endOffset()).isLessThanOrEqualTo(parent.endOffset()); + }); + + if (element instanceof Node node) { + node.children().forEach(child -> pending.addLast(new PendingElement(child, Optional.of(span)))); + } + else if (element instanceof Token token) { + int start = source.offsetByCodePoints(0, span.startOffset()); + int end = source.offsetByCodePoints(0, span.endOffset()); + assertThat(source.substring(start, end)).isEqualTo(token.text()); + } + } + } + + private static void assertPosition(String source, int targetOffset, int expectedLine, int expectedColumn) + { + Position position = positionAt(source, targetOffset); + assertThat(expectedLine).isEqualTo(position.line()); + assertThat(expectedColumn).isEqualTo(position.column()); + } + + private static Position positionAt(String source, int targetOffset) + { + int line = 1; + int column = 1; + int codePointOffset = 0; + boolean previousWasCarriageReturn = false; + for (int charOffset = 0; codePointOffset < targetOffset; codePointOffset++) { + int codePoint = source.codePointAt(charOffset); + charOffset += Character.charCount(codePoint); + if (codePoint == '\n' && previousWasCarriageReturn) { + previousWasCarriageReturn = false; + } + else if (codePoint == '\n' || codePoint == '\r') { + line++; + column = 1; + previousWasCarriageReturn = codePoint == '\r'; + } + else { + column++; + previousWasCarriageReturn = false; + } + } + return new Position(line, column); + } + + private static List loadMalformedRegressions() + throws IOException + { + try (InputStream input = TestHogQlParserRobustness.class.getResourceAsStream(MALFORMED_RESOURCE)) { + if (input == null) { + fail("missing test resource: " + MALFORMED_RESOURCE); + } + return Arrays.stream(new String(input.readAllBytes(), StandardCharsets.UTF_8).split("\\R")) + .filter(line -> !line.isBlank()) + .map(MALFORMED_REGRESSION_CODEC::fromJson) + .toList(); + } + } + + private static String display(String source) + { + return source.replace("\r", "\\r").replace("\n", "\\n"); + } + + private record GeneratedInput(String id, String source) {} + + private record PendingElement(Element element, Optional parentSpan) {} + + private record Position(int line, int column) {} + + public record MalformedRegression(String id, String source) + { + @JsonCreator + public MalformedRegression( + @JsonProperty("id") String id, + @JsonProperty("source") String source) + { + this.id = requireNonNull(id, "id is null"); + this.source = requireNonNull(source, "source is null"); + } + } +} diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlSyntaxAstManifest.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlSyntaxAstManifest.java new file mode 100644 index 000000000000..6c9838214ef1 --- /dev/null +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlSyntaxAstManifest.java @@ -0,0 +1,124 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import io.airlift.json.JsonCodec; +import io.trino.hogql.parser.HogQlCompatibilityManifest.PublishedGrammarFeatureManifest; +import io.trino.hogql.parser.HogQlSyntaxAstManifest.Feature; +import io.trino.hogql.parser.HogQlSyntaxAstManifest.SourceSpanGuarantee; +import io.trino.hogql.parser.HogQlSyntaxAstManifest.SyntaxNodeKind; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.HexFormat; +import java.util.List; + +import static io.airlift.json.JsonCodec.jsonCodec; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlSyntaxAstManifest +{ + private static final String LANGUAGE_RESOURCE_ROOT = "/io/trino/hogql/parser/language/1.0.0/"; + private static final String GRAMMAR_FEATURE_RESOURCE = LANGUAGE_RESOURCE_ROOT + "grammar-features.json"; + private static final String ALTERNATIVE_IDENTITIES_RESOURCE = LANGUAGE_RESOURCE_ROOT + "grammar-alternative-identities.json"; + private static final JsonCodec GRAMMAR_FEATURE_MANIFEST_CODEC = jsonCodec(PublishedGrammarFeatureManifest.class); + + @Test + public void testBindsTheCompletePrivateSyntaxAstContract() + { + HogQlSyntaxAstManifest manifest = HogQlSyntaxAstManifest.current(); + + assertThat(manifest.schemaVersion()).isEqualTo(1); + assertThat(manifest.languageVersion()).isEqualTo(HogQlLanguageContract.current().languageVersion()); + assertThat(manifest.grammarSha256()).isEqualTo(HogQlLanguageContract.current().grammarSha256()); + assertThat(manifest.grammarFeatureManifestSha256()).isEqualTo(sha256(readResourceBytes(GRAMMAR_FEATURE_RESOURCE))); + assertThat(manifest.grammarAlternativeIdentityManifestSha256()).isEqualTo(sha256(readResourceBytes(ALTERNATIVE_IDENTITIES_RESOURCE))); + assertThat(manifest.syntaxTreeKind()).isEqualTo(SyntaxNodeKind.TREE); + assertThat(manifest.sourceSpanGuarantee()).isEqualTo(SourceSpanGuarantee.CODE_POINT_OFFSETS_END_EXCLUSIVE_ONE_BASED_LINE_COLUMNS); + assertThat(manifest.features()).hasSize(583); + assertThat(manifest.features()).extracting(Feature::id).doesNotHaveDuplicates(); + assertThat(manifest.features()).filteredOn(feature -> feature.syntaxNodeKind() == SyntaxNodeKind.TOKEN).hasSize(207); + assertThat(manifest.features()).filteredOn(feature -> feature.syntaxNodeKind() == SyntaxNodeKind.RULE).hasSize(376); + assertThat(manifest.features()) + .allSatisfy(feature -> assertThat(feature.syntaxNodeKind() == SyntaxNodeKind.TOKEN) + .isEqualTo(feature.id().startsWith("token:"))); + } + + @Test + public void testRejectsIncompleteOrInvalidFeatureCoverage() + { + PublishedGrammarFeatureManifest published = loadPublishedGrammarFeatures(); + HogQlGrammarAlternativeIdentities alternativeIdentities = HogQlGrammarAlternativeIdentities.current(); + List current = HogQlSyntaxAstManifest.current().features(); + + assertThatThrownBy(() -> HogQlSyntaxAstManifest.validateFeatureCoverage(current.subList(1, current.size()), published, alternativeIdentities)) + .isInstanceOf(IllegalStateException.class) + .hasMessage("HogQL syntax AST manifest does not account for every grammar feature"); + + List duplicate = new ArrayList<>(current); + duplicate.add(current.getFirst()); + assertThatThrownBy(() -> HogQlSyntaxAstManifest.validateFeatureCoverage(duplicate, published, alternativeIdentities)) + .isInstanceOf(IllegalStateException.class) + .hasMessage("duplicate HogQL syntax AST feature: " + current.getFirst().id()); + + List unknown = new ArrayList<>(current); + unknown.add(new Feature("token:NOT_A_CANONICAL_HOGQL_TOKEN", SyntaxNodeKind.TOKEN)); + assertThatThrownBy(() -> HogQlSyntaxAstManifest.validateFeatureCoverage(unknown, published, alternativeIdentities)) + .isInstanceOf(IllegalStateException.class) + .hasMessage("unknown HogQL syntax AST feature: token:NOT_A_CANONICAL_HOGQL_TOKEN"); + + List wrongNodeKind = new ArrayList<>(current); + Feature original = wrongNodeKind.getFirst(); + SyntaxNodeKind replacementKind = original.syntaxNodeKind() == SyntaxNodeKind.TOKEN ? SyntaxNodeKind.RULE : SyntaxNodeKind.TOKEN; + wrongNodeKind.set(0, new Feature(original.id(), replacementKind)); + assertThatThrownBy(() -> HogQlSyntaxAstManifest.validateFeatureCoverage(wrongNodeKind, published, alternativeIdentities)) + .isInstanceOf(IllegalStateException.class) + .hasMessage("HogQL syntax AST feature has the wrong node kind: " + original.id()); + } + + private static PublishedGrammarFeatureManifest loadPublishedGrammarFeatures() + { + return GRAMMAR_FEATURE_MANIFEST_CODEC.fromJson(new String(readResourceBytes(GRAMMAR_FEATURE_RESOURCE), StandardCharsets.UTF_8)); + } + + private static byte[] readResourceBytes(String path) + { + try (InputStream input = TestHogQlSyntaxAstManifest.class.getResourceAsStream(path)) { + if (input == null) { + throw new IllegalStateException("missing test resource: " + path); + } + return input.readAllBytes(); + } + catch (IOException e) { + throw new UncheckedIOException("failed to read test resource: " + path, e); + } + } + + private static String sha256(byte[] content) + { + try { + return HexFormat.of().formatHex(MessageDigest.getInstance("SHA-256").digest(content)); + } + catch (NoSuchAlgorithmException e) { + throw new AssertionError(e); + } + } +} diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlTemplateStringParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlTemplateStringParser.java new file mode 100644 index 000000000000..d6921dac1d47 --- /dev/null +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlTemplateStringParser.java @@ -0,0 +1,61 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; +import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; +import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TestHogQlTemplateStringParser +{ + private final HogQlParser parser = new HogQlParser(); + + @Test + public void testDesugarsInterpolatedTemplateToTypedConcatenation() + { + HogQlQuery query = parser.parseStatement("SELECT f'{year}-{month}-{day}'"); + FunctionCall concat = (FunctionCall) ((ExpressionProjection) query.projections().getFirst()).expression(); + + assertThat(concat.name().value()).isEqualTo("concat"); + assertThat(concat.arguments()).hasSize(5); + assertThat(concat.arguments().get(0)).isInstanceOfSatisfying( + FunctionCall.class, + function -> assertThat(function.name().value()).isEqualTo("toString")); + assertThat(concat.arguments().get(1)).isInstanceOfSatisfying( + Literal.class, + literal -> assertThat(literal.value()).isEqualTo("-")); + } + + @Test + public void testKeepsLiteralOnlyTemplateAsLiteral() + { + HogQlQuery query = parser.parseStatement("SELECT f'hello\\nworld'"); + Literal literal = (Literal) ((ExpressionProjection) query.projections().getFirst()).expression(); + + assertThat(literal.value()).isEqualTo("hello\nworld"); + } + + @Test + public void testKeepsEmptyTemplateAsEmptyLiteral() + { + HogQlQuery query = parser.parseStatement("SELECT f''"); + Literal literal = (Literal) ((ExpressionProjection) query.projections().getFirst()).expression(); + + assertThat(literal.value()).isEmpty(); + } +} diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlWindowParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlWindowParser.java new file mode 100644 index 000000000000..72d81f64ef18 --- /dev/null +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlWindowParser.java @@ -0,0 +1,118 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; +import io.trino.hogql.parser.tree.HogQlQuery.FrameBoundType; +import io.trino.hogql.parser.tree.HogQlQuery.FrameType; +import io.trino.hogql.parser.tree.HogQlQuery.FunctionCall; +import io.trino.hogql.parser.tree.HogQlQuery.NullTreatment; +import io.trino.hogql.parser.tree.HogQlQuery.Placeholder; +import io.trino.hogql.parser.tree.HogQlQuery.SortDirection; +import io.trino.hogql.parser.tree.HogQlQuery.WindowReference; +import io.trino.hogql.parser.tree.HogQlQuery.WindowSpecification; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlWindowParser +{ + private final HogQlParser parser = new HogQlParser(); + + @Test + public void testBuildsInlineWindowSpecificationAndFrame() + { + String hogql = "SELECT sum({value}) FILTER (WHERE {filter}) OVER " + + "(PARTITION BY {partition}, team_id ORDER BY {sort} DESC NULLS LAST " + + "ROWS BETWEEN {preceding} PRECEDING AND CURRENT ROW)"; + + HogQlQuery query = parser.parseStatement(hogql); + FunctionCall function = (FunctionCall) ((ExpressionProjection) query.projections().getFirst()).expression(); + + assertThat(function.name().value()).isEqualTo("sum"); + assertThat(function.arguments()).singleElement().isInstanceOf(Placeholder.class); + assertThat(function.filter()).get().isInstanceOf(Placeholder.class); + WindowSpecification window = (WindowSpecification) function.window().orElseThrow(); + assertThat(window.partitionBy()).hasSize(2); + assertThat(window.orderBy()).singleElement().satisfies(sortItem -> { + assertThat(sortItem.expression()).isInstanceOf(Placeholder.class); + assertThat(sortItem.direction()).isEqualTo(SortDirection.DESCENDING); + }); + assertThat(window.span().startOffset()).isEqualTo(hogql.indexOf("PARTITION")); + assertThat(window.span().endOffset()).isEqualTo(hogql.length() - 1); + assertThat(window.frame()).get().satisfies(frame -> { + assertThat(frame.type()).isEqualTo(FrameType.ROWS); + assertThat(frame.start().type()).isEqualTo(FrameBoundType.PRECEDING); + assertThat(frame.start().value()).get().isInstanceOf(Placeholder.class); + assertThat(frame.end()).get().extracting(HogQlQuery.FrameBound::type).isEqualTo(FrameBoundType.CURRENT_ROW); + }); + } + + @Test + public void testBuildsNamedWindowReferenceAndDefinitions() + { + HogQlQuery query = parser.parseStatement( + "SELECT row_number() OVER recent " + + "WINDOW recent AS (PARTITION BY team_id ORDER BY timestamp), " + + "trailing AS (ORDER BY timestamp RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING)"); + + FunctionCall function = (FunctionCall) ((ExpressionProjection) query.projections().getFirst()).expression(); + assertThat(function.window()).get().isInstanceOf(WindowReference.class); + WindowReference reference = (WindowReference) function.window().orElseThrow(); + assertThat(reference.name().value()).isEqualTo("recent"); + assertThat(query.windows()).hasSize(2); + assertThat(query.windows()).extracting(definition -> definition.name().value()) + .containsExactly("recent", "trailing"); + assertThat(query.windows().getFirst().specification().partitionBy()).hasSize(1); + assertThat(query.windows().get(1).specification().frame()).get().satisfies(frame -> { + assertThat(frame.type()).isEqualTo(FrameType.RANGE); + assertThat(frame.start().type()).isEqualTo(FrameBoundType.CURRENT_ROW); + assertThat(frame.end()).get().extracting(HogQlQuery.FrameBound::type).isEqualTo(FrameBoundType.UNBOUNDED_FOLLOWING); + }); + } + + @Test + public void testBuildsCanonicalWindowNullTreatment() + { + HogQlQuery query = parser.parseStatement("SELECT first_value(value) OVER (ORDER BY timestamp) IGNORE NULLS"); + + FunctionCall function = (FunctionCall) ((ExpressionProjection) query.projections().getFirst()).expression(); + assertThat(function.nullTreatment()).contains(NullTreatment.IGNORE); + assertThat(function.window()).get().isInstanceOf(WindowSpecification.class); + } + + @ParameterizedTest + @ValueSource(strings = { + "SELECT quantile(0.9)(value) OVER ()", + "SELECT count()(DISTINCT value) OVER named WINDOW named AS ()", + }) + public void testRejectsParametricWindowFunctions(String hogql) + { + assertThatThrownBy(() -> parser.parseStatement(hogql)) + .isInstanceOf(HogQlParsingException.class) + .hasMessageContaining("parametric window function"); + } + + @Test + public void testRejectsNullTreatmentOutsideWindowFunction() + { + assertThatThrownBy(() -> parser.parseStatement("SELECT first_value(value) IGNORE NULLS")) + .isInstanceOf(HogQlParsingException.class) + .hasMessageContaining("IGNORE NULLS outside window function"); + } +} diff --git a/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlXParser.java b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlXParser.java new file mode 100644 index 000000000000..c57032d29393 --- /dev/null +++ b/core/trino-hogql-parser/src/test/java/io/trino/hogql/parser/TestHogQlXParser.java @@ -0,0 +1,53 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql.parser; + +import io.trino.hogql.parser.tree.HogQlQuery; +import io.trino.hogql.parser.tree.HogQlQuery.ExpressionProjection; +import io.trino.hogql.parser.tree.HogQlQuery.Literal; +import io.trino.hogql.parser.tree.HogQlQuery.TupleExpression; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlXParser +{ + private final HogQlParser parser = new HogQlParser(); + + @Test + public void testLowersNestedTagToHxTuple() + { + HogQlQuery query = parser.parseStatement( + "SELECT {event}Bold! FROM events"); + TupleExpression tag = (TupleExpression) ((ExpressionProjection) query.projections().getFirst()).expression(); + + assertThat(tag.values()).hasSize(8); + assertThat(tag.values().get(0)).isEqualTo(new Literal(HogQlQuery.LiteralKind.STRING, "__hx_tag", tag.values().get(0).span())); + assertThat(tag.values().get(1)).isInstanceOfSatisfying( + Literal.class, + literal -> assertThat(literal.value()).isEqualTo("a")); + assertThat(tag.values().get(7)).isInstanceOfSatisfying( + TupleExpression.class, + children -> assertThat(children.values()).hasSize(2)); + } + + @Test + public void testRejectsMismatchedClosingTag() + { + assertThatThrownBy(() -> parser.parseStatement("SELECT text")) + .isInstanceOf(HogQlParsingException.class) + .hasMessageContaining("mismatched HogQLX closing tag"); + } +} diff --git a/core/trino-hogql-parser/src/test/resources/io/trino/hogql/parser/language/1.0.0/robustness/malformed-regressions.jsonl b/core/trino-hogql-parser/src/test/resources/io/trino/hogql/parser/language/1.0.0/robustness/malformed-regressions.jsonl new file mode 100644 index 000000000000..c5c7814622a9 --- /dev/null +++ b/core/trino-hogql-parser/src/test/resources/io/trino/hogql/parser/language/1.0.0/robustness/malformed-regressions.jsonl @@ -0,0 +1,8 @@ +{"id":"missing-projection","source":"SELECT"} +{"id":"trailing-binary-operator","source":"SELECT 1 +"} +{"id":"unclosed-tuple","source":"SELECT (1, 2"} +{"id":"unexpected-character","source":"SELECT value @ 1"} +{"id":"missing-source","source":"SELECT 1 FROM"} +{"id":"missing-predicate","source":"SELECT 1 WHERE"} +{"id":"unclosed-string","source":"SELECT 'example"} +{"id":"second-statement","source":"SELECT 1; SELECT 2"} diff --git a/core/trino-main/pom.xml b/core/trino-main/pom.xml index eae766945833..57aee720c48f 100644 --- a/core/trino-main/pom.xml +++ b/core/trino-main/pom.xml @@ -245,6 +245,16 @@ trino-geospatial-toolkit + + io.trino + trino-hogql-compiler + + + + io.trino + trino-hogql-parser + + io.trino trino-matching diff --git a/core/trino-main/src/main/java/io/trino/connector/CatalogLifecycleListener.java b/core/trino-main/src/main/java/io/trino/connector/CatalogLifecycleListener.java new file mode 100644 index 000000000000..24c20a3ae608 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/connector/CatalogLifecycleListener.java @@ -0,0 +1,25 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.connector; + +import io.trino.spi.catalog.CatalogName; + +@FunctionalInterface +public interface CatalogLifecycleListener +{ + /** + * Implementations run inline with catalog registration and must return without waiting for external work. + */ + void catalogLoaded(CatalogName catalogName); +} diff --git a/core/trino-main/src/main/java/io/trino/connector/CatalogLifecycleListeners.java b/core/trino-main/src/main/java/io/trino/connector/CatalogLifecycleListeners.java new file mode 100644 index 000000000000..c32526836196 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/connector/CatalogLifecycleListeners.java @@ -0,0 +1,48 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.connector; + +import com.google.inject.Inject; +import io.airlift.log.Logger; +import io.trino.spi.catalog.CatalogName; + +import java.util.Set; + +import static java.util.Objects.requireNonNull; + +public final class CatalogLifecycleListeners +{ + private static final Logger log = Logger.get(CatalogLifecycleListeners.class); + + private final Set listeners; + + @Inject + public CatalogLifecycleListeners(Set listeners) + { + this.listeners = Set.copyOf(requireNonNull(listeners, "listeners is null")); + } + + public void catalogLoaded(CatalogName catalogName) + { + requireNonNull(catalogName, "catalogName is null"); + for (CatalogLifecycleListener listener : listeners) { + try { + listener.catalogLoaded(catalogName); + } + catch (RuntimeException failure) { + log.warn(failure, "Catalog lifecycle listener failed for %s", catalogName); + } + } + } +} diff --git a/core/trino-main/src/main/java/io/trino/connector/CatalogManagerModule.java b/core/trino-main/src/main/java/io/trino/connector/CatalogManagerModule.java index d792824432b2..421be6f98923 100644 --- a/core/trino-main/src/main/java/io/trino/connector/CatalogManagerModule.java +++ b/core/trino-main/src/main/java/io/trino/connector/CatalogManagerModule.java @@ -17,6 +17,7 @@ import com.google.inject.Scopes; import io.airlift.configuration.AbstractConfigurationAwareModule; +import static com.google.inject.multibindings.Multibinder.newSetBinder; import static com.google.inject.multibindings.OptionalBinder.newOptionalBinder; public class CatalogManagerModule @@ -28,6 +29,8 @@ protected void setup(Binder binder) binder.bind(DefaultCatalogFactory.class).in(Scopes.SINGLETON); binder.bind(LazyCatalogFactory.class).in(Scopes.SINGLETON); binder.bind(CatalogFactory.class).to(LazyCatalogFactory.class).in(Scopes.SINGLETON); + binder.bind(CatalogLifecycleListeners.class).in(Scopes.SINGLETON); + newSetBinder(binder, CatalogLifecycleListener.class); newOptionalBinder(binder, CatalogStoreManager.class); CatalogManagerConfig config = buildConfigObject(CatalogManagerConfig.class); diff --git a/core/trino-main/src/main/java/io/trino/connector/CoordinatorDynamicCatalogManager.java b/core/trino-main/src/main/java/io/trino/connector/CoordinatorDynamicCatalogManager.java index 89f06274c8dc..5f297d455199 100644 --- a/core/trino-main/src/main/java/io/trino/connector/CoordinatorDynamicCatalogManager.java +++ b/core/trino-main/src/main/java/io/trino/connector/CoordinatorDynamicCatalogManager.java @@ -43,6 +43,7 @@ import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.Executor; import static com.google.common.base.Preconditions.checkArgument; @@ -74,6 +75,7 @@ private enum State private final CatalogStore catalogStore; private final CatalogFactory catalogFactory; private final CacheManagerRegistry cacheManagerRegistry; + private final CatalogLifecycleListeners catalogLifecycleListeners; private final Executor executor; private final Object catalogsUpdateLock = new Object(); @@ -92,12 +94,13 @@ private enum State private State state = State.CREATED; @Inject - public CoordinatorDynamicCatalogManager(CatalogStore catalogStore, CatalogFactory catalogFactory, CacheManagerRegistry cacheManagerRegistry, @ForStartup Executor executor) + public CoordinatorDynamicCatalogManager(CatalogStore catalogStore, CatalogFactory catalogFactory, CacheManagerRegistry cacheManagerRegistry, @ForStartup Executor executor, CatalogLifecycleListeners catalogLifecycleListeners) { this.catalogStore = requireNonNull(catalogStore, "catalogStore is null"); this.catalogFactory = requireNonNull(catalogFactory, "catalogFactory is null"); this.cacheManagerRegistry = requireNonNull(cacheManagerRegistry, "cacheManagerRegistry is null"); this.executor = requireNonNull(executor, "executor is null"); + this.catalogLifecycleListeners = requireNonNull(catalogLifecycleListeners, "catalogLifecycleListeners is null"); } @PreDestroy @@ -124,6 +127,7 @@ public void stop() @Override public void loadInitialCatalogs() { + List loadedCatalogs = new CopyOnWriteArrayList<>(); synchronized (catalogsUpdateLock) { if (state == State.INITIALIZED) { return; @@ -142,6 +146,7 @@ public void loadInitialCatalogs() CatalogConnector newCatalog = catalogFactory.createCatalog(catalog); activeCatalogs.put(storedCatalog.name(), newCatalog.getCatalog()); allCatalogs.put(newCatalog.getCatalogHandle(), new RegisteredCatalog(new RegistrationToken(), newCatalog)); + loadedCatalogs.add(storedCatalog.name()); log.debug("-- Added catalog %s using connector %s --", storedCatalog.name(), catalog.connectorName()); } catch (Throwable e) { @@ -154,6 +159,7 @@ public void loadInitialCatalogs() }) .collect(toImmutableList())); } + loadedCatalogs.forEach(catalogLifecycleListeners::catalogLoaded); } @Override @@ -285,6 +291,7 @@ public void createCatalog(CatalogName catalogName, ConnectorName connectorName, log.debug("Added catalog: %s", catalog.getCatalogHandle()); } + catalogLifecycleListeners.catalogLoaded(catalogName); } public void registerGlobalSystemConnector(GlobalSystemConnector connector) diff --git a/core/trino-main/src/main/java/io/trino/connector/StaticCatalogManager.java b/core/trino-main/src/main/java/io/trino/connector/StaticCatalogManager.java index f3d54ff9a5a1..f56e826e76e2 100644 --- a/core/trino-main/src/main/java/io/trino/connector/StaticCatalogManager.java +++ b/core/trino-main/src/main/java/io/trino/connector/StaticCatalogManager.java @@ -69,6 +69,7 @@ private enum State } private final CatalogFactory catalogFactory; + private final CatalogLifecycleListeners catalogLifecycleListeners; private final List catalogProperties; private final Executor executor; @@ -77,9 +78,10 @@ private enum State private final AtomicReference state = new AtomicReference<>(State.CREATED); @Inject - public StaticCatalogManager(CatalogFactory catalogFactory, StaticCatalogManagerConfig config, @ForStartup Executor executor) + public StaticCatalogManager(CatalogFactory catalogFactory, StaticCatalogManagerConfig config, @ForStartup Executor executor, CatalogLifecycleListeners catalogLifecycleListeners) { this.catalogFactory = requireNonNull(catalogFactory, "catalogFactory is null"); + this.catalogLifecycleListeners = requireNonNull(catalogLifecycleListeners, "catalogLifecycleListeners is null"); List disabledCatalogs = requireNonNullElse(config.getDisabledCatalogs(), ImmutableList.of()); ImmutableList.Builder catalogProperties = ImmutableList.builder(); @@ -161,6 +163,7 @@ public void loadInitialCatalogs() log.info("-- Loading catalog %s --", catalogName); CatalogConnector newCatalog = catalogFactory.createCatalog(catalog); catalogs.put(catalogName, newCatalog); + catalogLifecycleListeners.catalogLoaded(catalogName); log.info("-- Added catalog %s using connector %s --", catalogName, catalog.connectorName()); return null; }) diff --git a/core/trino-main/src/main/java/io/trino/dispatcher/DispatchManager.java b/core/trino-main/src/main/java/io/trino/dispatcher/DispatchManager.java index 08de0825365e..be3ee7f938ac 100644 --- a/core/trino-main/src/main/java/io/trino/dispatcher/DispatchManager.java +++ b/core/trino-main/src/main/java/io/trino/dispatcher/DispatchManager.java @@ -31,6 +31,7 @@ import io.trino.execution.QueryManagerStats; import io.trino.execution.QueryPreparer; import io.trino.execution.QueryPreparer.PreparedQuery; +import io.trino.execution.QuerySubmission; import io.trino.execution.QueryTracker; import io.trino.execution.resourcegroups.ResourceGroupManager; import io.trino.metadata.SessionPropertyManager; @@ -63,6 +64,7 @@ import static io.trino.execution.QueryState.QUEUED; import static io.trino.execution.QueryState.RUNNING; import static io.trino.execution.QueryState.WAITING_FOR_RESOURCES; +import static io.trino.execution.QuerySubmission.trino; import static io.trino.spi.StandardErrorCode.QUERY_TEXT_TOO_LARGE; import static io.trino.tracing.ScopedSpan.scopedSpan; import static io.trino.util.Failures.toFailure; @@ -174,12 +176,17 @@ public QueryId createQueryId() } public ListenableFuture createQuery(QueryId queryId, Span querySpan, Slug slug, SessionContext sessionContext, String query) + { + return createQuery(queryId, querySpan, slug, sessionContext, trino(query)); + } + + public ListenableFuture createQuery(QueryId queryId, Span querySpan, Slug slug, SessionContext sessionContext, QuerySubmission submission) { requireNonNull(queryId, "queryId is null"); requireNonNull(querySpan, "querySpan is null"); requireNonNull(sessionContext, "sessionContext is null"); - requireNonNull(query, "query is null"); - checkArgument(!query.isEmpty(), "query must not be empty string"); + requireNonNull(submission, "submission is null"); + checkArgument(!submission.originalText().isEmpty(), "query must not be empty string"); checkArgument(!queryTracker.hasQuery(queryId), "query %s already exists", queryId); // It is important to return a future implementation which ignores cancellation request. @@ -192,7 +199,7 @@ public ListenableFuture createQuery(QueryId queryId, Span querySpan, Slug .setParent(Context.current().with(querySpan)) .startSpan(); try (var _ = scopedSpan(span)) { - createQueryInternal(queryId, querySpan, slug, sessionContext, query, resourceGroupManager); + createQueryInternal(queryId, querySpan, slug, sessionContext, submission, resourceGroupManager); } finally { queryCreationFuture.set(null); @@ -205,8 +212,9 @@ public ListenableFuture createQuery(QueryId queryId, Span querySpan, Slug * Creates and registers a dispatch query with the query tracker. This method will never fail to register a query with the query * tracker. If an error occurs while creating a dispatch query, a failed dispatch will be created and registered. */ - private void createQueryInternal(QueryId queryId, Span querySpan, Slug slug, SessionContext sessionContext, String query, ResourceGroupManager resourceGroupManager) + private void createQueryInternal(QueryId queryId, Span querySpan, Slug slug, SessionContext sessionContext, QuerySubmission submission, ResourceGroupManager resourceGroupManager) { + String query = submission.originalText(); Session session = null; PreparedQuery preparedQuery = null; try { @@ -223,7 +231,7 @@ private void createQueryInternal(QueryId queryId, Span querySpan, Slug slug, accessControl.checkCanExecuteQuery(sessionContext.getIdentity(), queryId); // prepare query - preparedQuery = queryPreparer.prepareQuery(session, query); + preparedQuery = queryPreparer.prepareQuery(session, submission); // select resource group Optional queryType = getQueryType(preparedQuery.getStatement()).map(Enum::name); diff --git a/core/trino-main/src/main/java/io/trino/dispatcher/HogQlPhysicalCatalogResource.java b/core/trino-main/src/main/java/io/trino/dispatcher/HogQlPhysicalCatalogResource.java new file mode 100644 index 000000000000..03b3e21e40eb --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/dispatcher/HogQlPhysicalCatalogResource.java @@ -0,0 +1,99 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.dispatcher; + +import com.google.inject.Inject; +import io.opentelemetry.api.trace.Span; +import io.trino.Session; +import io.trino.execution.QueryIdGenerator; +import io.trino.hogql.HogQlPhysicalCatalog; +import io.trino.hogql.HogQlPhysicalCatalogProvider; +import io.trino.server.HttpRequestSessionContextFactory; +import io.trino.server.SessionContext; +import io.trino.server.SessionSupplier; +import io.trino.server.security.InternalPrincipal; +import io.trino.server.security.ResourceSecurity; +import io.trino.spi.security.Identity; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.ws.rs.BadRequestException; +import jakarta.ws.rs.ForbiddenException; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.QueryParam; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.HttpHeaders; + +import java.util.Optional; + +import static io.trino.server.ServletSecurityUtils.authenticatedIdentity; +import static io.trino.server.security.ResourceSecurity.AccessType.AUTHENTICATED_USER; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; +import static java.util.Locale.ENGLISH; +import static java.util.Objects.requireNonNull; + +@Path("/v1/hogql/compatibility/physical-catalog") +public class HogQlPhysicalCatalogResource +{ + private final HttpRequestSessionContextFactory sessionContextFactory; + private final SessionSupplier sessionSupplier; + private final QueryIdGenerator queryIdGenerator; + private final HogQlPhysicalCatalogProvider physicalCatalogProvider; + + @Inject + public HogQlPhysicalCatalogResource( + HttpRequestSessionContextFactory sessionContextFactory, + SessionSupplier sessionSupplier, + QueryIdGenerator queryIdGenerator, + HogQlPhysicalCatalogProvider physicalCatalogProvider) + { + this.sessionContextFactory = requireNonNull(sessionContextFactory, "sessionContextFactory is null"); + this.sessionSupplier = requireNonNull(sessionSupplier, "sessionSupplier is null"); + this.queryIdGenerator = requireNonNull(queryIdGenerator, "queryIdGenerator is null"); + this.physicalCatalogProvider = requireNonNull(physicalCatalogProvider, "physicalCatalogProvider is null"); + } + + @ResourceSecurity(AUTHENTICATED_USER) + @GET + @Produces(APPLICATION_JSON) + public HogQlPhysicalCatalog getPhysicalCatalog( + @QueryParam("catalog") String catalog, + @QueryParam("protocolVersion") Integer protocolVersion, + @Context HttpServletRequest servletRequest, + @Context HttpHeaders httpHeaders) + { + if (catalog == null || catalog.isBlank()) { + throw new BadRequestException("Catalog is required"); + } + if (protocolVersion == null || protocolVersion != HogQlPhysicalCatalog.PROTOCOL_VERSION) { + throw new BadRequestException("Unsupported physical catalog protocol version"); + } + catalog = catalog.toLowerCase(ENGLISH); + + Optional identity = authenticatedIdentity(servletRequest); + if (identity.flatMap(Identity::getPrincipal).map(InternalPrincipal.class::isInstance).orElse(false)) { + throw new ForbiddenException("Internal communication can not be used to read a physical catalog"); + } + + SessionContext sessionContext = sessionContextFactory.createSessionContext( + httpHeaders.getRequestHeaders(), + Optional.ofNullable(servletRequest.getRemoteAddr()), + identity); + if (sessionContext.getTransactionId().isPresent()) { + throw new BadRequestException("A transaction ID is not supported for physical catalog requests"); + } + Session session = sessionSupplier.createSession(queryIdGenerator.createNextQueryId(), Span.getInvalid(), sessionContext); + return physicalCatalogProvider.load(session, catalog); + } +} diff --git a/core/trino-main/src/main/java/io/trino/dispatcher/HogQlRequest.java b/core/trino-main/src/main/java/io/trino/dispatcher/HogQlRequest.java new file mode 100644 index 000000000000..96af362b02ab --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/dispatcher/HogQlRequest.java @@ -0,0 +1,263 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.dispatcher; + +import com.fasterxml.jackson.databind.JsonNode; +import com.google.common.base.Utf8; +import io.airlift.json.JsonCodec; +import io.trino.execution.QuerySubmission.HogQlExplain; +import io.trino.hogql.compiler.HogQlCompileEnvelope; +import io.trino.hogql.compiler.HogQlTypedValue; +import io.trino.hogql.compiler.HogQlTypedValue.ArrayValue; +import io.trino.hogql.compiler.HogQlTypedValue.BooleanValue; +import io.trino.hogql.compiler.HogQlTypedValue.NullValue; +import io.trino.hogql.compiler.HogQlTypedValue.NumberValue; +import io.trino.hogql.compiler.HogQlTypedValue.ObjectValue; +import io.trino.hogql.compiler.HogQlTypedValue.StringValue; +import io.trino.hogql.compiler.HogQlTypedValue.Value; +import io.trino.hogql.parser.HogQlLanguageVersion; +import io.trino.sql.tree.ExplainFormat; +import io.trino.sql.tree.ExplainType; + +import java.util.ArrayList; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.Set; + +import static io.airlift.json.JsonCodec.jsonCodec; +import static java.util.Objects.requireNonNull; + +record HogQlRequest( + String query, + int protocolVersion, + HogQlLanguageVersion languageVersion, + Map parameters, + Map variables, + Map filters, + Map modifiers, + OptionalLong catalogGeneration, + Optional explain) +{ + private static final JsonCodec JSON_CODEC = jsonCodec(JsonNode.class); + private static final Set FIELDS = Set.of( + "query", + "protocolVersion", + "languageVersion", + "parameters", + "variables", + "filters", + "modifiers", + "catalogGeneration", + "explain"); + private static final Set TYPED_VALUE_FIELDS = Set.of("type", "value"); + private static final Set EXPLAIN_FIELDS = Set.of("type", "format"); + private static final int MAX_VALUE_DEPTH = 64; + private static final int MAX_REQUEST_BYTES = 2 * 1024 * 1024; + private static final int MAX_BINDINGS_PER_FIELD = 1_000; + private static final int MAX_TOTAL_BINDINGS = 1_000; + + HogQlRequest + { + query = requireNonNull(query, "query is null"); + languageVersion = requireNonNull(languageVersion, "languageVersion is null"); + parameters = Map.copyOf(requireNonNull(parameters, "parameters is null")); + variables = Map.copyOf(requireNonNull(variables, "variables is null")); + filters = Map.copyOf(requireNonNull(filters, "filters is null")); + modifiers = Map.copyOf(requireNonNull(modifiers, "modifiers is null")); + catalogGeneration = requireNonNull(catalogGeneration, "catalogGeneration is null"); + explain = requireNonNull(explain, "explain is null"); + if (parameters.size() + variables.size() + filters.size() + modifiers.size() > MAX_TOTAL_BINDINGS) { + throw new IllegalArgumentException("HogQL request has too many bindings"); + } + } + + static HogQlRequest fromJson(String requestBody) + { + requestBody = requireNonNull(requestBody, "requestBody is null"); + if (Utf8.encodedLength(requestBody) > MAX_REQUEST_BYTES) { + throw new IllegalArgumentException("HogQL request is too large"); + } + JsonNode root = JSON_CODEC.fromJson(requestBody); + if (root == null || !root.isObject()) { + throw new IllegalArgumentException("HogQL request must be an object"); + } + rejectUnknownFields(root, FIELDS, "unknown HogQL request field"); + + return new HogQlRequest( + requiredText(root, "query"), + requiredInt(root, "protocolVersion"), + languageVersion(root), + typedValues(root, "parameters"), + typedValues(root, "variables"), + typedValues(root, "filters"), + typedValues(root, "modifiers"), + catalogGeneration(root), + explain(root)); + } + + HogQlCompileEnvelope toCompileEnvelope() + { + return new HogQlCompileEnvelope( + query, + protocolVersion, + languageVersion, + parameters, + variables, + filters, + modifiers, + catalogGeneration); + } + + private static HogQlLanguageVersion languageVersion(JsonNode root) + { + try { + return HogQlLanguageVersion.valueOf(requiredText(root, "languageVersion")); + } + catch (RuntimeException _) { + throw new IllegalArgumentException("invalid HogQL language version"); + } + } + + private static OptionalLong catalogGeneration(JsonNode root) + { + JsonNode value = root.get("catalogGeneration"); + if (value == null) { + return OptionalLong.empty(); + } + if (!value.isIntegralNumber() || !value.canConvertToLong() || value.longValue() <= 0) { + throw new IllegalArgumentException("invalid HogQL catalog generation"); + } + return OptionalLong.of(value.longValue()); + } + + private static Optional explain(JsonNode root) + { + JsonNode value = root.get("explain"); + if (value == null) { + return Optional.empty(); + } + if (!value.isObject()) { + throw new IllegalArgumentException("invalid HogQL explain request"); + } + rejectUnknownFields(value, EXPLAIN_FIELDS, "unknown HogQL explain field"); + try { + return Optional.of(new HogQlExplain( + ExplainType.Type.valueOf(requiredText(value, "type")), + ExplainFormat.Type.valueOf(requiredText(value, "format")))); + } + catch (RuntimeException _) { + throw new IllegalArgumentException("invalid HogQL explain request"); + } + } + + private static Map typedValues(JsonNode root, String field) + { + JsonNode values = root.get(field); + if (values == null) { + return Map.of(); + } + if (!values.isObject()) { + throw new IllegalArgumentException("invalid HogQL semantic field"); + } + if (values.size() > MAX_BINDINGS_PER_FIELD) { + throw new IllegalArgumentException("HogQL semantic field has too many bindings"); + } + + Map result = new LinkedHashMap<>(); + Iterator names = values.fieldNames(); + while (names.hasNext()) { + String name = names.next(); + result.put(name, typedValue(values.get(name))); + } + return Map.copyOf(result); + } + + private static HogQlTypedValue typedValue(JsonNode node) + { + if (node == null || !node.isObject()) { + throw new IllegalArgumentException("invalid HogQL typed value"); + } + rejectUnknownFields(node, TYPED_VALUE_FIELDS, "unknown HogQL typed value field"); + if (!node.has("value")) { + throw new IllegalArgumentException("missing HogQL typed value"); + } + return new HogQlTypedValue(requiredText(node, "type"), value(node.get("value"), 0)); + } + + private static Value value(JsonNode node, int depth) + { + if (depth > MAX_VALUE_DEPTH) { + throw new IllegalArgumentException("HogQL typed value is too deeply nested"); + } + if (node == null || node.isNull()) { + return NullValue.NULL; + } + if (node.isBoolean()) { + return new BooleanValue(node.booleanValue()); + } + if (node.isNumber()) { + return new NumberValue(node.asText()); + } + if (node.isTextual()) { + return new StringValue(node.textValue()); + } + if (node.isArray()) { + List values = new ArrayList<>(node.size()); + node.elements().forEachRemaining(element -> values.add(value(element, depth + 1))); + return new ArrayValue(values); + } + if (node.isObject()) { + Map values = new LinkedHashMap<>(); + Iterator names = node.fieldNames(); + while (names.hasNext()) { + String name = names.next(); + values.put(name, value(node.get(name), depth + 1)); + } + return new ObjectValue(values); + } + throw new IllegalArgumentException("invalid HogQL typed value payload"); + } + + private static String requiredText(JsonNode root, String field) + { + JsonNode value = root.get(field); + if (value == null || !value.isTextual()) { + throw new IllegalArgumentException("missing HogQL text field"); + } + return value.textValue(); + } + + private static int requiredInt(JsonNode root, String field) + { + JsonNode value = root.get(field); + if (value == null || !value.isIntegralNumber() || !value.canConvertToInt()) { + throw new IllegalArgumentException("missing HogQL integer field"); + } + return value.intValue(); + } + + private static void rejectUnknownFields(JsonNode object, Set fields, String message) + { + Iterator names = object.fieldNames(); + while (names.hasNext()) { + if (!fields.contains(names.next())) { + throw new IllegalArgumentException(message); + } + } + } +} diff --git a/core/trino-main/src/main/java/io/trino/dispatcher/HogQlStatementResource.java b/core/trino-main/src/main/java/io/trino/dispatcher/HogQlStatementResource.java new file mode 100644 index 000000000000..7c49ca6eed53 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/dispatcher/HogQlStatementResource.java @@ -0,0 +1,67 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.dispatcher; + +import com.google.inject.Inject; +import io.trino.execution.QuerySubmission; +import io.trino.server.ExternalUriInfo; +import io.trino.server.security.ResourceSecurity; +import jakarta.servlet.http.HttpServletRequest; +import jakarta.ws.rs.BadRequestException; +import jakarta.ws.rs.BeanParam; +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.HttpHeaders; +import jakarta.ws.rs.core.Response; + +import static io.trino.execution.QuerySubmission.hogQl; +import static io.trino.server.security.ResourceSecurity.AccessType.AUTHENTICATED_USER; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; +import static java.util.Objects.requireNonNull; + +@Path("/v1/hogql") +public class HogQlStatementResource +{ + private final QueuedStatementResource queuedStatementResource; + + @Inject + public HogQlStatementResource(QueuedStatementResource queuedStatementResource) + { + this.queuedStatementResource = requireNonNull(queuedStatementResource, "queuedStatementResource is null"); + } + + @ResourceSecurity(AUTHENTICATED_USER) + @POST + @Consumes(APPLICATION_JSON) + @Produces(APPLICATION_JSON) + public Response postStatement( + String requestBody, + @Context HttpServletRequest servletRequest, + @Context HttpHeaders httpHeaders, + @BeanParam ExternalUriInfo externalUriInfo) + { + QuerySubmission submission; + try { + HogQlRequest request = HogQlRequest.fromJson(requestBody); + submission = hogQl(request.toCompileEnvelope(), request.explain()); + } + catch (RuntimeException _) { + throw new BadRequestException("Invalid HogQL request"); + } + return queuedStatementResource.postStatement(submission, servletRequest, httpHeaders, externalUriInfo); + } +} diff --git a/core/trino-main/src/main/java/io/trino/dispatcher/QueuedStatementResource.java b/core/trino-main/src/main/java/io/trino/dispatcher/QueuedStatementResource.java index eff1bfadcead..975e78ff6ad6 100644 --- a/core/trino-main/src/main/java/io/trino/dispatcher/QueuedStatementResource.java +++ b/core/trino-main/src/main/java/io/trino/dispatcher/QueuedStatementResource.java @@ -31,6 +31,7 @@ import io.trino.execution.ExecutionFailureInfo; import io.trino.execution.QueryManagerConfig; import io.trino.execution.QueryState; +import io.trino.execution.QuerySubmission; import io.trino.server.ExternalUriInfo; import io.trino.server.GoneException; import io.trino.server.HttpRequestSessionContextFactory; @@ -89,8 +90,10 @@ import static io.trino.dispatcher.QueuedStatementResource.SubmissionState.ABANDONED; import static io.trino.dispatcher.QueuedStatementResource.SubmissionState.NOT_SUBMITTED; import static io.trino.dispatcher.QueuedStatementResource.SubmissionState.SUBMITTED; +import static io.trino.execution.QueryLanguage.TRINO; import static io.trino.execution.QueryState.FAILED; import static io.trino.execution.QueryState.QUEUED; +import static io.trino.execution.QuerySubmission.trino; import static io.trino.server.ServletSecurityUtils.authenticatedIdentity; import static io.trino.server.ServletSecurityUtils.clearAuthenticatedIdentity; import static io.trino.server.protocol.QueryInfoUrlFactory.getQueryInfoUri; @@ -175,15 +178,25 @@ public Response postStatement( @Context HttpHeaders httpHeaders, @BeanParam ExternalUriInfo externalUriInfo) { - if (isNullOrEmpty(statement)) { - throw new BadRequestException("SQL statement is empty"); + return postStatement(trino(statement), servletRequest, httpHeaders, externalUriInfo); + } + + Response postStatement( + QuerySubmission submission, + HttpServletRequest servletRequest, + HttpHeaders httpHeaders, + ExternalUriInfo externalUriInfo) + { + requireNonNull(submission, "submission is null"); + if (isNullOrEmpty(submission.originalText())) { + throw new BadRequestException(submission.language() == TRINO ? "SQL statement is empty" : "HogQL query is empty"); } - Query query = registerQuery(statement, servletRequest, httpHeaders); + Query query = registerQuery(submission, servletRequest, httpHeaders); return createQueryResultsResponse(query.getQueryResults(query.getLastToken(), externalUriInfo), query.sessionContext.getQueryDataEncoding()); } - private Query registerQuery(String statement, HttpServletRequest servletRequest, HttpHeaders httpHeaders) + private Query registerQuery(QuerySubmission submission, HttpServletRequest servletRequest, HttpHeaders httpHeaders) { Optional remoteAddress = Optional.ofNullable(servletRequest.getRemoteAddr()); Optional identity = authenticatedIdentity(servletRequest); @@ -194,7 +207,7 @@ private Query registerQuery(String statement, HttpServletRequest servletRequest, MultivaluedMap headers = httpHeaders.getRequestHeaders(); SessionContext sessionContext = sessionContextFactory.createSessionContext(headers, remoteAddress, identity); - Query query = new Query(statement, sessionContext, dispatchManager, queryInfoUrlFactory, tracer); + Query query = new Query(submission, sessionContext, dispatchManager, queryInfoUrlFactory, tracer); queryManager.registerQuery(query); // let authentication filter know that identity lifecycle has been handed off @@ -314,7 +327,7 @@ enum SubmissionState private static final class Query { - private final String query; + private final QuerySubmission submission; private final SessionContext sessionContext; private final DispatchManager dispatchManager; private final QueryId queryId; @@ -327,9 +340,9 @@ private static final class Query private final AtomicReference submissionGate = new AtomicReference<>(NOT_SUBMITTED); private final SettableFuture creationFuture = SettableFuture.create(); - public Query(String query, SessionContext sessionContext, DispatchManager dispatchManager, QueryInfoUrlFactory queryInfoUrlFactory, Tracer tracer) + public Query(QuerySubmission submission, SessionContext sessionContext, DispatchManager dispatchManager, QueryInfoUrlFactory queryInfoUrlFactory, Tracer tracer) { - this.query = requireNonNull(query, "query is null"); + this.submission = requireNonNull(submission, "submission is null"); this.sessionContext = requireNonNull(sessionContext, "sessionContext is null"); this.dispatchManager = requireNonNull(dispatchManager, "dispatchManager is null"); this.queryId = dispatchManager.createQueryId(); @@ -385,7 +398,7 @@ private void submitIfNeeded() { if (submissionGate.compareAndSet(NOT_SUBMITTED, SUBMITTED)) { querySpan.addEvent("submit"); - creationFuture.setFuture(dispatchManager.createQuery(queryId, querySpan, slug, sessionContext, query)); + creationFuture.setFuture(dispatchManager.createQuery(queryId, querySpan, slug, sessionContext, submission)); } } diff --git a/core/trino-main/src/main/java/io/trino/execution/HogQlCompilationExecutor.java b/core/trino-main/src/main/java/io/trino/execution/HogQlCompilationExecutor.java new file mode 100644 index 000000000000..6e3346c3634e --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/execution/HogQlCompilationExecutor.java @@ -0,0 +1,142 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.execution; + +import com.google.inject.Inject; +import io.trino.hogql.HogQlConfig; +import io.trino.spi.TrinoException; +import jakarta.annotation.PreDestroy; + +import java.util.Optional; +import java.util.OptionalLong; +import java.util.Set; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executor; +import java.util.concurrent.FutureTask; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeoutException; +import java.util.function.Supplier; + +import static io.airlift.concurrent.Threads.daemonThreadsNamed; +import static io.trino.hogql.HogQlCoordinatorErrorCode.HOGQL_COMPILATION_QUEUE_FULL; +import static io.trino.hogql.HogQlCoordinatorErrorCode.HOGQL_COMPILATION_TIMEOUT; +import static io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; +import static java.util.Objects.requireNonNull; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +public final class HogQlCompilationExecutor +{ + private final Executor executor; + private final Optional ownedExecutor; + private final OptionalLong timeoutMillis; + private final Set> tasks = ConcurrentHashMap.newKeySet(); + + @Inject + public HogQlCompilationExecutor(HogQlConfig config) + { + requireNonNull(config, "config is null"); + BlockingQueue queue = config.getCompilationQueueCapacity() == 0 + ? new SynchronousQueue<>() + : new ArrayBlockingQueue<>(config.getCompilationQueueCapacity()); + ThreadPoolExecutor executor = new ThreadPoolExecutor( + config.getCompilationThreads(), + config.getCompilationThreads(), + 0, + MILLISECONDS, + queue, + daemonThreadsNamed("hogql-compilation-%s"), + new ThreadPoolExecutor.AbortPolicy()); + this.executor = executor; + this.ownedExecutor = Optional.of(executor); + this.timeoutMillis = OptionalLong.of(config.getCompilationTimeout().toMillis()); + } + + private HogQlCompilationExecutor(Executor executor) + { + this.executor = requireNonNull(executor, "executor is null"); + ownedExecutor = Optional.empty(); + timeoutMillis = OptionalLong.empty(); + } + + static HogQlCompilationExecutor directExecutor() + { + return new HogQlCompilationExecutor(Runnable::run); + } + + public T execute(Supplier compilation) + { + requireNonNull(compilation, "compilation is null"); + FutureTask task = new FutureTask<>(compilation::get); + tasks.add(task); + try { + try { + executor.execute(task); + } + catch (java.util.concurrent.RejectedExecutionException failure) { + throw new TrinoException(HOGQL_COMPILATION_QUEUE_FULL, "HogQL compilation capacity is exhausted; retry later", failure); + } + + try { + if (timeoutMillis.isPresent()) { + return task.get(timeoutMillis.orElseThrow(), MILLISECONDS); + } + return task.get(); + } + catch (InterruptedException failure) { + task.cancel(true); + Thread.currentThread().interrupt(); + throw new TrinoException(GENERIC_INTERNAL_ERROR, "HogQL compilation was interrupted", failure); + } + catch (CancellationException failure) { + throw new TrinoException(HOGQL_COMPILATION_QUEUE_FULL, "HogQL compilation capacity is unavailable; retry later", failure); + } + catch (TimeoutException failure) { + task.cancel(true); + throw new TrinoException(HOGQL_COMPILATION_TIMEOUT, "HogQL compilation exceeded its time limit; retry later", failure); + } + catch (ExecutionException failure) { + Throwable cause = failure.getCause(); + if (cause instanceof RuntimeException runtimeException) { + throw runtimeException; + } + if (cause instanceof Error error) { + throw error; + } + throw new TrinoException(GENERIC_INTERNAL_ERROR, "HogQL compilation failed", cause); + } + } + finally { + tasks.remove(task); + } + } + + @PreDestroy + public void shutdown() + { + ownedExecutor.ifPresent(executor -> { + executor.shutdownNow(); + tasks.forEach(task -> task.cancel(true)); + }); + } + + boolean isShutdown() + { + return ownedExecutor.map(ThreadPoolExecutor::isShutdown).orElse(false); + } +} diff --git a/core/trino-main/src/main/java/io/trino/execution/HogQlParameterDecoder.java b/core/trino-main/src/main/java/io/trino/execution/HogQlParameterDecoder.java new file mode 100644 index 000000000000..01e6f9c8ba09 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/execution/HogQlParameterDecoder.java @@ -0,0 +1,574 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.execution; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.BooleanNode; +import com.fasterxml.jackson.databind.node.DecimalNode; +import com.fasterxml.jackson.databind.node.JsonNodeFactory; +import com.fasterxml.jackson.databind.node.NullNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import com.google.common.collect.ImmutableList; +import io.trino.hogql.compiler.HogQlCompileEnvelope; +import io.trino.hogql.compiler.HogQlCompilationResult; +import io.trino.hogql.compiler.HogQlModifierBinding; +import io.trino.hogql.compiler.HogQlTypedValue; +import io.trino.hogql.compiler.HogQlTypedValue.ArrayValue; +import io.trino.hogql.compiler.HogQlTypedValue.BooleanValue; +import io.trino.hogql.compiler.HogQlTypedValue.NullValue; +import io.trino.hogql.compiler.HogQlTypedValue.NumberValue; +import io.trino.hogql.compiler.HogQlTypedValue.ObjectValue; +import io.trino.hogql.compiler.HogQlTypedValue.StringValue; +import io.trino.hogql.compiler.HogQlTypedValue.Value; +import io.trino.spi.Location; +import io.trino.spi.TrinoException; +import io.trino.sql.parser.SqlParser; +import io.trino.sql.tree.Array; +import io.trino.sql.tree.BinaryLiteral; +import io.trino.sql.tree.BooleanLiteral; +import io.trino.sql.tree.Cast; +import io.trino.sql.tree.DataType; +import io.trino.sql.tree.DataTypeParameter; +import io.trino.sql.tree.DateTimeDataType; +import io.trino.sql.tree.DecimalLiteral; +import io.trino.sql.tree.DoubleLiteral; +import io.trino.sql.tree.Expression; +import io.trino.sql.tree.FunctionCall; +import io.trino.sql.tree.GenericDataType; +import io.trino.sql.tree.GenericLiteral; +import io.trino.sql.tree.LongLiteral; +import io.trino.sql.tree.NodeLocation; +import io.trino.sql.tree.NullLiteral; +import io.trino.sql.tree.NumericParameter; +import io.trino.sql.tree.Parameter; +import io.trino.sql.tree.QualifiedName; +import io.trino.sql.tree.Row; +import io.trino.sql.tree.RowDataType; +import io.trino.sql.tree.StringLiteral; +import io.trino.sql.tree.TypeParameter; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.RoundingMode; +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.UUID; + +import static io.airlift.slice.Slices.utf8Slice; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_BINDING_ERROR; +import static io.trino.spi.type.CharType.MAX_LENGTH; +import static io.trino.type.DateTimes.extractTimePrecision; +import static io.trino.type.DateTimes.extractTimestampPrecision; +import static io.trino.type.DateTimes.parseTime; +import static io.trino.type.DateTimes.parseTimeWithTimeZone; +import static io.trino.type.DateTimes.parseTimestamp; +import static io.trino.type.DateTimes.parseTimestampWithTimeZone; +import static io.trino.type.DateTimes.timeHasTimeZone; +import static io.trino.type.DateTimes.timestampHasTimeZone; +import static io.trino.util.DateTimeUtils.parseDate; +import static java.lang.Float.isFinite; +import static java.lang.Math.toIntExact; +import static java.util.Objects.requireNonNull; + +final class HogQlParameterDecoder +{ + private static final Set INTEGRAL_TYPES = Set.of("TINYINT", "SMALLINT", "INTEGER", "BIGINT"); + private static final Set STRING_TYPES = Set.of("VARCHAR", "CHAR", "DATE", "UUID", "IPADDRESS", "VARBINARY"); + + private final SqlParser sqlParser; + + HogQlParameterDecoder(SqlParser sqlParser) + { + this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); + } + + List decode(HogQlCompilationResult result, HogQlCompileEnvelope envelope) + { + requireNonNull(result, "result is null"); + requireNonNull(envelope, "envelope is null"); + List parameters = ParameterExtractor.extractParameters(result.statement()); + if (parameters.size() != result.parameterNames().size()) { + throw new IllegalStateException("HogQL compiler returned inconsistent parameter metadata"); + } + + ImmutableList.Builder values = ImmutableList.builderWithExpectedSize(parameters.size()); + for (int index = 0; index < parameters.size(); index++) { + String name = result.parameterNames().get(index); + NodeLocation location = parameters.get(index).getLocation().orElseThrow(); + try { + HogQlTypedValue binding = envelope.bindingForPlaceholder(name).orElseThrow(() -> new NullPointerException("binding is null")); + values.add(decode(binding, location)); + } + catch (RuntimeException _) { + throw bindingError(name, location); + } + } + return values.build(); + } + + Expression decode(HogQlTypedValue binding, NodeLocation location) + { + requireNonNull(binding, "binding is null"); + requireNonNull(location, "location is null"); + DataType type = sqlParser.createType(binding.type()); + return decode(binding.value(), type, location); + } + + Expression decode(HogQlModifierBinding binding, NodeLocation location) + { + requireNonNull(binding, "binding is null"); + try { + return decode(binding.value(), location); + } + catch (RuntimeException _) { + throw modifierBindingError(binding.modifierName(), location); + } + } + + private static Expression decode(Value value, DataType type, NodeLocation location) + { + if (value instanceof NullValue) { + validateSupportedType(type); + return new Cast(location, new NullLiteral(location), type); + } + if (type instanceof DateTimeDataType dateTimeType) { + return new Cast(location, decodeDateTime(value, dateTimeType, location), type); + } + if (type instanceof RowDataType rowType) { + return decodeRow(value, rowType, location); + } + if (!(type instanceof GenericDataType genericType)) { + throw new IllegalArgumentException("unsupported type"); + } + + String typeName = genericType.getName().getCanonicalValue(); + if (typeName.equals("JSON")) { + requireNoArguments(genericType); + return new GenericLiteral(location, "JSON", toJson(value).toString()); + } + if (typeName.equals("ARRAY")) { + return decodeArray(value, genericType, location); + } + if (typeName.equals("MAP")) { + return decodeMap(value, genericType, location); + } + + Expression scalar = switch (value) { + case BooleanValue booleanValue -> decodeBoolean(booleanValue, genericType, location); + case NumberValue numberValue -> decodeNumber(numberValue, genericType, location); + case StringValue stringValue -> decodeString(stringValue, genericType, location); + case ArrayValue _, ObjectValue _ -> throw new IllegalArgumentException("container value requires a container type"); + case NullValue _ -> throw new IllegalStateException("null handled above"); + }; + return new Cast(location, scalar, type); + } + + private static Expression decodeBoolean(BooleanValue value, GenericDataType type, NodeLocation location) + { + requireType(type, "BOOLEAN"); + return new BooleanLiteral(location, Boolean.toString(value.value())); + } + + private static Expression decodeNumber(NumberValue value, GenericDataType type, NodeLocation location) + { + String typeName = type.getName().getCanonicalValue(); + BigDecimal number = new BigDecimal(value.value()); + if (INTEGRAL_TYPES.contains(typeName)) { + requireNoArguments(type); + BigInteger integer = number.toBigIntegerExact(); + validateIntegerRange(typeName, integer); + return new LongLiteral(location, integer.toString()); + } + if (typeName.equals("REAL") || typeName.equals("DOUBLE")) { + requireNoArguments(type); + double doubleValue = number.doubleValue(); + if (!Double.isFinite(doubleValue) || (typeName.equals("REAL") && !isFinite(number.floatValue()))) { + throw new IllegalArgumentException("floating-point value is out of range"); + } + return new DoubleLiteral(location, value.value()); + } + if (typeName.equals("DECIMAL")) { + List arguments = validateDecimalType(type); + int precision = toIntExact(arguments.get(0)); + int scale = toIntExact(arguments.get(1)); + BigDecimal scaled = number.setScale(scale, RoundingMode.UNNECESSARY); + if (scaled.precision() > precision) { + throw new IllegalArgumentException("decimal value is out of range"); + } + return new DecimalLiteral(location, value.value()); + } + throw new IllegalArgumentException("number value has an incompatible type"); + } + + private static Expression decodeString(StringValue value, GenericDataType type, NodeLocation location) + { + String typeName = type.getName().getCanonicalValue(); + if (!STRING_TYPES.contains(typeName)) { + throw new IllegalArgumentException("string value has an incompatible type"); + } + + return switch (typeName) { + case "VARCHAR" -> { + validateStringLength(value.value(), type, false); + yield new StringLiteral(location, value.value()); + } + case "CHAR" -> { + validateStringLength(value.value(), type, true); + yield new StringLiteral(location, value.value()); + } + case "DATE" -> { + requireNoArguments(type); + parseDate(utf8Slice(value.value())); + yield new GenericLiteral(location, "DATE", value.value()); + } + case "UUID" -> { + requireNoArguments(type); + UUID.fromString(value.value()); + if (value.value().length() != 36) { + throw new IllegalArgumentException("UUID value is not canonical"); + } + yield new GenericLiteral(location, "UUID", value.value()); + } + case "IPADDRESS" -> { + requireNoArguments(type); + InetAddress.ofLiteral(value.value()); + yield new GenericLiteral(location, "IPADDRESS", value.value()); + } + case "VARBINARY" -> { + requireNoArguments(type); + yield new BinaryLiteral(location, value.value()); + } + default -> throw new IllegalArgumentException("unsupported string type"); + }; + } + + private static Expression decodeDateTime(Value value, DateTimeDataType type, NodeLocation location) + { + if (!(value instanceof StringValue stringValue)) { + throw new IllegalArgumentException("date-time value must be a string"); + } + validateDateTimePrecision(type); + String literal = stringValue.value(); + if (type.getType() == DateTimeDataType.Type.TIMESTAMP) { + int precision = extractTimestampPrecision(literal); + validateLiteralPrecision(precision); + if (timestampHasTimeZone(literal) != type.isWithTimeZone()) { + throw new IllegalArgumentException("timestamp time zone does not match type"); + } + if (type.isWithTimeZone()) { + parseTimestampWithTimeZone(precision, literal); + } + else { + parseTimestamp(precision, literal); + } + } + else { + int precision = extractTimePrecision(literal); + validateLiteralPrecision(precision); + if (timeHasTimeZone(literal) != type.isWithTimeZone()) { + throw new IllegalArgumentException("time zone does not match type"); + } + if (type.isWithTimeZone()) { + parseTimeWithTimeZone(precision, literal); + } + else { + parseTime(literal); + } + } + return new GenericLiteral(location, type.getType().name(), literal); + } + + private static Expression decodeArray(Value value, GenericDataType type, NodeLocation location) + { + if (!(value instanceof ArrayValue arrayValue)) { + throw new IllegalArgumentException("array value is required"); + } + DataType elementType = typeArgument(type, 0, 1); + List elements = arrayValue.value().stream() + .map(element -> decode(element, elementType, location)) + .toList(); + return new Cast(location, new Array(location, elements), type); + } + + private static Expression decodeMap(Value value, GenericDataType type, NodeLocation location) + { + if (!(value instanceof ObjectValue objectValue)) { + throw new IllegalArgumentException("object value is required"); + } + DataType keyType = typeArgument(type, 0, 2); + DataType valueType = typeArgument(type, 1, 2); + if (!(keyType instanceof GenericDataType genericKeyType) || !genericKeyType.getName().getCanonicalValue().equals("VARCHAR")) { + throw new IllegalArgumentException("object keys require varchar map keys"); + } + validateStringType(genericKeyType, false); + + List> entries = objectValue.value().entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .toList(); + FunctionCall map = entries.isEmpty() + ? new FunctionCall(location, QualifiedName.of("map"), List.of()) + : new FunctionCall( + location, + QualifiedName.of("map"), + List.of( + new Array(location, entries.stream() + .map(entry -> decode(new StringValue(entry.getKey()), keyType, location)) + .toList()), + new Array(location, entries.stream() + .map(entry -> decode(entry.getValue(), valueType, location)) + .toList()))); + return new Cast(location, map, type); + } + + private static Expression decodeRow(Value value, RowDataType type, NodeLocation location) + { + List fieldValues; + if (value instanceof ObjectValue objectValue) { + if (type.getFields().stream().anyMatch(field -> field.getName().isEmpty())) { + throw new IllegalArgumentException("object values require named row fields"); + } + Set fieldNames = type.getFields().stream() + .map(field -> field.getName().orElseThrow().getValue()) + .collect(java.util.stream.Collectors.toUnmodifiableSet()); + if (!fieldNames.equals(objectValue.value().keySet())) { + throw new IllegalArgumentException("row fields do not match"); + } + fieldValues = type.getFields().stream() + .map(field -> objectValue.value().get(field.getName().orElseThrow().getValue())) + .toList(); + } + else if (value instanceof ArrayValue arrayValue && arrayValue.value().size() == type.getFields().size()) { + fieldValues = arrayValue.value(); + } + else { + throw new IllegalArgumentException("row value has an incompatible shape"); + } + + List fields = new ArrayList<>(fieldValues.size()); + for (int index = 0; index < fieldValues.size(); index++) { + fields.add(new Row.Field(location, Optional.empty(), decode(fieldValues.get(index), type.getFields().get(index).getType(), location))); + } + return new Cast(location, new Row(location, fields), type); + } + + private static JsonNode toJson(Value value) + { + return switch (value) { + case NullValue _ -> NullNode.getInstance(); + case BooleanValue booleanValue -> BooleanNode.valueOf(booleanValue.value()); + case NumberValue numberValue -> DecimalNode.valueOf(new BigDecimal(numberValue.value())); + case StringValue stringValue -> TextNode.valueOf(stringValue.value()); + case ArrayValue arrayValue -> { + ArrayNode array = JsonNodeFactory.instance.arrayNode(); + arrayValue.value().forEach(element -> array.add(toJson(element))); + yield array; + } + case ObjectValue objectValue -> { + ObjectNode object = JsonNodeFactory.instance.objectNode(); + objectValue.value().entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .forEach(entry -> object.set(entry.getKey(), toJson(entry.getValue()))); + yield object; + } + }; + } + + private static void validateSupportedType(DataType type) + { + if (type instanceof DateTimeDataType dateTimeType) { + validateDateTimePrecision(dateTimeType); + return; + } + if (type instanceof RowDataType rowType) { + rowType.getFields().forEach(field -> validateSupportedType(field.getType())); + return; + } + if (!(type instanceof GenericDataType genericType)) { + throw new IllegalArgumentException("unsupported type"); + } + String typeName = genericType.getName().getCanonicalValue(); + if (Set.of("BOOLEAN", "TINYINT", "SMALLINT", "INTEGER", "BIGINT", "REAL", "DOUBLE", "DATE", "UUID", "IPADDRESS", "VARBINARY", "JSON").contains(typeName)) { + requireNoArguments(genericType); + return; + } + if (typeName.equals("VARCHAR")) { + validateStringType(genericType, false); + return; + } + if (typeName.equals("CHAR")) { + validateStringType(genericType, true); + return; + } + if (typeName.equals("DECIMAL")) { + validateDecimalType(genericType); + return; + } + if (typeName.equals("ARRAY")) { + validateSupportedType(typeArgument(genericType, 0, 1)); + return; + } + if (typeName.equals("MAP")) { + validateSupportedType(typeArgument(genericType, 0, 2)); + validateSupportedType(typeArgument(genericType, 1, 2)); + return; + } + throw new IllegalArgumentException("unsupported type"); + } + + private static void validateIntegerRange(String type, BigInteger value) + { + BigInteger minimum; + BigInteger maximum; + switch (type) { + case "TINYINT" -> { + minimum = BigInteger.valueOf(Byte.MIN_VALUE); + maximum = BigInteger.valueOf(Byte.MAX_VALUE); + } + case "SMALLINT" -> { + minimum = BigInteger.valueOf(Short.MIN_VALUE); + maximum = BigInteger.valueOf(Short.MAX_VALUE); + } + case "INTEGER" -> { + minimum = BigInteger.valueOf(Integer.MIN_VALUE); + maximum = BigInteger.valueOf(Integer.MAX_VALUE); + } + case "BIGINT" -> { + minimum = BigInteger.valueOf(Long.MIN_VALUE); + maximum = BigInteger.valueOf(Long.MAX_VALUE); + } + default -> throw new IllegalArgumentException("unsupported integral type"); + } + if (value.compareTo(minimum) < 0 || value.compareTo(maximum) > 0) { + throw new IllegalArgumentException("integral value is out of range"); + } + } + + private static void validateStringLength(String value, GenericDataType type, boolean lengthRequired) + { + validateStringType(type, lengthRequired); + if (!type.getArguments().isEmpty()) { + long maximumLength = numericArguments(type, 1).getFirst(); + if (value.codePointCount(0, value.length()) > maximumLength) { + throw new IllegalArgumentException("string value is too long"); + } + } + } + + private static void validateStringType(GenericDataType type, boolean lengthRequired) + { + int argumentCount = type.getArguments().size(); + if ((lengthRequired && argumentCount != 1) || (!lengthRequired && argumentCount > 1)) { + throw new IllegalArgumentException("invalid string type"); + } + if (argumentCount == 1) { + long length = numericArguments(type, 1).getFirst(); + long maximumLength = type.getName().getCanonicalValue().equals("CHAR") ? MAX_LENGTH : io.trino.spi.type.VarcharType.MAX_LENGTH; + if (length < 0 || length > maximumLength) { + throw new IllegalArgumentException("invalid string length"); + } + } + } + + private static List validateDecimalType(GenericDataType type) + { + List arguments = numericArguments(type, 2); + long precision = arguments.get(0); + long scale = arguments.get(1); + if (precision < 1 || precision > 38 || scale < 0 || scale > precision) { + throw new IllegalArgumentException("invalid decimal type"); + } + return arguments; + } + + private static void validateDateTimePrecision(DateTimeDataType type) + { + type.getPrecision().ifPresent(precision -> { + if (!(precision instanceof NumericParameter numericPrecision) || numericPrecision.getParsedValue() < 0 || numericPrecision.getParsedValue() > 12) { + throw new IllegalArgumentException("invalid date-time precision"); + } + }); + } + + private static void validateLiteralPrecision(int precision) + { + if (precision > 12) { + throw new IllegalArgumentException("date-time literal precision is out of range"); + } + } + + private static void requireType(GenericDataType type, String expected) + { + if (!type.getName().getCanonicalValue().equals(expected)) { + throw new IllegalArgumentException("value has an incompatible type"); + } + requireNoArguments(type); + } + + private static void requireNoArguments(GenericDataType type) + { + if (!type.getArguments().isEmpty()) { + throw new IllegalArgumentException("type does not accept arguments"); + } + } + + private static DataType typeArgument(GenericDataType type, int index, int expectedCount) + { + if (type.getArguments().size() != expectedCount || !(type.getArguments().get(index) instanceof TypeParameter typeParameter)) { + throw new IllegalArgumentException("invalid container type"); + } + return typeParameter.getValue(); + } + + private static List numericArguments(GenericDataType type, int expectedCount) + { + if (type.getArguments().size() != expectedCount) { + throw new IllegalArgumentException("invalid type parameters"); + } + return type.getArguments().stream() + .map(HogQlParameterDecoder::numericArgument) + .toList(); + } + + private static long numericArgument(DataTypeParameter parameter) + { + if (!(parameter instanceof NumericParameter numericParameter)) { + throw new IllegalArgumentException("numeric type parameter is required"); + } + return numericParameter.getParsedValue(); + } + + private static TrinoException bindingError(String name, NodeLocation location) + { + return new TrinoException( + HOGQL_BINDING_ERROR, + Optional.of(new Location(location.getLineNumber(), location.getColumnNumber())), + "Invalid HogQL parameter binding: " + name, + null); + } + + private static TrinoException modifierBindingError(String name, NodeLocation location) + { + return new TrinoException( + HOGQL_BINDING_ERROR, + Optional.of(new Location(location.getLineNumber(), location.getColumnNumber())), + "Invalid HogQL modifier binding: " + name, + null); + } +} diff --git a/core/trino-main/src/main/java/io/trino/execution/QueryLanguage.java b/core/trino-main/src/main/java/io/trino/execution/QueryLanguage.java new file mode 100644 index 000000000000..2d56e69400d2 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/execution/QueryLanguage.java @@ -0,0 +1,20 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.execution; + +public enum QueryLanguage +{ + TRINO, + HOGQL, +} diff --git a/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java b/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java index dc093feb4a01..2508679a1cdb 100644 --- a/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java +++ b/core/trino-main/src/main/java/io/trino/execution/QueryPreparer.java @@ -16,20 +16,40 @@ import com.google.common.collect.ImmutableList; import com.google.inject.Inject; import io.trino.Session; +import io.trino.hogql.HogQlCompilationEvent.Dimensions; +import io.trino.hogql.HogQlCompilationObserver; +import io.trino.hogql.HogQlCompilationTracker; +import io.trino.hogql.compiler.HogQlCompilationResult; +import io.trino.hogql.compiler.HogQlCompileEnvelope; +import io.trino.hogql.compiler.HogQlCompiler; +import io.trino.hogql.compiler.HogQlModifierBinding; +import io.trino.hogql.compiler.HogQlSemanticCatalogContext; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider; import io.trino.spi.TrinoException; import io.trino.spi.resourcegroups.QueryType; import io.trino.sql.parser.ParsingException; import io.trino.sql.parser.SqlParser; import io.trino.sql.tree.Execute; import io.trino.sql.tree.ExecuteImmediate; +import io.trino.sql.tree.Explain; import io.trino.sql.tree.ExplainAnalyze; +import io.trino.sql.tree.ExplainFormat; +import io.trino.sql.tree.ExplainType; import io.trino.sql.tree.Expression; +import io.trino.sql.tree.Identifier; +import io.trino.sql.tree.NodeLocation; +import io.trino.sql.tree.QualifiedName; +import io.trino.sql.tree.SessionProperty; import io.trino.sql.tree.Statement; import java.util.List; import java.util.Optional; import static io.trino.execution.ParameterExtractor.getParameterCount; +import static io.trino.hogql.HogQlCatalogIdentifiers.physicalCatalog; +import static io.trino.hogql.HogQlCompilationEvent.Phase.COMPILATION; +import static io.trino.hogql.HogQlCompilationEvent.Phase.PARAMETER_BINDING; +import static io.trino.hogql.HogQlCompilationObserver.NOOP; import static io.trino.spi.StandardErrorCode.INVALID_PARAMETER_USAGE; import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; import static io.trino.sql.analyzer.ConstantExpressionVerifier.verifyExpressionIsConstant; @@ -41,22 +61,157 @@ public class QueryPreparer { private final SqlParser sqlParser; + private final Optional hogQlCompiler; + private final HogQlParameterDecoder hogQlParameterDecoder; + private final HogQlCompilationObserver hogQlCompilationObserver; + private final Optional hogQlSemanticCatalogSnapshotProvider; + private final HogQlCompilationExecutor hogQlCompilationExecutor; - @Inject public QueryPreparer(SqlParser sqlParser) + { + this(sqlParser, Optional.empty(), NOOP, Optional.empty(), HogQlCompilationExecutor.directExecutor()); + } + + public QueryPreparer(SqlParser sqlParser, HogQlCompiler hogQlCompiler) + { + this(sqlParser, Optional.of(requireNonNull(hogQlCompiler, "hogQlCompiler is null")), NOOP, Optional.empty(), HogQlCompilationExecutor.directExecutor()); + } + + public QueryPreparer(SqlParser sqlParser, HogQlCompiler hogQlCompiler, HogQlCompilationObserver hogQlCompilationObserver) + { + this(sqlParser, Optional.of(requireNonNull(hogQlCompiler, "hogQlCompiler is null")), hogQlCompilationObserver, Optional.empty(), HogQlCompilationExecutor.directExecutor()); + } + + public QueryPreparer( + SqlParser sqlParser, + HogQlCompiler hogQlCompiler, + HogQlCompilationObserver hogQlCompilationObserver, + Optional hogQlSemanticCatalogSnapshotProvider) + { + this(sqlParser, + Optional.of(requireNonNull(hogQlCompiler, "hogQlCompiler is null")), + hogQlCompilationObserver, + hogQlSemanticCatalogSnapshotProvider, + HogQlCompilationExecutor.directExecutor()); + } + + @Inject + public QueryPreparer( + SqlParser sqlParser, + HogQlCompiler hogQlCompiler, + HogQlCompilationObserver hogQlCompilationObserver, + Optional hogQlSemanticCatalogSnapshotProvider, + HogQlCompilationExecutor hogQlCompilationExecutor) + { + this(sqlParser, + Optional.of(requireNonNull(hogQlCompiler, "hogQlCompiler is null")), + hogQlCompilationObserver, + hogQlSemanticCatalogSnapshotProvider, + hogQlCompilationExecutor); + } + + private QueryPreparer( + SqlParser sqlParser, + Optional hogQlCompiler, + HogQlCompilationObserver hogQlCompilationObserver, + Optional hogQlSemanticCatalogSnapshotProvider, + HogQlCompilationExecutor hogQlCompilationExecutor) { this.sqlParser = requireNonNull(sqlParser, "sqlParser is null"); + this.hogQlCompiler = requireNonNull(hogQlCompiler, "hogQlCompiler is null"); + this.hogQlParameterDecoder = new HogQlParameterDecoder(sqlParser); + this.hogQlCompilationObserver = requireNonNull(hogQlCompilationObserver, "hogQlCompilationObserver is null"); + this.hogQlSemanticCatalogSnapshotProvider = requireNonNull(hogQlSemanticCatalogSnapshotProvider, "hogQlSemanticCatalogSnapshotProvider is null"); + this.hogQlCompilationExecutor = requireNonNull(hogQlCompilationExecutor, "hogQlCompilationExecutor is null"); } public PreparedQuery prepareQuery(Session session, String query) throws ParsingException, TrinoException { - Statement wrappedStatement = sqlParser.createStatement(query); - return prepareQuery(session, wrappedStatement); + return prepareQuery(session, QuerySubmission.trino(query)); + } + + public PreparedQuery prepareQuery(Session session, QuerySubmission submission) + throws ParsingException, TrinoException + { + requireNonNull(submission, "submission is null"); + return switch (submission.language()) { + case TRINO -> prepareQuery(session, sqlParser.createStatement(submission.originalText())); + case HOGQL -> { + HogQlCompileEnvelope envelope = submission.hogQlEnvelope().orElseThrow(); + HogQlCompilationTracker tracker = new HogQlCompilationTracker(hogQlCompilationObserver, Dimensions.fromEnvelope(envelope)); + PreparedQuery preparedQuery; + try { + HogQlCompilationResult result = tracker.observe(COMPILATION, () -> hogQlCompilationExecutor.execute(() -> hogQlCompiler + .orElseThrow(() -> new TrinoException(NOT_SUPPORTED, "HogQL query submission is disabled")) + .compileV0(envelope, semanticCatalogContext(session)))); + tracker.catalogGeneration(result.catalogGeneration()); + Statement statement = explain(result.statement(), submission.hogQlExplain()); + preparedQuery = tracker.observe(PARAMETER_BINDING, () -> prepareQuery( + session, + statement, + Optional.of(hogQlParameterDecoder.decode(result, envelope)), + sessionPropertyOverrides(result, statement))); + } + catch (RuntimeException | Error failure) { + tracker.failed(failure); + throw failure; + } + tracker.succeeded(); + yield preparedQuery; + } + }; + } + + private static Statement explain(Statement statement, Optional explain) + { + if (explain.isEmpty()) { + return statement; + } + NodeLocation location = statement.getLocation().orElseThrow(); + QuerySubmission.HogQlExplain options = explain.orElseThrow(); + return new Explain( + location, + statement, + List.of( + new ExplainType(location, options.type()), + new ExplainFormat(location, options.format()))); + } + + private Optional semanticCatalogContext(Session session) + { + if (hogQlSemanticCatalogSnapshotProvider.isEmpty() || session.getCatalog().isEmpty()) { + return Optional.empty(); + } + return Optional.of(new HogQlSemanticCatalogContext( + physicalCatalog(session.getCatalog().orElseThrow()), + hogQlSemanticCatalogSnapshotProvider.orElseThrow())); } public PreparedQuery prepareQuery(Session session, Statement wrappedStatement) throws ParsingException, TrinoException + { + return prepareQuery(session, wrappedStatement, Optional.empty()); + } + + public PreparedQuery prepareQuery(Session session, Statement wrappedStatement, List suppliedParameters) + throws ParsingException, TrinoException + { + return prepareQuery(session, wrappedStatement, Optional.of(ImmutableList.copyOf(suppliedParameters))); + } + + private PreparedQuery prepareQuery(Session session, Statement wrappedStatement, Optional> suppliedParameters) + throws ParsingException, TrinoException + { + return prepareQuery(session, wrappedStatement, suppliedParameters, List.of()); + } + + private PreparedQuery prepareQuery( + Session session, + Statement wrappedStatement, + Optional> suppliedParameters, + List sessionPropertyOverrides) + throws ParsingException, TrinoException { Statement statement = wrappedStatement; Optional prepareSql = Optional.empty(); @@ -77,16 +232,42 @@ else if (statement instanceof ExplainAnalyze explainAnalyzeStatement) { } } - List parameters = ImmutableList.of(); - if (wrappedStatement instanceof Execute executeStatement) { + List parameters; + if (suppliedParameters.isPresent()) { + parameters = suppliedParameters.orElseThrow(); + } + else if (wrappedStatement instanceof Execute executeStatement) { parameters = executeStatement.getParameters(); } else if (wrappedStatement instanceof ExecuteImmediate executeImmediateStatement) { parameters = executeImmediateStatement.getParameters(); } + else { + parameters = ImmutableList.of(); + } validateParameters(statement, parameters); - return new PreparedQuery(statement, parameters, prepareSql); + return new PreparedQuery(statement, parameters, prepareSql, sessionPropertyOverrides); + } + + private List sessionPropertyOverrides(HogQlCompilationResult result, Statement statement) + { + NodeLocation location = statement.getLocation().orElseThrow(); + return result.modifierBindings().stream() + .map(binding -> modifierBinding(binding, location)) + .flatMap(Optional::stream) + .toList(); + } + + private Optional modifierBinding(HogQlModifierBinding binding, NodeLocation location) + { + Expression value = hogQlParameterDecoder.decode(binding, location); + return binding.sessionProperty().map(property -> { + QualifiedName name = QualifiedName.of(property.stream() + .map(part -> new Identifier(location, part.value(), part.delimited())) + .toList()); + return new SessionProperty(location, name, value); + }); } private static void validateParameters(Statement node, List parameterValues) @@ -105,12 +286,19 @@ public static class PreparedQuery private final Statement statement; private final List parameters; private final Optional prepareSql; + private final List sessionPropertyOverrides; public PreparedQuery(Statement statement, List parameters, Optional prepareSql) + { + this(statement, parameters, prepareSql, List.of()); + } + + public PreparedQuery(Statement statement, List parameters, Optional prepareSql, List sessionPropertyOverrides) { this.statement = requireNonNull(statement, "statement is null"); this.parameters = ImmutableList.copyOf(requireNonNull(parameters, "parameters is null")); this.prepareSql = requireNonNull(prepareSql, "prepareSql is null"); + this.sessionPropertyOverrides = ImmutableList.copyOf(requireNonNull(sessionPropertyOverrides, "sessionPropertyOverrides is null")); } public Statement getStatement() @@ -127,5 +315,10 @@ public Optional getPrepareSql() { return prepareSql; } + + public List getSessionPropertyOverrides() + { + return sessionPropertyOverrides; + } } } diff --git a/core/trino-main/src/main/java/io/trino/execution/QuerySubmission.java b/core/trino-main/src/main/java/io/trino/execution/QuerySubmission.java new file mode 100644 index 000000000000..c44ab011e3d9 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/execution/QuerySubmission.java @@ -0,0 +1,70 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.execution; + +import io.trino.hogql.compiler.HogQlCompileEnvelope; +import io.trino.sql.tree.ExplainFormat; +import io.trino.sql.tree.ExplainType; + +import java.util.Optional; + +import static io.trino.execution.QueryLanguage.HOGQL; +import static io.trino.execution.QueryLanguage.TRINO; +import static java.util.Objects.requireNonNull; + +public record QuerySubmission( + QueryLanguage language, + String originalText, + Optional hogQlEnvelope, + Optional hogQlExplain) +{ + public QuerySubmission + { + requireNonNull(language, "language is null"); + requireNonNull(originalText, "originalText is null"); + hogQlEnvelope = requireNonNull(hogQlEnvelope, "hogQlEnvelope is null"); + hogQlExplain = requireNonNull(hogQlExplain, "hogQlExplain is null"); + if (language == TRINO && (hogQlEnvelope.isPresent() || hogQlExplain.isPresent())) { + throw new IllegalArgumentException("Trino submission has HogQL context"); + } + if (language == HOGQL && (hogQlEnvelope.isEmpty() || !hogQlEnvelope.orElseThrow().query().equals(originalText))) { + throw new IllegalArgumentException("HogQL submission envelope does not match its query"); + } + } + + public static QuerySubmission trino(String originalText) + { + return new QuerySubmission(TRINO, originalText, Optional.empty(), Optional.empty()); + } + + public static QuerySubmission hogQl(HogQlCompileEnvelope envelope) + { + return hogQl(envelope, Optional.empty()); + } + + public static QuerySubmission hogQl(HogQlCompileEnvelope envelope, Optional explain) + { + requireNonNull(envelope, "envelope is null"); + return new QuerySubmission(HOGQL, envelope.query(), Optional.of(envelope), requireNonNull(explain, "explain is null")); + } + + public record HogQlExplain(ExplainType.Type type, ExplainFormat.Type format) + { + public HogQlExplain + { + requireNonNull(type, "type is null"); + requireNonNull(format, "format is null"); + } + } +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/ForHogQlExchangeRate.java b/core/trino-main/src/main/java/io/trino/hogql/ForHogQlExchangeRate.java new file mode 100644 index 000000000000..db82d8b5077e --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/ForHogQlExchangeRate.java @@ -0,0 +1,29 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import com.google.inject.BindingAnnotation; + +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@BindingAnnotation +@Target({FIELD, PARAMETER, METHOD}) +@Retention(RUNTIME) +public @interface ForHogQlExchangeRate {} diff --git a/core/trino-main/src/main/java/io/trino/hogql/ForHogQlSemanticCatalog.java b/core/trino-main/src/main/java/io/trino/hogql/ForHogQlSemanticCatalog.java new file mode 100644 index 000000000000..65e39faab550 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/ForHogQlSemanticCatalog.java @@ -0,0 +1,29 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import com.google.inject.BindingAnnotation; + +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Retention(RUNTIME) +@Target({FIELD, PARAMETER, METHOD}) +@BindingAnnotation +public @interface ForHogQlSemanticCatalog {} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlCatalogIdentifiers.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlCatalogIdentifiers.java new file mode 100644 index 000000000000..4096ebe8ae22 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlCatalogIdentifiers.java @@ -0,0 +1,35 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; + +import java.util.Locale; +import java.util.regex.Pattern; + +import static java.util.Objects.requireNonNull; + +public final class HogQlCatalogIdentifiers +{ + private static final Pattern UNDELIMITED_IDENTIFIER = Pattern.compile("[A-Za-z_][A-Za-z0-9_]*"); + + private HogQlCatalogIdentifiers() {} + + public static PhysicalIdentifier physicalCatalog(String catalog) + { + requireNonNull(catalog, "catalog is null"); + boolean delimited = !UNDELIMITED_IDENTIFIER.matcher(catalog).matches() || !catalog.equals(catalog.toLowerCase(Locale.ENGLISH)); + return new PhysicalIdentifier(catalog, delimited); + } +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationEvent.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationEvent.java new file mode 100644 index 000000000000..49bb6a6c0aa3 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationEvent.java @@ -0,0 +1,117 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import io.trino.hogql.compiler.HogQlCompileEnvelope; +import io.trino.hogql.parser.HogQlLanguageVersion; + +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; + +import static io.trino.hogql.HogQlCompilationEvent.Outcome.SUCCESS; +import static java.util.Objects.requireNonNull; + +public record HogQlCompilationEvent( + Dimensions dimensions, + Outcome outcome, + Optional failedPhase, + long totalNanos, + Map phaseNanos) +{ + public HogQlCompilationEvent + { + dimensions = requireNonNull(dimensions, "dimensions is null"); + outcome = requireNonNull(outcome, "outcome is null"); + failedPhase = requireNonNull(failedPhase, "failedPhase is null"); + if (outcome == SUCCESS && failedPhase.isPresent()) { + throw new IllegalArgumentException("successful compilation has a failed phase"); + } + if (totalNanos < 0) { + throw new IllegalArgumentException("totalNanos is negative"); + } + phaseNanos = Map.copyOf(requireNonNull(phaseNanos, "phaseNanos is null")); + if (phaseNanos.values().stream().anyMatch(duration -> duration == null || duration < 0)) { + throw new IllegalArgumentException("phaseNanos contains an invalid duration"); + } + } + + public enum Phase + { + COMPILATION, + PARSE, + BIND, + LOWER, + PARAMETER_BINDING, + } + + public enum Outcome + { + SUCCESS, + USER_ERROR, + INTERNAL_ERROR, + EXTERNAL_ERROR, + INSUFFICIENT_RESOURCES, + } + + public record Dimensions( + int protocolVersion, + HogQlLanguageVersion languageVersion, + int parameterCount, + int variableCount, + int filterCount, + int modifierCount, + OptionalLong catalogGeneration) + { + public Dimensions + { + if (protocolVersion <= 0) { + throw new IllegalArgumentException("protocolVersion must be positive"); + } + languageVersion = requireNonNull(languageVersion, "languageVersion is null"); + if (parameterCount < 0 || variableCount < 0 || filterCount < 0 || modifierCount < 0) { + throw new IllegalArgumentException("semantic field count is negative"); + } + catalogGeneration = requireNonNull(catalogGeneration, "catalogGeneration is null"); + if (catalogGeneration.isPresent() && catalogGeneration.orElseThrow() <= 0) { + throw new IllegalArgumentException("catalogGeneration must be positive"); + } + } + + public static Dimensions fromEnvelope(HogQlCompileEnvelope envelope) + { + requireNonNull(envelope, "envelope is null"); + return new Dimensions( + envelope.protocolVersion(), + envelope.languageVersion(), + envelope.parameters().size(), + envelope.variables().size(), + envelope.filters().size(), + envelope.modifiers().size(), + envelope.catalogGeneration()); + } + + public Dimensions withCatalogGeneration(OptionalLong catalogGeneration) + { + return new Dimensions( + protocolVersion, + languageVersion, + parameterCount, + variableCount, + filterCount, + modifierCount, + catalogGeneration); + } + } +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationObserver.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationObserver.java new file mode 100644 index 000000000000..e1b74ed4498a --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationObserver.java @@ -0,0 +1,22 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +@FunctionalInterface +public interface HogQlCompilationObserver +{ + HogQlCompilationObserver NOOP = _ -> {}; + + void compilationCompleted(HogQlCompilationEvent event); +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationStats.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationStats.java new file mode 100644 index 000000000000..5e4d7f47c0db --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationStats.java @@ -0,0 +1,159 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import io.airlift.stats.CounterStat; +import io.airlift.stats.TimeStat; +import io.trino.hogql.HogQlCompilationEvent.Phase; +import org.weakref.jmx.Managed; +import org.weakref.jmx.Nested; + +import java.util.concurrent.atomic.AtomicLong; + +import static java.util.concurrent.TimeUnit.NANOSECONDS; + +public final class HogQlCompilationStats + implements HogQlCompilationObserver +{ + private final CounterStat completedCompilations = new CounterStat(); + private final CounterStat successfulCompilations = new CounterStat(); + private final CounterStat userErrorFailures = new CounterStat(); + private final CounterStat internalErrorFailures = new CounterStat(); + private final CounterStat externalErrorFailures = new CounterStat(); + private final CounterStat insufficientResourcesFailures = new CounterStat(); + private final TimeStat totalTime = new TimeStat(NANOSECONDS); + private final TimeStat compilationTime = new TimeStat(NANOSECONDS); + private final TimeStat parseTime = new TimeStat(NANOSECONDS); + private final TimeStat bindTime = new TimeStat(NANOSECONDS); + private final TimeStat lowerTime = new TimeStat(NANOSECONDS); + private final TimeStat parameterBindingTime = new TimeStat(NANOSECONDS); + private final AtomicLong lastCatalogGeneration = new AtomicLong(-1); + + @Override + public void compilationCompleted(HogQlCompilationEvent event) + { + completedCompilations.update(1); + totalTime.addNanos(event.totalNanos()); + event.dimensions().catalogGeneration().ifPresent(lastCatalogGeneration::set); + event.phaseNanos().forEach((phase, duration) -> phaseTime(phase).addNanos(duration)); + switch (event.outcome()) { + case SUCCESS -> successfulCompilations.update(1); + case USER_ERROR -> userErrorFailures.update(1); + case INTERNAL_ERROR -> internalErrorFailures.update(1); + case EXTERNAL_ERROR -> externalErrorFailures.update(1); + case INSUFFICIENT_RESOURCES -> insufficientResourcesFailures.update(1); + } + } + + private TimeStat phaseTime(Phase phase) + { + return switch (phase) { + case COMPILATION -> compilationTime; + case PARSE -> parseTime; + case BIND -> bindTime; + case LOWER -> lowerTime; + case PARAMETER_BINDING -> parameterBindingTime; + }; + } + + @Managed + @Nested + public CounterStat getCompletedCompilations() + { + return completedCompilations; + } + + @Managed + @Nested + public CounterStat getSuccessfulCompilations() + { + return successfulCompilations; + } + + @Managed + @Nested + public CounterStat getUserErrorFailures() + { + return userErrorFailures; + } + + @Managed + @Nested + public CounterStat getInternalErrorFailures() + { + return internalErrorFailures; + } + + @Managed + @Nested + public CounterStat getExternalErrorFailures() + { + return externalErrorFailures; + } + + @Managed + @Nested + public CounterStat getInsufficientResourcesFailures() + { + return insufficientResourcesFailures; + } + + @Managed + public long getLastCatalogGeneration() + { + return lastCatalogGeneration.get(); + } + + @Managed + @Nested + public TimeStat getTotalTime() + { + return totalTime; + } + + @Managed + @Nested + public TimeStat getCompilationTime() + { + return compilationTime; + } + + @Managed + @Nested + public TimeStat getParseTime() + { + return parseTime; + } + + @Managed + @Nested + public TimeStat getBindTime() + { + return bindTime; + } + + @Managed + @Nested + public TimeStat getLowerTime() + { + return lowerTime; + } + + @Managed + @Nested + public TimeStat getParameterBindingTime() + { + return parameterBindingTime; + } +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationTracker.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationTracker.java new file mode 100644 index 000000000000..0330694f96c3 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlCompilationTracker.java @@ -0,0 +1,132 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import io.trino.hogql.HogQlCompilationEvent.Dimensions; +import io.trino.hogql.HogQlCompilationEvent.Outcome; +import io.trino.hogql.HogQlCompilationEvent.Phase; +import io.trino.spi.ErrorType; +import io.trino.spi.TrinoException; + +import java.util.EnumMap; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.function.LongSupplier; +import java.util.function.Supplier; + +import static io.trino.hogql.HogQlCompilationEvent.Outcome.EXTERNAL_ERROR; +import static io.trino.hogql.HogQlCompilationEvent.Outcome.INSUFFICIENT_RESOURCES; +import static io.trino.hogql.HogQlCompilationEvent.Outcome.INTERNAL_ERROR; +import static io.trino.hogql.HogQlCompilationEvent.Outcome.SUCCESS; +import static io.trino.hogql.HogQlCompilationEvent.Outcome.USER_ERROR; +import static java.lang.Math.max; +import static java.util.Objects.requireNonNull; + +public final class HogQlCompilationTracker +{ + private final HogQlCompilationObserver observer; + private Dimensions dimensions; + private final LongSupplier ticker; + private final long startedNanos; + private final EnumMap phaseNanos = new EnumMap<>(Phase.class); + + private Phase failedPhase; + private boolean completed; + + public HogQlCompilationTracker(HogQlCompilationObserver observer, Dimensions dimensions) + { + this(observer, dimensions, System::nanoTime); + } + + HogQlCompilationTracker(HogQlCompilationObserver observer, Dimensions dimensions, LongSupplier ticker) + { + this.observer = requireNonNull(observer, "observer is null"); + this.dimensions = requireNonNull(dimensions, "dimensions is null"); + this.ticker = requireNonNull(ticker, "ticker is null"); + startedNanos = ticker.getAsLong(); + } + + public T observe(Phase phase, Supplier operation) + { + requireNonNull(phase, "phase is null"); + requireNonNull(operation, "operation is null"); + if (completed) { + throw new IllegalStateException("HogQL compilation observation is complete"); + } + long phaseStartedNanos = ticker.getAsLong(); + try { + return operation.get(); + } + catch (RuntimeException | Error failure) { + failedPhase = phase; + throw failure; + } + finally { + phaseNanos.merge(phase, elapsedNanos(phaseStartedNanos, ticker.getAsLong()), Long::sum); + } + } + + public void succeeded() + { + complete(SUCCESS, Optional.empty()); + } + + public void catalogGeneration(OptionalLong catalogGeneration) + { + if (completed) { + throw new IllegalStateException("HogQL compilation observation is complete"); + } + dimensions = dimensions.withCatalogGeneration(requireNonNull(catalogGeneration, "catalogGeneration is null")); + } + + public void failed(Throwable failure) + { + requireNonNull(failure, "failure is null"); + complete(outcome(failure), Optional.ofNullable(failedPhase)); + } + + private void complete(Outcome outcome, Optional failedPhase) + { + if (completed) { + throw new IllegalStateException("HogQL compilation observation is complete"); + } + completed = true; + observer.compilationCompleted(new HogQlCompilationEvent( + dimensions, + outcome, + failedPhase, + elapsedNanos(startedNanos, ticker.getAsLong()), + Map.copyOf(phaseNanos))); + } + + private static Outcome outcome(Throwable failure) + { + if (!(failure instanceof TrinoException trinoException)) { + return INTERNAL_ERROR; + } + ErrorType errorType = trinoException.getErrorCode().getType(); + return switch (errorType) { + case USER_ERROR -> USER_ERROR; + case INTERNAL_ERROR -> INTERNAL_ERROR; + case EXTERNAL -> EXTERNAL_ERROR; + case INSUFFICIENT_RESOURCES -> INSUFFICIENT_RESOURCES; + }; + } + + private static long elapsedNanos(long start, long end) + { + return max(0, end - start); + } +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlConfig.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlConfig.java new file mode 100644 index 000000000000..bba91b55c345 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlConfig.java @@ -0,0 +1,87 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import io.airlift.configuration.Config; +import io.airlift.configuration.ConfigDescription; +import io.airlift.units.Duration; +import io.airlift.units.MinDuration; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotNull; + +import static java.util.concurrent.TimeUnit.SECONDS; + +public class HogQlConfig +{ + private boolean enabled; + private int compilationThreads = 2; + private int compilationQueueCapacity = 32; + private Duration compilationTimeout = new Duration(10, SECONDS); + + public boolean isEnabled() + { + return enabled; + } + + @Config("hogql.enabled") + @ConfigDescription("Enable the native HogQL query submission endpoint") + public HogQlConfig setEnabled(boolean enabled) + { + this.enabled = enabled; + return this; + } + + @Min(1) + public int getCompilationThreads() + { + return compilationThreads; + } + + @Config("hogql.compilation-threads") + @ConfigDescription("Number of coordinator threads dedicated to HogQL compilation") + public HogQlConfig setCompilationThreads(int compilationThreads) + { + this.compilationThreads = compilationThreads; + return this; + } + + @Min(0) + public int getCompilationQueueCapacity() + { + return compilationQueueCapacity; + } + + @Config("hogql.compilation-queue-capacity") + @ConfigDescription("Maximum number of HogQL compilations waiting for a compiler thread") + public HogQlConfig setCompilationQueueCapacity(int compilationQueueCapacity) + { + this.compilationQueueCapacity = compilationQueueCapacity; + return this; + } + + @NotNull + @MinDuration("1ms") + public Duration getCompilationTimeout() + { + return compilationTimeout; + } + + @Config("hogql.compilation-timeout") + @ConfigDescription("Maximum wall time for one HogQL compilation, including queue wait") + public HogQlConfig setCompilationTimeout(Duration compilationTimeout) + { + this.compilationTimeout = compilationTimeout; + return this; + } +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlCoordinatorErrorCode.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlCoordinatorErrorCode.java new file mode 100644 index 000000000000..346a8e96b31a --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlCoordinatorErrorCode.java @@ -0,0 +1,41 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import io.trino.spi.ErrorCode; +import io.trino.spi.ErrorCodeSupplier; +import io.trino.spi.ErrorType; + +import static io.trino.spi.ErrorType.INSUFFICIENT_RESOURCES; + +public enum HogQlCoordinatorErrorCode + implements ErrorCodeSupplier +{ + HOGQL_COMPILATION_QUEUE_FULL(0, INSUFFICIENT_RESOURCES), + HOGQL_COMPILATION_TIMEOUT(1, INSUFFICIENT_RESOURCES), + ; + + private final ErrorCode errorCode; + + HogQlCoordinatorErrorCode(int code, ErrorType type) + { + errorCode = new ErrorCode(code + 0x0522_0000, name(), type); + } + + @Override + public ErrorCode toErrorCode() + { + return errorCode; + } +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlEmptyFunctions.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlEmptyFunctions.java new file mode 100644 index 000000000000..d134af00a5d6 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlEmptyFunctions.java @@ -0,0 +1,55 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import io.airlift.slice.Slice; +import io.trino.spi.block.Block; +import io.trino.spi.block.SqlMap; +import io.trino.spi.function.ScalarFunction; +import io.trino.spi.function.SqlType; +import io.trino.spi.function.TypeParameter; + +import static io.trino.spi.type.StandardTypes.BOOLEAN; +import static io.trino.spi.type.StandardTypes.VARCHAR; + +public final class HogQlEmptyFunctions +{ + public static final String NAME = "hogql_empty"; + + private HogQlEmptyFunctions() {} + + @ScalarFunction(value = NAME, hidden = true, neverFails = true) + @SqlType(BOOLEAN) + public static boolean varcharIsEmpty(@SqlType(VARCHAR) Slice value) + { + return value.length() == 0; + } + + @ScalarFunction(value = NAME, hidden = true, neverFails = true) + @TypeParameter("E") + @SqlType(BOOLEAN) + public static boolean arrayIsEmpty(@SqlType("array(E)") Block value) + { + return value.getPositionCount() == 0; + } + + @ScalarFunction(value = NAME, hidden = true, neverFails = true) + @TypeParameter("K") + @TypeParameter("V") + @SqlType(BOOLEAN) + public static boolean mapIsEmpty(@SqlType("map(K,V)") SqlMap value) + { + return value.getSize() == 0; + } +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateConversionEngine.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateConversionEngine.java new file mode 100644 index 000000000000..24b74bb19202 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateConversionEngine.java @@ -0,0 +1,96 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import com.google.common.collect.ImmutableMap; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshot; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshot.ExchangeRate; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.math.RoundingMode; +import java.time.LocalDate; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.NavigableMap; +import java.util.Optional; +import java.util.TreeMap; + +import static io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshot.DECIMAL_SCALE; +import static java.util.Objects.requireNonNull; + +public final class HogQlExchangeRateConversionEngine +{ + private static final BigDecimal ZERO = BigDecimal.ZERO.setScale(DECIMAL_SCALE); + + private final long generation; + private final Map> ratesByCurrency; + + public HogQlExchangeRateConversionEngine(HogQlExchangeRateSnapshot snapshot) + { + requireNonNull(snapshot, "snapshot is null"); + generation = snapshot.generation(); + + Map> mutableRates = new HashMap<>(); + for (ExchangeRate rate : snapshot.rates()) { + mutableRates.computeIfAbsent(rate.currency(), _ -> new TreeMap<>()) + .put(LocalDate.parse(rate.effectiveDate()), new BigDecimal(new BigInteger(rate.unscaledRate()), DECIMAL_SCALE)); + } + ImmutableMap.Builder> immutableRates = ImmutableMap.builder(); + mutableRates.forEach((currency, rates) -> immutableRates.put(currency, Collections.unmodifiableNavigableMap(rates))); + ratesByCurrency = immutableRates.buildOrThrow(); + } + + public long generation() + { + return generation; + } + + public Optional rate(String currency, LocalDate date) + { + requireNonNull(currency, "currency is null"); + requireNonNull(date, "date is null"); + NavigableMap currencyRates = ratesByCurrency.get(currency); + if (currencyRates == null) { + return Optional.empty(); + } + Map.Entry rate = currencyRates.floorEntry(date); + return rate == null ? Optional.empty() : Optional.of(rate.getValue()); + } + + public BigDecimal convert(String sourceCurrency, String targetCurrency, BigDecimal amount, LocalDate date) + { + requireNonNull(sourceCurrency, "sourceCurrency is null"); + requireNonNull(targetCurrency, "targetCurrency is null"); + requireNonNull(amount, "amount is null"); + requireNonNull(date, "date is null"); + + BigDecimal decimalAmount = amount.setScale(DECIMAL_SCALE, RoundingMode.DOWN); + if (sourceCurrency.equals(targetCurrency)) { + return decimalAmount; + } + + BigDecimal sourceRate = rate(sourceCurrency, date).orElse(ZERO); + BigDecimal targetRate = rate(targetCurrency, date).orElse(ZERO); + if (sourceRate.signum() == 0 || targetRate.signum() == 0) { + return ZERO; + } + + return decimalAmount + .divide(sourceRate, DECIMAL_SCALE, RoundingMode.DOWN) + .multiply(targetRate) + .setScale(DECIMAL_SCALE, RoundingMode.DOWN); + } +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateFunction.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateFunction.java new file mode 100644 index 000000000000..7cf727080a0c --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateFunction.java @@ -0,0 +1,114 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import com.google.common.collect.ImmutableList; +import io.airlift.slice.Slice; +import io.trino.annotation.UsedByGeneratedCode; +import io.trino.metadata.SqlScalarFunction; +import io.trino.operator.scalar.ChoicesSpecializedSqlScalarFunction; +import io.trino.operator.scalar.SpecializedSqlScalarFunction; +import io.trino.spi.function.BoundSignature; +import io.trino.spi.function.FunctionMetadata; +import io.trino.spi.function.Signature; +import io.trino.spi.type.Int128; + +import java.math.BigDecimal; +import java.time.LocalDate; +import java.util.Optional; + +import static io.trino.spi.function.InvocationConvention.InvocationArgumentConvention.NEVER_NULL; +import static io.trino.spi.function.InvocationConvention.InvocationReturnConvention.FAIL_ON_NULL; +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.DateType.DATE; +import static io.trino.spi.type.DecimalType.createDecimalType; +import static io.trino.spi.type.Decimals.encodeScaledValue; +import static io.trino.spi.type.VarcharType.VARCHAR; +import static io.trino.util.Reflection.constructorMethodHandle; +import static io.trino.util.Reflection.methodHandle; +import static java.util.Objects.requireNonNull; + +public final class HogQlExchangeRateFunction + extends SqlScalarFunction +{ + public static final String NAME = "hogql_convert_currency"; + private static final int DECIMAL_SCALE = 10; + private static final java.lang.invoke.MethodHandle METHOD_HANDLE = methodHandle( + State.class, + "convert", + long.class, + Slice.class, + Slice.class, + Int128.class, + long.class); + + private final HogQlExchangeRateManager manager; + + public HogQlExchangeRateFunction(HogQlExchangeRateManager manager) + { + super(FunctionMetadata.scalarBuilder(NAME) + .signature(Signature.builder() + .returnType(createDecimalType(38, DECIMAL_SCALE)) + .argumentType(BIGINT) + .argumentType(VARCHAR) + .argumentType(VARCHAR) + .argumentType(createDecimalType(38, DECIMAL_SCALE)) + .argumentType(DATE) + .build()) + .hidden() + .description("Convert a decimal amount using a pinned HogQL exchange-rate generation") + .build()); + this.manager = requireNonNull(manager, "manager is null"); + } + + @Override + protected SpecializedSqlScalarFunction specialize(BoundSignature boundSignature) + { + java.lang.invoke.MethodHandle instanceFactory = constructorMethodHandle(State.class, HogQlExchangeRateManager.class).bindTo(manager); + return new ChoicesSpecializedSqlScalarFunction( + boundSignature, + FAIL_ON_NULL, + ImmutableList.of(NEVER_NULL, NEVER_NULL, NEVER_NULL, NEVER_NULL, NEVER_NULL), + METHOD_HANDLE, + Optional.of(instanceFactory)); + } + + public static final class State + { + private final HogQlExchangeRateManager manager; + private long generation = -1; + private HogQlExchangeRateConversionEngine engine; + + public State(HogQlExchangeRateManager manager) + { + this.manager = requireNonNull(manager, "manager is null"); + } + + @UsedByGeneratedCode + public Int128 convert(long generation, Slice sourceCurrency, Slice targetCurrency, Int128 amount, long effectiveDate) + { + if (engine == null || this.generation != generation) { + engine = manager.engine(generation); + this.generation = generation; + } + BigDecimal decimalAmount = new BigDecimal(amount.toBigInteger(), DECIMAL_SCALE); + BigDecimal converted = engine.convert( + sourceCurrency.toStringUtf8(), + targetCurrency.toStringUtf8(), + decimalAmount, + LocalDate.ofEpochDay(effectiveDate)); + return encodeScaledValue(converted, DECIMAL_SCALE); + } + } +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateHttpTransport.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateHttpTransport.java new file mode 100644 index 000000000000..187bb73603f0 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateHttpTransport.java @@ -0,0 +1,222 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import com.google.common.net.MediaType; +import com.google.inject.Inject; +import io.airlift.http.client.HttpClient; +import io.airlift.http.client.Request; +import io.airlift.http.client.Response; +import io.airlift.http.client.ResponseHandler; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateException; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateException.Failure; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotJsonDecoder; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotLoader.JsonTransport; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotLoader.LoadRequest; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.function.Supplier; + +import static com.google.common.net.MediaType.JSON_UTF_8; +import static io.airlift.concurrent.MoreFutures.toCompletableFuture; +import static io.airlift.http.client.HeaderNames.ACCEPT; +import static io.airlift.http.client.HeaderNames.CONTENT_TYPE; +import static io.airlift.http.client.HttpUriBuilder.uriBuilderFrom; +import static io.airlift.http.client.Request.Builder.prepareGet; +import static java.util.Locale.ENGLISH; +import static java.util.Objects.requireNonNull; + +public final class HogQlExchangeRateHttpTransport + implements JsonTransport +{ + private static final String METADATA_PATH = "v1/hogql/compatibility/exchange-rates"; + private static final String AUTHENTICATION_HEADER = "X-Duckgres-Internal-Secret"; + private static final int MAXIMUM_AUTHENTICATION_TOKEN_BYTES = 4096; + private static final MediaType JSON = JSON_UTF_8.withoutParameters(); + + private final URI baseUri; + private final HttpClient httpClient; + private final int maximumResponseBytes; + private final Supplier authenticationTokenSupplier; + + @Inject + public HogQlExchangeRateHttpTransport( + HogQlSemanticCatalogConfig config, + @ForHogQlExchangeRate HttpClient httpClient) + { + this(requireNonNull(config, "config is null").getUri(), + httpClient, + Math.toIntExact(config.getMaximumResponseSize().toBytes()), + tokenSupplier(config.getAuthenticationTokenFile())); + } + + HogQlExchangeRateHttpTransport(URI baseUri, HttpClient httpClient, Supplier authenticationTokenSupplier) + { + this(baseUri, httpClient, HogQlExchangeRateSnapshotJsonDecoder.MAXIMUM_PAYLOAD_BYTES, authenticationTokenSupplier); + } + + HogQlExchangeRateHttpTransport(URI baseUri, HttpClient httpClient, int maximumResponseBytes, Supplier authenticationTokenSupplier) + { + this.baseUri = validateBaseUri(baseUri); + this.httpClient = requireNonNull(httpClient, "httpClient is null"); + if (maximumResponseBytes <= 0 || maximumResponseBytes > HogQlExchangeRateSnapshotJsonDecoder.MAXIMUM_PAYLOAD_BYTES) { + throw new IllegalArgumentException("invalid HogQL exchange-rate response size limit"); + } + this.maximumResponseBytes = maximumResponseBytes; + this.authenticationTokenSupplier = requireNonNull(authenticationTokenSupplier, "authenticationTokenSupplier is null"); + } + + @Override + public CompletionStage load(LoadRequest request) + { + requireNonNull(request, "request is null"); + String authenticationToken; + try { + authenticationToken = validateAuthenticationToken(authenticationTokenSupplier.get()); + } + catch (RuntimeException _) { + return CompletableFuture.failedFuture(unavailable()); + } + Request httpRequest = prepareGet() + .setUri(buildUri(baseUri, request)) + .setHeader(ACCEPT, JSON.toString()) + .setHeader(AUTHENTICATION_HEADER, authenticationToken) + .build(); + return toCompletableFuture(httpClient.executeAsync(httpRequest, new MetadataResponseHandler(request, maximumResponseBytes))); + } + + static URI buildUri(URI baseUri, LoadRequest request) + { + var uriBuilder = uriBuilderFrom(validateBaseUri(baseUri)) + .appendPath(METADATA_PATH) + .addParameter("protocolVersion", Integer.toString(1)); + request.expectedGeneration().ifPresent(generation -> uriBuilder.addParameter("generation", Long.toString(generation))); + return uriBuilder.build(); + } + + private static Supplier tokenSupplier(String authenticationTokenFile) + { + if (authenticationTokenFile == null || authenticationTokenFile.isBlank()) { + throw new IllegalArgumentException("HogQL exchange-rate authentication token file is not configured"); + } + Path path = Path.of(authenticationTokenFile); + return () -> { + try (InputStream input = Files.newInputStream(path)) { + byte[] token = input.readNBytes(MAXIMUM_AUTHENTICATION_TOKEN_BYTES + 1); + if (token.length > MAXIMUM_AUTHENTICATION_TOKEN_BYTES) { + throw new IllegalArgumentException("invalid HogQL exchange-rate authentication token"); + } + return new String(token, StandardCharsets.UTF_8).strip(); + } + catch (IOException e) { + throw new IllegalStateException("HogQL exchange-rate authentication token is unavailable", e); + } + }; + } + + private static String validateAuthenticationToken(String token) + { + requireNonNull(token, "authentication token is null"); + if (token.isBlank() || token.indexOf('\r') >= 0 || token.indexOf('\n') >= 0) { + throw new IllegalArgumentException("invalid HogQL exchange-rate authentication token"); + } + return token; + } + + private static URI validateBaseUri(URI baseUri) + { + requireNonNull(baseUri, "HogQL exchange-rate URI is null"); + String scheme = baseUri.getScheme(); + if (scheme == null || !(scheme.toLowerCase(ENGLISH).equals("http") || scheme.toLowerCase(ENGLISH).equals("https")) || + baseUri.getHost() == null || baseUri.getUserInfo() != null || baseUri.getQuery() != null || baseUri.getFragment() != null) { + throw new IllegalArgumentException("invalid HogQL exchange-rate URI"); + } + return baseUri; + } + + private static HogQlExchangeRateException unavailable() + { + return new HogQlExchangeRateException(Failure.UNAVAILABLE, "HogQL exchange-rate metadata is unavailable"); + } + + private static final class MetadataResponseHandler + implements ResponseHandler + { + private final LoadRequest loadRequest; + private final int maximumResponseBytes; + + private MetadataResponseHandler(LoadRequest loadRequest, int maximumResponseBytes) + { + this.loadRequest = loadRequest; + this.maximumResponseBytes = maximumResponseBytes; + } + + @Override + public byte[] handleException(Request request, Exception exception) + { + throw unavailable(); + } + + @Override + public byte[] handle(Request request, Response response) + { + int statusCode = response.getStatusCode(); + if (statusCode < 200 || statusCode >= 300) { + throw statusFailure(statusCode, loadRequest); + } + if (!isJson(response)) { + throw unavailable(); + } + try (InputStream input = response.getInputStream()) { + byte[] payload = input.readNBytes(maximumResponseBytes + 1); + if (payload.length > maximumResponseBytes) { + throw unavailable(); + } + return payload; + } + catch (IOException e) { + throw unavailable(); + } + } + + private static boolean isJson(Response response) + { + return response.getHeader(CONTENT_TYPE) + .map(value -> { + try { + return MediaType.parse(value).withoutParameters().equals(JSON); + } + catch (IllegalArgumentException _) { + return false; + } + }) + .orElse(false); + } + + private static HogQlExchangeRateException statusFailure(int statusCode, LoadRequest loadRequest) + { + if (statusCode == 409 || (statusCode == 404 && loadRequest.expectedGeneration().isPresent())) { + return new HogQlExchangeRateException(Failure.GENERATION_MISMATCH, "HogQL exchange-rate generation is unavailable"); + } + return unavailable(); + } + } +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateManager.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateManager.java new file mode 100644 index 000000000000..5ceb580243bd --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateManager.java @@ -0,0 +1,151 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import com.google.inject.Inject; +import io.trino.hogql.compiler.catalog.BoundedAsyncHogQlExchangeRateSnapshotCache; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateException; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateException.Failure; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshot; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotJsonDecoder; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotLoader; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotLoader.LoadRequest; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotProvider; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotProvider.PinnedSnapshot; +import jakarta.annotation.PreDestroy; + +import java.time.Duration; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +import static io.airlift.concurrent.Threads.daemonThreadsNamed; +import static java.util.Objects.requireNonNull; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +public final class HogQlExchangeRateManager + implements HogQlExchangeRateSnapshotProvider +{ + private final Object engineLock = new Object(); + private final int maximumEngines; + private final Duration requestTimeout; + private final ThreadPoolExecutor loaderExecutor; + private final BoundedAsyncHogQlExchangeRateSnapshotCache cache; + private final LinkedHashMap engines = new LinkedHashMap<>(16, 0.75f, true); + + @Inject + public HogQlExchangeRateManager( + HogQlSemanticCatalogConfig config, + HogQlExchangeRateHttpTransport transport, + HogQlExchangeRateSnapshotJsonDecoder decoder) + { + requireNonNull(config, "config is null"); + maximumEngines = config.getMaximumEntries(); + requestTimeout = config.getRequestTimeout().toJavaTime(); + loaderExecutor = new ThreadPoolExecutor( + config.getLoaderThreads(), + config.getLoaderThreads(), + 0, + MILLISECONDS, + new ArrayBlockingQueue<>(config.getLoaderQueueCapacity()), + daemonThreadsNamed("hogql-exchange-rate-loader-%s"), + new ThreadPoolExecutor.AbortPolicy()); + HogQlExchangeRateSnapshotLoader loader = HogQlExchangeRateSnapshotLoader.fromJsonTransport(transport, decoder); + cache = new BoundedAsyncHogQlExchangeRateSnapshotCache( + config.getMaximumEntries(), + config.getRefreshAfter().toJavaTime(), + config.getExpireAfter().toJavaTime(), + config.getFailureBackoff().toJavaTime(), + System::nanoTime, + loaderExecutor, + expectedGeneration -> loader.load(expectedGeneration.isPresent() + ? LoadRequest.pinned(expectedGeneration.orElseThrow()) + : LoadRequest.latest())); + cache.currentSnapshot(OptionalLong.empty()); + } + + @Override + public PinnedSnapshot pin(OptionalLong expectedGeneration) + { + return new PinnedSnapshot(loadSnapshot(expectedGeneration)); + } + + public HogQlExchangeRateConversionEngine engine(long generation) + { + synchronized (engineLock) { + HogQlExchangeRateConversionEngine engine = engines.get(generation); + if (engine != null) { + return engine; + } + } + + HogQlExchangeRateSnapshot snapshot = loadSnapshot(OptionalLong.of(generation)); + HogQlExchangeRateConversionEngine created = new HogQlExchangeRateConversionEngine(snapshot); + synchronized (engineLock) { + HogQlExchangeRateConversionEngine existing = engines.get(generation); + if (existing != null) { + return existing; + } + engines.put(generation, created); + while (engines.size() > maximumEngines) { + Long oldestGeneration = engines.keySet().iterator().next(); + engines.remove(oldestGeneration); + } + return created; + } + } + + private HogQlExchangeRateSnapshot loadSnapshot(OptionalLong expectedGeneration) + { + Optional current = cache.currentSnapshot(expectedGeneration); + if (current.isPresent()) { + return current.orElseThrow(); + } + try { + return cache.prewarm(expectedGeneration) + .toCompletableFuture() + .get(requestTimeout.toMillis(), TimeUnit.MILLISECONDS); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw unavailable(e); + } + catch (ExecutionException e) { + if (e.getCause() instanceof HogQlExchangeRateException exchangeRateException) { + throw exchangeRateException; + } + throw unavailable(e.getCause()); + } + catch (TimeoutException | RuntimeException e) { + throw unavailable(e); + } + } + + private static HogQlExchangeRateException unavailable(Throwable cause) + { + return new HogQlExchangeRateException(Failure.UNAVAILABLE, "HogQL exchange-rate snapshot is unavailable", cause); + } + + @PreDestroy + public void shutdown() + { + loaderExecutor.shutdownNow(); + } +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateModule.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateModule.java new file mode 100644 index 000000000000..e1bd0af7e281 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlExchangeRateModule.java @@ -0,0 +1,63 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import com.google.inject.Binder; +import com.google.inject.Provides; +import com.google.inject.Scopes; +import com.google.inject.Singleton; +import com.google.inject.multibindings.ProvidesIntoSet; +import com.google.inject.multibindings.OptionalBinder; +import io.airlift.configuration.AbstractConfigurationAwareModule; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotJsonDecoder; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotProvider; +import io.trino.metadata.InternalFunctionBundle; +import io.trino.spi.function.FunctionBundle; + +import static io.airlift.http.client.HttpClientBinder.httpClientBinder; + +public final class HogQlExchangeRateModule + extends AbstractConfigurationAwareModule +{ + @Override + protected void setup(Binder binder) + { + HogQlSemanticCatalogConfig config = buildConfigObject(HogQlSemanticCatalogConfig.class); + httpClientBinder(binder) + .bindHttpClient("hogql-exchange-rate", ForHogQlExchangeRate.class) + .withConfigDefaults(httpClientConfig -> httpClientConfig + .setRequestTimeout(config.getRequestTimeout()) + .setMaxResponseContentLength(config.getMaximumResponseSize()) + .setMaxRequestsQueuedPerDestination(config.getLoaderQueueCapacity())); + binder.bind(HogQlExchangeRateHttpTransport.class).in(Scopes.SINGLETON); + binder.bind(HogQlExchangeRateManager.class).in(Scopes.SINGLETON); + OptionalBinder.newOptionalBinder(binder, HogQlExchangeRateSnapshotProvider.class) + .setBinding() + .to(HogQlExchangeRateManager.class); + } + + @Provides + @Singleton + public static HogQlExchangeRateSnapshotJsonDecoder provideDecoder() + { + return new HogQlExchangeRateSnapshotJsonDecoder(); + } + + @ProvidesIntoSet + @Singleton + public static FunctionBundle provideFunctionBundle(HogQlExchangeRateManager manager) + { + return new InternalFunctionBundle(new HogQlExchangeRateFunction(manager)); + } +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlFunctionModule.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlFunctionModule.java new file mode 100644 index 000000000000..44098eccaadb --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlFunctionModule.java @@ -0,0 +1,32 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import com.google.inject.AbstractModule; +import com.google.inject.Provides; +import com.google.inject.Singleton; +import com.google.inject.multibindings.ProvidesIntoSet; +import io.trino.metadata.InternalFunctionBundle; +import io.trino.spi.function.FunctionBundle; + +public final class HogQlFunctionModule + extends AbstractModule +{ + @ProvidesIntoSet + @Singleton + public static FunctionBundle provideFunctionBundle() + { + return InternalFunctionBundle.extractFunctions(HogQlEmptyFunctions.class); + } +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlPhysicalCatalog.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlPhysicalCatalog.java new file mode 100644 index 000000000000..87e61acbde63 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlPhysicalCatalog.java @@ -0,0 +1,94 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import com.google.common.collect.ImmutableList; + +import java.util.List; +import java.util.Locale; +import java.util.regex.Pattern; + +import static java.util.Objects.requireNonNull; + +public record HogQlPhysicalCatalog( + int protocolVersion, + int schemaVersion, + Identifier catalog, + String catalogHandleVersion, + List
tables) +{ + public static final int PROTOCOL_VERSION = 1; + public static final int SCHEMA_VERSION = 1; + + private static final Pattern UNDELIMITED_IDENTIFIER = Pattern.compile("[A-Za-z_][A-Za-z0-9_]*"); + + public HogQlPhysicalCatalog + { + if (protocolVersion != PROTOCOL_VERSION) { + throw new IllegalArgumentException("unsupported protocolVersion"); + } + if (schemaVersion != SCHEMA_VERSION) { + throw new IllegalArgumentException("unsupported schemaVersion"); + } + requireNonNull(catalog, "catalog is null"); + requireNonNull(catalogHandleVersion, "catalogHandleVersion is null"); + tables = ImmutableList.copyOf(requireNonNull(tables, "tables is null")); + } + + public static Identifier identifier(String value) + { + requireNonNull(value, "value is null"); + boolean delimited = !UNDELIMITED_IDENTIFIER.matcher(value).matches() || !value.equals(value.toLowerCase(Locale.ENGLISH)); + return new Identifier(value, delimited); + } + + public record Identifier(String value, boolean delimited) + { + public Identifier + { + requireNonNull(value, "value is null"); + } + } + + public record Table(Identifier schema, Identifier table, List columns) + { + public Table + { + requireNonNull(schema, "schema is null"); + requireNonNull(table, "table is null"); + columns = ImmutableList.copyOf(requireNonNull(columns, "columns is null")); + } + } + + public record Column( + Identifier name, + int ordinal, + String type, + boolean nullable, + boolean hidden, + boolean starVisible) + { + public Column + { + requireNonNull(name, "name is null"); + requireNonNull(type, "type is null"); + if (ordinal < 1) { + throw new IllegalArgumentException("ordinal must be positive"); + } + if (hidden == starVisible) { + throw new IllegalArgumentException("hidden and starVisible must be opposites"); + } + } + } +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlPhysicalCatalogProvider.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlPhysicalCatalogProvider.java new file mode 100644 index 000000000000..21dc6b2f014a --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlPhysicalCatalogProvider.java @@ -0,0 +1,263 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import com.google.common.collect.ImmutableList; +import com.google.inject.Inject; +import io.trino.Session; +import io.trino.connector.CatalogHandle; +import io.trino.hogql.HogQlPhysicalCatalog.Column; +import io.trino.hogql.HogQlPhysicalCatalog.Table; +import io.trino.metadata.Catalog; +import io.trino.metadata.CatalogManager; +import io.trino.metadata.Metadata; +import io.trino.metadata.QualifiedTablePrefix; +import io.trino.security.AccessControl; +import io.trino.spi.catalog.CatalogName; +import io.trino.spi.connector.ColumnMetadata; +import io.trino.spi.connector.SchemaTableName; +import io.trino.spi.connector.TableColumnsMetadata; +import io.trino.transaction.TransactionId; +import io.trino.transaction.TransactionManager; +import jakarta.ws.rs.NotFoundException; +import jakarta.ws.rs.WebApplicationException; + +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.collect.ImmutableSet.toImmutableSet; +import static io.airlift.concurrent.MoreFutures.getFutureValue; +import static io.trino.hogql.HogQlPhysicalCatalog.identifier; +import static io.trino.metadata.MetadataListing.handleListingException; +import static io.trino.spi.transaction.IsolationLevel.READ_UNCOMMITTED; +import static jakarta.ws.rs.core.Response.Status.CONFLICT; +import static jakarta.ws.rs.core.Response.Status.REQUEST_ENTITY_TOO_LARGE; +import static java.util.Objects.requireNonNull; + +public class HogQlPhysicalCatalogProvider +{ + static final int MAX_TABLES = 10_000; + static final int MAX_COLUMNS = 100_000; + static final int MAX_COLUMNS_PER_TABLE = 10_000; + static final int MAX_VALUE_CHARACTERS = 8 * 1024 * 1024; + + private final Metadata metadata; + private final AccessControl accessControl; + private final TransactionManager transactionManager; + private final CatalogManager catalogManager; + + @Inject + public HogQlPhysicalCatalogProvider( + Metadata metadata, + AccessControl accessControl, + TransactionManager transactionManager, + CatalogManager catalogManager) + { + this.metadata = requireNonNull(metadata, "metadata is null"); + this.accessControl = requireNonNull(accessControl, "accessControl is null"); + this.transactionManager = requireNonNull(transactionManager, "transactionManager is null"); + this.catalogManager = requireNonNull(catalogManager, "catalogManager is null"); + } + + public HogQlPhysicalCatalog load(Session session, String catalogName) + { + requireNonNull(session, "session is null"); + CatalogName requestedCatalog = new CatalogName(requireNonNull(catalogName, "catalogName is null")); + CatalogHandle expectedHandle = currentCatalogHandle(requestedCatalog); + + TransactionId transactionId = transactionManager.beginTransaction(READ_UNCOMMITTED, true, true); + boolean success = false; + try { + Session transactionSession = session.beginTransactionId(transactionId, transactionManager, accessControl); + metadata.beginQuery(transactionSession); + try { + HogQlPhysicalCatalog result = load(transactionSession, catalogName, requestedCatalog, expectedHandle); + success = true; + return result; + } + finally { + metadata.cleanupQuery(transactionSession); + } + } + finally { + if (transactionManager.transactionExists(transactionId)) { + if (success) { + getFutureValue(transactionManager.asyncCommit(transactionId)); + } + else { + getFutureValue(transactionManager.asyncAbort(transactionId)); + } + } + } + } + + private HogQlPhysicalCatalog load( + Session transactionSession, + String catalogName, + CatalogName requestedCatalog, + CatalogHandle expectedHandle) + { + CatalogHandle transactionHandle = metadata.getCatalogHandle(transactionSession, catalogName) + .orElseThrow(NotFoundException::new); + verifyCatalogUnchanged(requestedCatalog, expectedHandle, transactionHandle); + verifyCatalogVisible(transactionSession, catalogName); + + Map> tableColumns = listPhysicalTableColumns(transactionSession, catalogName); + HogQlPhysicalCatalog result = createCatalog(catalogName, expectedHandle, tableColumns); + verifyCatalogUnchanged(requestedCatalog, expectedHandle, currentCatalogHandle(requestedCatalog)); + return result; + } + + private void verifyCatalogVisible(Session session, String catalogName) + { + Set visibleCatalogs = accessControl.filterCatalogs(session.toSecurityContext(), Set.of(catalogName)); + if (!visibleCatalogs.contains(catalogName)) { + throw new NotFoundException(); + } + } + + private CatalogHandle currentCatalogHandle(CatalogName catalogName) + { + return catalogManager.getCatalog(catalogName) + .filter(catalog -> !catalog.isFailed()) + .map(Catalog::getCatalogHandle) + .orElseThrow(NotFoundException::new); + } + + private static void verifyCatalogUnchanged(CatalogName catalogName, CatalogHandle expected, CatalogHandle actual) + { + if (!expected.equals(actual)) { + throw new WebApplicationException("Catalog '%s' changed while its physical inventory was read".formatted(catalogName), CONFLICT); + } + } + + private Map> listPhysicalTableColumns(Session session, String catalogName) + { + QualifiedTablePrefix prefix = new QualifiedTablePrefix(catalogName); + AtomicInteger filteredCount = new AtomicInteger(); + List tables; + try { + tables = metadata.listTableColumns( + session, + prefix, + relationNames -> { + Set filtered = accessControl.filterTables(session.toSecurityContext(), catalogName, relationNames); + filteredCount.addAndGet(filtered.size()); + return filtered; + }); + } + catch (RuntimeException exception) { + throw handleListingException(exception, "table columns", catalogName); + } + checkState(filteredCount.get() >= tables.size(), "relation filter was not applied to every returned relation"); + + Map> columnsByTable = tables.stream() + .filter(table -> !table.getTable().getSchemaName().equals("information_schema")) + .collect(toImmutableMap( + TableColumnsMetadata::getTable, + table -> table.getColumns().orElseThrow(() -> new WebApplicationException( + "Redirected tables are not supported by the physical catalog endpoint", + CONFLICT)))); + Map> allowedColumns = accessControl.filterColumns( + session.toSecurityContext(), + catalogName, + columnsByTable.entrySet().stream() + .collect(toImmutableMap( + Map.Entry::getKey, + entry -> entry.getValue().stream() + .map(ColumnMetadata::getName) + .collect(toImmutableSet())))); + + Map> result = new HashMap<>(); + columnsByTable.forEach((table, columns) -> { + Set visible = allowedColumns.getOrDefault(table, Set.of()); + ImmutableList.Builder physicalColumns = ImmutableList.builder(); + for (int index = 0; index < columns.size(); index++) { + ColumnMetadata column = columns.get(index); + if (visible.contains(column.getName())) { + physicalColumns.add(new PhysicalColumn(column, index + 1)); + } + } + result.put(table, physicalColumns.build()); + }); + return Map.copyOf(result); + } + + private static HogQlPhysicalCatalog createCatalog( + String catalogName, + CatalogHandle catalogHandle, + Map> tableColumns) + { + if (tableColumns.size() > MAX_TABLES) { + throw tooLarge(); + } + + ImmutableList.Builder
tables = ImmutableList.builderWithExpectedSize(tableColumns.size()); + int totalColumns = 0; + int valueCharacters = catalogName.length() + catalogHandle.getVersion().toString().length(); + + List>> sortedTables = tableColumns.entrySet().stream() + .sorted(Map.Entry.comparingByKey(Comparator.comparing(SchemaTableName::getSchemaName) + .thenComparing(SchemaTableName::getTableName))) + .toList(); + for (Map.Entry> entry : sortedTables) { + SchemaTableName tableName = entry.getKey(); + List columnMetadata = entry.getValue(); + if (columnMetadata.size() > MAX_COLUMNS_PER_TABLE || totalColumns + columnMetadata.size() > MAX_COLUMNS) { + throw tooLarge(); + } + + ImmutableList.Builder columns = ImmutableList.builderWithExpectedSize(columnMetadata.size()); + for (int index = 0; index < columnMetadata.size(); index++) { + PhysicalColumn physicalColumn = columnMetadata.get(index); + ColumnMetadata column = physicalColumn.metadata(); + String typeSignature = column.getType().getTypeId().getId(); + valueCharacters += column.getName().length() + typeSignature.length(); + columns.add(new Column( + identifier(column.getName()), + physicalColumn.ordinal(), + typeSignature, + column.isNullable(), + column.isHidden(), + !column.isHidden())); + } + valueCharacters += tableName.getSchemaName().length() + tableName.getTableName().length(); + if (valueCharacters > MAX_VALUE_CHARACTERS) { + throw tooLarge(); + } + totalColumns += columnMetadata.size(); + tables.add(new Table(identifier(tableName.getSchemaName()), identifier(tableName.getTableName()), columns.build())); + } + + return new HogQlPhysicalCatalog( + HogQlPhysicalCatalog.PROTOCOL_VERSION, + HogQlPhysicalCatalog.SCHEMA_VERSION, + identifier(catalogName), + catalogHandle.getVersion().toString(), + tables.build()); + } + + private static WebApplicationException tooLarge() + { + return new WebApplicationException("Physical catalog inventory exceeds the compatibility endpoint limit", REQUEST_ENTITY_TOO_LARGE); + } + + private record PhysicalColumn(ColumnMetadata metadata, int ordinal) {} +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogConfig.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogConfig.java new file mode 100644 index 000000000000..639db487dce4 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogConfig.java @@ -0,0 +1,187 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import io.airlift.configuration.Config; +import io.airlift.configuration.ConfigDescription; +import io.airlift.units.DataSize; +import io.airlift.units.Duration; +import io.airlift.units.MaxDataSize; +import io.airlift.units.MinDataSize; +import io.airlift.units.MinDuration; +import jakarta.validation.constraints.AssertTrue; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotNull; + +import java.net.URI; + +import static io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotJsonDecoder.MAXIMUM_PAYLOAD_BYTES; +import static java.util.concurrent.TimeUnit.MINUTES; +import static java.util.concurrent.TimeUnit.SECONDS; + +public class HogQlSemanticCatalogConfig +{ + private URI uri; + private String authenticationTokenFile; + private int maximumEntries = 100; + private Duration refreshAfter = new Duration(1, MINUTES); + private Duration expireAfter = new Duration(5, MINUTES); + private Duration failureBackoff = new Duration(10, SECONDS); + private int loaderThreads = 4; + private int loaderQueueCapacity = 64; + private Duration requestTimeout = new Duration(10, SECONDS); + private DataSize maximumResponseSize = DataSize.ofBytes(MAXIMUM_PAYLOAD_BYTES); + + public URI getUri() + { + return uri; + } + + @Config("hogql.semantic-catalog.uri") + @ConfigDescription("Duckgres HogQL compatibility metadata base URI") + public HogQlSemanticCatalogConfig setUri(URI uri) + { + this.uri = uri; + return this; + } + + public String getAuthenticationTokenFile() + { + return authenticationTokenFile; + } + + @Config("hogql.semantic-catalog.authentication-token-file") + @ConfigDescription("File containing the Duckgres HogQL compatibility metadata authentication token") + public HogQlSemanticCatalogConfig setAuthenticationTokenFile(String authenticationTokenFile) + { + this.authenticationTokenFile = authenticationTokenFile; + return this; + } + + @AssertTrue(message = "hogql.semantic-catalog.authentication-token-file is required when hogql.semantic-catalog.uri is set") + public boolean isAuthenticationConfigured() + { + return uri == null || (authenticationTokenFile != null && !authenticationTokenFile.isBlank()); + } + + @Min(1) + public int getMaximumEntries() + { + return maximumEntries; + } + + @Config("hogql.semantic-catalog.maximum-entries") + public HogQlSemanticCatalogConfig setMaximumEntries(int maximumEntries) + { + this.maximumEntries = maximumEntries; + return this; + } + + @NotNull + @MinDuration("1ms") + public Duration getRefreshAfter() + { + return refreshAfter; + } + + @Config("hogql.semantic-catalog.refresh-after") + public HogQlSemanticCatalogConfig setRefreshAfter(Duration refreshAfter) + { + this.refreshAfter = refreshAfter; + return this; + } + + @NotNull + @MinDuration("1ms") + public Duration getExpireAfter() + { + return expireAfter; + } + + @Config("hogql.semantic-catalog.expire-after") + public HogQlSemanticCatalogConfig setExpireAfter(Duration expireAfter) + { + this.expireAfter = expireAfter; + return this; + } + + @NotNull + @MinDuration("0ms") + public Duration getFailureBackoff() + { + return failureBackoff; + } + + @Config("hogql.semantic-catalog.failure-backoff") + public HogQlSemanticCatalogConfig setFailureBackoff(Duration failureBackoff) + { + this.failureBackoff = failureBackoff; + return this; + } + + @Min(1) + public int getLoaderThreads() + { + return loaderThreads; + } + + @Config("hogql.semantic-catalog.loader-threads") + public HogQlSemanticCatalogConfig setLoaderThreads(int loaderThreads) + { + this.loaderThreads = loaderThreads; + return this; + } + + @Min(1) + public int getLoaderQueueCapacity() + { + return loaderQueueCapacity; + } + + @Config("hogql.semantic-catalog.loader-queue-capacity") + public HogQlSemanticCatalogConfig setLoaderQueueCapacity(int loaderQueueCapacity) + { + this.loaderQueueCapacity = loaderQueueCapacity; + return this; + } + + @NotNull + @MinDuration("1ms") + public Duration getRequestTimeout() + { + return requestTimeout; + } + + @Config("hogql.semantic-catalog.request-timeout") + public HogQlSemanticCatalogConfig setRequestTimeout(Duration requestTimeout) + { + this.requestTimeout = requestTimeout; + return this; + } + + @NotNull + @MinDataSize("1B") + @MaxDataSize("8MB") + public DataSize getMaximumResponseSize() + { + return maximumResponseSize; + } + + @Config("hogql.semantic-catalog.maximum-response-size") + public HogQlSemanticCatalogConfig setMaximumResponseSize(DataSize maximumResponseSize) + { + this.maximumResponseSize = maximumResponseSize; + return this; + } +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogHttpTransport.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogHttpTransport.java new file mode 100644 index 000000000000..8d30acfd34ce --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogHttpTransport.java @@ -0,0 +1,232 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import com.google.common.net.MediaType; +import com.google.inject.Inject; +import io.airlift.http.client.HttpClient; +import io.airlift.http.client.Request; +import io.airlift.http.client.Response; +import io.airlift.http.client.ResponseHandler; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogException; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogException.Failure; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotJsonDecoder; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotLoader.JsonTransport; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotLoader.LoadRequest; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.function.Supplier; + +import static com.google.common.net.MediaType.JSON_UTF_8; +import static io.airlift.concurrent.MoreFutures.toCompletableFuture; +import static io.airlift.http.client.HeaderNames.ACCEPT; +import static io.airlift.http.client.HeaderNames.CONTENT_TYPE; +import static io.airlift.http.client.HttpUriBuilder.uriBuilderFrom; +import static io.airlift.http.client.Request.Builder.prepareGet; +import static io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotJsonDecoder.PROTOCOL_VERSION; +import static java.lang.Math.toIntExact; +import static java.util.Locale.ENGLISH; +import static java.util.Objects.requireNonNull; + +public final class HogQlSemanticCatalogHttpTransport + implements JsonTransport +{ + private static final String METADATA_PATH = "v1/hogql/compatibility/semantic-catalog"; + private static final String AUTHENTICATION_HEADER = "X-Duckgres-Internal-Secret"; + private static final int MAXIMUM_AUTHENTICATION_TOKEN_BYTES = 4096; + private static final MediaType JSON = JSON_UTF_8.withoutParameters(); + + private final URI baseUri; + private final HttpClient httpClient; + private final int maximumResponseBytes; + private final Supplier authenticationTokenSupplier; + + @Inject + public HogQlSemanticCatalogHttpTransport( + HogQlSemanticCatalogConfig config, + @ForHogQlSemanticCatalog HttpClient httpClient) + { + this(requireNonNull(config, "config is null").getUri(), + httpClient, + toIntExact(config.getMaximumResponseSize().toBytes()), + tokenSupplier(config.getAuthenticationTokenFile())); + } + + HogQlSemanticCatalogHttpTransport(URI baseUri, HttpClient httpClient, Supplier authenticationTokenSupplier) + { + this(baseUri, httpClient, HogQlSemanticCatalogSnapshotJsonDecoder.MAXIMUM_PAYLOAD_BYTES, authenticationTokenSupplier); + } + + private HogQlSemanticCatalogHttpTransport(URI baseUri, HttpClient httpClient, int maximumResponseBytes, Supplier authenticationTokenSupplier) + { + this.baseUri = validateBaseUri(baseUri); + this.httpClient = requireNonNull(httpClient, "httpClient is null"); + if (maximumResponseBytes <= 0 || maximumResponseBytes > HogQlSemanticCatalogSnapshotJsonDecoder.MAXIMUM_PAYLOAD_BYTES) { + throw new IllegalArgumentException("invalid HogQL semantic catalog response size limit"); + } + this.maximumResponseBytes = maximumResponseBytes; + this.authenticationTokenSupplier = requireNonNull(authenticationTokenSupplier, "authenticationTokenSupplier is null"); + } + + @Override + public CompletionStage load(LoadRequest request) + { + requireNonNull(request, "request is null"); + String authenticationToken; + try { + authenticationToken = validateAuthenticationToken(authenticationTokenSupplier.get()); + } + catch (RuntimeException _) { + return CompletableFuture.failedFuture(unavailable()); + } + Request httpRequest = prepareGet() + .setUri(buildUri(baseUri, request)) + .setHeader(ACCEPT, JSON.toString()) + .setHeader(AUTHENTICATION_HEADER, authenticationToken) + .build(); + return toCompletableFuture(httpClient.executeAsync(httpRequest, new MetadataResponseHandler(request, maximumResponseBytes))); + } + + private static Supplier tokenSupplier(String authenticationTokenFile) + { + if (authenticationTokenFile == null || authenticationTokenFile.isBlank()) { + throw new IllegalArgumentException("HogQL semantic catalog authentication token file is not configured"); + } + Path path = Path.of(authenticationTokenFile); + return () -> { + try (InputStream input = Files.newInputStream(path)) { + byte[] token = input.readNBytes(MAXIMUM_AUTHENTICATION_TOKEN_BYTES + 1); + if (token.length > MAXIMUM_AUTHENTICATION_TOKEN_BYTES) { + throw new IllegalArgumentException("invalid HogQL semantic catalog authentication token"); + } + return new String(token, StandardCharsets.UTF_8).strip(); + } + catch (IOException e) { + throw new IllegalStateException("HogQL semantic catalog authentication token is unavailable", e); + } + }; + } + + private static String validateAuthenticationToken(String token) + { + requireNonNull(token, "authentication token is null"); + if (token.isBlank() || token.indexOf('\r') >= 0 || token.indexOf('\n') >= 0) { + throw new IllegalArgumentException("invalid HogQL semantic catalog authentication token"); + } + return token; + } + + static URI buildUri(URI baseUri, LoadRequest request) + { + var uriBuilder = uriBuilderFrom(validateBaseUri(baseUri)) + .appendPath(METADATA_PATH) + .addParameter("protocolVersion", Integer.toString(PROTOCOL_VERSION)) + .addParameter("languageVersion", request.languageVersion().toString()) + .addParameter("catalog", request.catalog().value()) + .addParameter("catalogDelimited", Boolean.toString(request.catalog().delimited())); + request.expectedGeneration().ifPresent(generation -> uriBuilder.addParameter("generation", Long.toString(generation))); + return uriBuilder.build(); + } + + private static URI validateBaseUri(URI baseUri) + { + requireNonNull(baseUri, "HogQL semantic catalog URI is null"); + String scheme = baseUri.getScheme(); + if (scheme == null || !(scheme.toLowerCase(ENGLISH).equals("http") || scheme.toLowerCase(ENGLISH).equals("https")) || + baseUri.getHost() == null || baseUri.getUserInfo() != null || baseUri.getQuery() != null || baseUri.getFragment() != null) { + throw new IllegalArgumentException("invalid HogQL semantic catalog URI"); + } + return baseUri; + } + + private static HogQlSemanticCatalogException unavailable() + { + return new HogQlSemanticCatalogException(Failure.UNAVAILABLE, "HogQL semantic catalog metadata is unavailable"); + } + + private static final class MetadataResponseHandler + implements ResponseHandler + { + private final LoadRequest loadRequest; + private final int maximumResponseBytes; + + private MetadataResponseHandler(LoadRequest loadRequest, int maximumResponseBytes) + { + this.loadRequest = loadRequest; + this.maximumResponseBytes = maximumResponseBytes; + } + + @Override + public byte[] handleException(Request request, Exception exception) + { + throw unavailable(); + } + + @Override + public byte[] handle(Request request, Response response) + { + int statusCode = response.getStatusCode(); + if (statusCode < 200 || statusCode >= 300) { + throw statusFailure(statusCode, loadRequest); + } + if (!isJson(response)) { + throw unavailable(); + } + try (InputStream input = response.getInputStream()) { + byte[] payload = input.readNBytes(maximumResponseBytes + 1); + if (payload.length > maximumResponseBytes) { + throw unavailable(); + } + return payload; + } + catch (IOException e) { + throw unavailable(); + } + } + + private static boolean isJson(Response response) + { + return response.getHeader(CONTENT_TYPE) + .map(value -> { + try { + return MediaType.parse(value).withoutParameters().equals(JSON); + } + catch (IllegalArgumentException _) { + return false; + } + }) + .orElse(false); + } + + private static HogQlSemanticCatalogException statusFailure(int statusCode, LoadRequest loadRequest) + { + if (statusCode == 409 || (statusCode == 404 && loadRequest.expectedGeneration().isPresent())) { + return new HogQlSemanticCatalogException(Failure.GENERATION_MISMATCH, "HogQL semantic catalog generation is unavailable"); + } + return unavailable(); + } + + private static HogQlSemanticCatalogException unavailable() + { + return HogQlSemanticCatalogHttpTransport.unavailable(); + } + } +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogManager.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogManager.java new file mode 100644 index 000000000000..2bbdb36491b6 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogManager.java @@ -0,0 +1,134 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import com.google.common.annotations.VisibleForTesting; +import com.google.inject.Inject; +import io.trino.hogql.compiler.catalog.BoundedAsyncHogQlSemanticCatalogSnapshotCache; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotCache; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotJsonDecoder; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotLoader; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotLoader.LoadRequest; +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.hogql.parser.HogQlLanguageVersion; +import jakarta.annotation.PreDestroy; + +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Executor; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.function.Function; +import java.util.function.LongSupplier; + +import static io.airlift.concurrent.Threads.daemonThreadsNamed; +import static java.util.Objects.requireNonNull; +import static java.util.concurrent.TimeUnit.MILLISECONDS; + +public final class HogQlSemanticCatalogManager +{ + private final ThreadPoolExecutor loaderExecutor; + private final HogQlSemanticCatalogSnapshotLoader loader; + private final BoundedAsyncHogQlSemanticCatalogSnapshotCache cache; + + @Inject + public HogQlSemanticCatalogManager( + HogQlSemanticCatalogConfig config, + HogQlSemanticCatalogHttpTransport transport, + HogQlSemanticCatalogSnapshotJsonDecoder decoder) + { + this(config, + HogQlLanguageContract.current().languageVersion(), + System::nanoTime, + jsonLoaderFactory(transport, decoder)); + } + + @VisibleForTesting + HogQlSemanticCatalogManager( + HogQlSemanticCatalogConfig config, + HogQlSemanticCatalogSnapshotLoader loader, + HogQlLanguageVersion languageVersion, + LongSupplier ticker) + { + this(config, languageVersion, ticker, _ -> loader); + } + + private HogQlSemanticCatalogManager( + HogQlSemanticCatalogConfig config, + HogQlLanguageVersion languageVersion, + LongSupplier ticker, + Function loaderFactory) + { + requireNonNull(config, "config is null"); + requireNonNull(languageVersion, "languageVersion is null"); + requireNonNull(ticker, "ticker is null"); + requireNonNull(loaderFactory, "loaderFactory is null"); + loaderExecutor = new ThreadPoolExecutor( + config.getLoaderThreads(), + config.getLoaderThreads(), + 0, + MILLISECONDS, + new ArrayBlockingQueue<>(config.getLoaderQueueCapacity()), + daemonThreadsNamed("hogql-semantic-catalog-loader-%s"), + new ThreadPoolExecutor.AbortPolicy()); + loader = requireNonNull(loaderFactory.apply(loaderExecutor), "loader is null"); + cache = new BoundedAsyncHogQlSemanticCatalogSnapshotCache( + config.getMaximumEntries(), + config.getRefreshAfter().toJavaTime(), + config.getExpireAfter().toJavaTime(), + config.getFailureBackoff().toJavaTime(), + ticker, + loaderExecutor, + (catalog, expectedGeneration) -> loader.load(expectedGeneration.isPresent() + ? LoadRequest.pinned(catalog, languageVersion, expectedGeneration.orElseThrow()) + : LoadRequest.latest(catalog, languageVersion))); + } + + public HogQlSemanticCatalogSnapshotLoader loader() + { + return loader; + } + + private static Function jsonLoaderFactory( + HogQlSemanticCatalogHttpTransport transport, + HogQlSemanticCatalogSnapshotJsonDecoder decoder) + { + requireNonNull(transport, "transport is null"); + requireNonNull(decoder, "decoder is null"); + return executor -> request -> transport.load(request).thenApplyAsync(payload -> decoder.decode(payload, request), executor); + } + + public HogQlSemanticCatalogSnapshotCache cache() + { + return cache; + } + + public CompletionStage prewarm(PhysicalIdentifier catalog) + { + return cache.prewarm(catalog); + } + + @PreDestroy + public void shutdown() + { + loaderExecutor.shutdownNow(); + } + + @VisibleForTesting + boolean isLoaderExecutorShutdown() + { + return loaderExecutor.isShutdown(); + } +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogModule.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogModule.java new file mode 100644 index 000000000000..7a986c367b88 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogModule.java @@ -0,0 +1,96 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import com.google.inject.Binder; +import com.google.inject.Inject; +import com.google.inject.Provider; +import com.google.inject.Provides; +import com.google.inject.Scopes; +import com.google.inject.Singleton; +import com.google.inject.multibindings.OptionalBinder; +import io.airlift.configuration.AbstractConfigurationAwareModule; +import io.trino.connector.CatalogLifecycleListener; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotCache; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotJsonDecoder; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotLoader; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider; + +import static com.google.inject.multibindings.Multibinder.newSetBinder; +import static io.airlift.http.client.HttpClientBinder.httpClientBinder; + +public class HogQlSemanticCatalogModule + extends AbstractConfigurationAwareModule +{ + @Override + protected void setup(Binder binder) + { + HogQlSemanticCatalogConfig config = buildConfigObject(HogQlSemanticCatalogConfig.class); + httpClientBinder(binder) + .bindHttpClient("hogql-semantic-catalog", ForHogQlSemanticCatalog.class) + .withConfigDefaults(httpClientConfig -> httpClientConfig + .setRequestTimeout(config.getRequestTimeout()) + .setMaxResponseContentLength(config.getMaximumResponseSize()) + .setMaxRequestsQueuedPerDestination(config.getLoaderQueueCapacity())); + binder.bind(HogQlSemanticCatalogHttpTransport.class).in(Scopes.SINGLETON); + binder.bind(HogQlSemanticCatalogManager.class).asEagerSingleton(); + newSetBinder(binder, CatalogLifecycleListener.class) + .addBinding() + .to(HogQlSemanticCatalogPrewarmListener.class) + .in(Scopes.SINGLETON); + OptionalBinder.newOptionalBinder(binder, HogQlSemanticCatalogSnapshotProvider.class) + .setBinding() + .toProvider(SnapshotProviderFactory.class) + .in(Scopes.SINGLETON); + } + + @Provides + @Singleton + public static HogQlSemanticCatalogSnapshotJsonDecoder provideDecoder() + { + return new HogQlSemanticCatalogSnapshotJsonDecoder(); + } + + @Provides + @Singleton + public static HogQlSemanticCatalogSnapshotLoader provideLoader(HogQlSemanticCatalogManager manager) + { + return manager.loader(); + } + + @Provides + @Singleton + public static HogQlSemanticCatalogSnapshotCache provideCache(HogQlSemanticCatalogManager manager) + { + return manager.cache(); + } + + public static final class SnapshotProviderFactory + implements Provider + { + private final HogQlSemanticCatalogSnapshotCache cache; + + @Inject + public SnapshotProviderFactory(HogQlSemanticCatalogSnapshotCache cache) + { + this.cache = cache; + } + + @Override + public HogQlSemanticCatalogSnapshotProvider get() + { + return HogQlSemanticCatalogSnapshotProvider.fromCache(cache); + } + } +} diff --git a/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogPrewarmListener.java b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogPrewarmListener.java new file mode 100644 index 000000000000..2347641418d0 --- /dev/null +++ b/core/trino-main/src/main/java/io/trino/hogql/HogQlSemanticCatalogPrewarmListener.java @@ -0,0 +1,47 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import com.google.inject.Inject; +import io.airlift.log.Logger; +import io.trino.connector.CatalogLifecycleListener; +import io.trino.spi.catalog.CatalogName; + +import static io.trino.hogql.HogQlCatalogIdentifiers.physicalCatalog; +import static java.util.Objects.requireNonNull; + +public final class HogQlSemanticCatalogPrewarmListener + implements CatalogLifecycleListener +{ + private static final Logger log = Logger.get(HogQlSemanticCatalogPrewarmListener.class); + + private final HogQlSemanticCatalogManager semanticCatalogManager; + + @Inject + public HogQlSemanticCatalogPrewarmListener(HogQlSemanticCatalogManager semanticCatalogManager) + { + this.semanticCatalogManager = requireNonNull(semanticCatalogManager, "semanticCatalogManager is null"); + } + + @Override + public void catalogLoaded(CatalogName catalogName) + { + semanticCatalogManager.prewarm(physicalCatalog(catalogName.toString())) + .whenComplete((_, failure) -> { + if (failure != null) { + log.warn(failure, "Failed to prewarm HogQL semantic catalog for %s", catalogName); + } + }); + } +} diff --git a/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java b/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java index c6e492a8f409..cdc11fcf00fc 100644 --- a/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java +++ b/core/trino-main/src/main/java/io/trino/server/CoordinatorModule.java @@ -41,6 +41,8 @@ import io.trino.dispatcher.DispatchManager; import io.trino.dispatcher.DispatchQueryFactory; import io.trino.dispatcher.FailedDispatchQueryFactory; +import io.trino.dispatcher.HogQlPhysicalCatalogResource; +import io.trino.dispatcher.HogQlStatementResource; import io.trino.dispatcher.LocalDispatchQueryFactory; import io.trino.dispatcher.QueuedStatementResource; import io.trino.event.QueryMonitor; @@ -51,6 +53,7 @@ import io.trino.execution.ExecutionFailureInfo; import io.trino.execution.ExplainAnalyzeContext; import io.trino.execution.ForQueryExecution; +import io.trino.execution.HogQlCompilationExecutor; import io.trino.execution.NodeTaskMap; import io.trino.execution.QueryExecution; import io.trino.execution.QueryExecutionMBean; @@ -97,6 +100,15 @@ import io.trino.execution.scheduler.policy.AllAtOnceExecutionPolicy; import io.trino.execution.scheduler.policy.ExecutionPolicy; import io.trino.execution.scheduler.policy.PhasedExecutionPolicy; +import io.trino.hogql.HogQlCompilationObserver; +import io.trino.hogql.HogQlCompilationStats; +import io.trino.hogql.HogQlConfig; +import io.trino.hogql.HogQlPhysicalCatalogProvider; +import io.trino.hogql.HogQlSemanticCatalogConfig; +import io.trino.hogql.HogQlSemanticCatalogModule; +import io.trino.hogql.compiler.HogQlCompiler; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotProvider; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider; import io.trino.memory.ClusterMemoryManager; import io.trino.memory.ForMemoryManager; import io.trino.memory.LeastWastedEffortTaskLowMemoryKiller; @@ -141,6 +153,7 @@ import io.trino.sql.rewrite.StatementRewrite.Rewrite; import java.util.List; +import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ScheduledExecutorService; @@ -175,6 +188,20 @@ protected void setup(Binder binder) jsonCodecBinder(binder).bindJsonCodec(TaskInfo.class); jaxrsBinder(binder).bind(QueuedStatementResource.class); jaxrsBinder(binder).bind(ExecutingStatementResource.class); + OptionalBinder.newOptionalBinder(binder, HogQlExchangeRateSnapshotProvider.class); + binder.bind(HogQlPhysicalCatalogProvider.class).in(Scopes.SINGLETON); + binder.bind(HogQlCompilationExecutor.class).in(Scopes.SINGLETON); + binder.bind(HogQlCompilationStats.class).in(Scopes.SINGLETON); + binder.bind(HogQlCompilationObserver.class).to(HogQlCompilationStats.class); + OptionalBinder.newOptionalBinder(binder, HogQlSemanticCatalogSnapshotProvider.class); + newExporter(binder).export(HogQlCompilationStats.class).withGeneratedName(); + if (buildConfigObject(HogQlConfig.class).isEnabled()) { + jaxrsBinder(binder).bind(HogQlStatementResource.class); + jaxrsBinder(binder).bind(HogQlPhysicalCatalogResource.class); + if (buildConfigObject(HogQlSemanticCatalogConfig.class).getUri() != null) { + install(new HogQlSemanticCatalogModule()); + } + } binder.bind(StatementHttpExecutionMBean.class).in(Scopes.SINGLETON); newExporter(binder).export(StatementHttpExecutionMBean.class).withGeneratedName(); binder.bind(QueryInfoUrlFactory.class).in(Scopes.SINGLETON); @@ -436,6 +463,15 @@ List getCompositeOutputDataSizeEstimatorDelegateFac closingBinder(binder).registerExecutor(Key.get(ScheduledExecutorService.class, ForScheduler.class)); } + @Provides + @Singleton + public static HogQlCompiler provideHogQlCompiler(Optional exchangeRateSnapshotProvider) + { + return exchangeRateSnapshotProvider + .map(HogQlCompiler::new) + .orElseGet(HogQlCompiler::new); + } + // working around circular dependency Metadata <-> PlannerContext private static class InitializeLanguageFunctionManager { diff --git a/core/trino-main/src/main/java/io/trino/server/ServerMainModule.java b/core/trino-main/src/main/java/io/trino/server/ServerMainModule.java index ddca3ad5a977..ba190e9a4e52 100644 --- a/core/trino-main/src/main/java/io/trino/server/ServerMainModule.java +++ b/core/trino-main/src/main/java/io/trino/server/ServerMainModule.java @@ -56,6 +56,10 @@ import io.trino.execution.executor.timesharing.MultilevelSplitQueue; import io.trino.execution.executor.timesharing.TimeSharingTaskExecutor; import io.trino.execution.scheduler.NodeSchedulerConfig; +import io.trino.hogql.HogQlConfig; +import io.trino.hogql.HogQlExchangeRateModule; +import io.trino.hogql.HogQlFunctionModule; +import io.trino.hogql.HogQlSemanticCatalogConfig; import io.trino.jsonpath.ir.IrJsonPath; import io.trino.memory.LocalMemoryManager; import io.trino.memory.LocalMemoryManagerExporter; @@ -200,6 +204,15 @@ protected void setup(Binder binder) { ServerConfig serverConfig = buildConfigObject(ServerConfig.class); + configBinder(binder).bindConfig(HogQlConfig.class); + configBinder(binder).bindConfig(HogQlSemanticCatalogConfig.class); + if (buildConfigObject(HogQlConfig.class).isEnabled()) { + install(new HogQlFunctionModule()); + } + if (buildConfigObject(HogQlSemanticCatalogConfig.class).getUri() != null) { + install(new HogQlExchangeRateModule()); + } + if (serverConfig.isCoordinator()) { install(new CoordinatorModule()); } diff --git a/core/trino-main/src/main/java/io/trino/sql/SessionPropertyResolver.java b/core/trino-main/src/main/java/io/trino/sql/SessionPropertyResolver.java index 886585a8b68e..a528c0f41506 100644 --- a/core/trino-main/src/main/java/io/trino/sql/SessionPropertyResolver.java +++ b/core/trino-main/src/main/java/io/trino/sql/SessionPropertyResolver.java @@ -14,6 +14,7 @@ package io.trino.sql; import com.google.common.collect.HashBasedTable; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Table; import com.google.inject.Inject; @@ -57,10 +58,16 @@ public SessionPropertyResolver(SessionPropertyEvaluator sessionPropertyEvaluator public SessionPropertiesApplier getSessionPropertiesApplier(PreparedQuery preparedQuery) { - if (!(preparedQuery.getStatement() instanceof Query queryStatement)) { + ImmutableList.Builder sessionProperties = ImmutableList.builder(); + if (preparedQuery.getStatement() instanceof Query queryStatement) { + sessionProperties.addAll(queryStatement.getSessionProperties()); + } + sessionProperties.addAll(preparedQuery.getSessionPropertyOverrides()); + List properties = sessionProperties.build(); + if (properties.isEmpty()) { return session -> session; } - return session -> prepareSession(session, queryStatement.getSessionProperties(), bindParameters(preparedQuery.getStatement(), preparedQuery.getParameters())); + return session -> prepareSession(session, properties, bindParameters(preparedQuery.getStatement(), preparedQuery.getParameters())); } private Session prepareSession(Session session, List sessionProperties, Map, Expression> parameters) diff --git a/core/trino-main/src/main/java/io/trino/sql/rewrite/ExplainRewrite.java b/core/trino-main/src/main/java/io/trino/sql/rewrite/ExplainRewrite.java index f09c477512f2..cc3ea775f7ca 100644 --- a/core/trino-main/src/main/java/io/trino/sql/rewrite/ExplainRewrite.java +++ b/core/trino-main/src/main/java/io/trino/sql/rewrite/ExplainRewrite.java @@ -71,7 +71,7 @@ public Statement rewrite( WarningCollector warningCollector, PlanOptimizersStatsCollector planOptimizersStatsCollector) { - return (Statement) new Visitor(session, sessionPropertyResolver, queryPreparer, queryExplainerFactory.createQueryExplainer(analyzerFactory), warningCollector, planOptimizersStatsCollector).process(node, null); + return (Statement) new Visitor(session, sessionPropertyResolver, queryPreparer, queryExplainerFactory.createQueryExplainer(analyzerFactory), parameter, warningCollector, planOptimizersStatsCollector).process(node, null); } private static final class Visitor @@ -81,6 +81,7 @@ private static final class Visitor private final SessionPropertyResolver sessionPropertyResolver; private final QueryPreparer queryPreparer; private final QueryExplainer queryExplainer; + private final List parameters; private final WarningCollector warningCollector; private final PlanOptimizersStatsCollector planOptimizersStatsCollector; @@ -89,6 +90,7 @@ public Visitor( SessionPropertyResolver sessionPropertyResolver, QueryPreparer queryPreparer, QueryExplainer queryExplainer, + List parameters, WarningCollector warningCollector, PlanOptimizersStatsCollector planOptimizersStatsCollector) { @@ -96,6 +98,7 @@ public Visitor( this.sessionPropertyResolver = requireNonNull(sessionPropertyResolver, "sessionPropertyResolver is null"); this.queryPreparer = requireNonNull(queryPreparer, "queryPreparer is null"); this.queryExplainer = requireNonNull(queryExplainer, "queryExplainer is null"); + this.parameters = List.copyOf(requireNonNull(parameters, "parameters is null")); this.warningCollector = requireNonNull(warningCollector, "warningCollector is null"); this.planOptimizersStatsCollector = planOptimizersStatsCollector; } @@ -138,7 +141,9 @@ protected Node visitExplain(Explain node, Void context) private Node getQueryPlan(Explain node, ExplainType.Type planType, ExplainFormat.Type planFormat) throws IllegalArgumentException { - PreparedQuery preparedQuery = queryPreparer.prepareQuery(session, node.getStatement()); + PreparedQuery preparedQuery = parameters.isEmpty() + ? queryPreparer.prepareQuery(session, node.getStatement()) + : queryPreparer.prepareQuery(session, node.getStatement(), parameters); Session resolvedSession = sessionPropertyResolver .getSessionPropertiesApplier(preparedQuery) .apply(session); diff --git a/core/trino-main/src/main/java/io/trino/testing/PlanTester.java b/core/trino-main/src/main/java/io/trino/testing/PlanTester.java index f4f9d6adbed9..9e62e7aabd34 100644 --- a/core/trino-main/src/main/java/io/trino/testing/PlanTester.java +++ b/core/trino-main/src/main/java/io/trino/testing/PlanTester.java @@ -36,6 +36,7 @@ import io.trino.cache.CacheManagerRegistry; import io.trino.connector.CatalogFactory; import io.trino.connector.CatalogHandle; +import io.trino.connector.CatalogLifecycleListeners; import io.trino.connector.CatalogServiceProviderModule; import io.trino.connector.ConnectorServicesProvider; import io.trino.connector.CoordinatorDynamicCatalogManager; @@ -386,7 +387,12 @@ private PlanTester(Session defaultSession, int nodeCountForStats) this.catalogFactory = catalogFactory; SecretsResolver secretsResolver = new SecretsResolver(ImmutableMap.of()); this.cacheManagerRegistry = new CacheManagerRegistry(noop(), noopTracer(), secretsResolver, new CacheManagerConfig()); - this.catalogManager = new CoordinatorDynamicCatalogManager(new InMemoryCatalogStore(), catalogFactory, cacheManagerRegistry, directExecutor()); + this.catalogManager = new CoordinatorDynamicCatalogManager( + new InMemoryCatalogStore(), + catalogFactory, + cacheManagerRegistry, + directExecutor(), + new CatalogLifecycleListeners(Set.of())); this.transactionManager = InMemoryTransactionManager.create( new TransactionManagerConfig().setIdleTimeout(new Duration(1, TimeUnit.DAYS)), yieldExecutor, diff --git a/core/trino-main/src/test/java/io/trino/connector/TestCatalogLifecycleListeners.java b/core/trino-main/src/test/java/io/trino/connector/TestCatalogLifecycleListeners.java new file mode 100644 index 000000000000..99aa3857cb9d --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/connector/TestCatalogLifecycleListeners.java @@ -0,0 +1,155 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.connector; + +import com.google.common.collect.ImmutableMap; +import com.google.common.util.concurrent.MoreExecutors; +import io.airlift.configuration.secrets.SecretsResolver; +import io.opentelemetry.api.OpenTelemetry; +import io.trino.cache.CacheManagerConfig; +import io.trino.cache.CacheManagerRegistry; +import io.trino.spi.catalog.CatalogName; +import io.trino.spi.catalog.CatalogProperties; +import io.trino.spi.connector.CatalogVersion; +import io.trino.spi.connector.Connector; +import io.trino.spi.connector.ConnectorFactory; +import io.trino.spi.connector.ConnectorName; +import io.trino.testing.TestingConnectorContext; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static io.airlift.tracing.Tracing.noopTracer; +import static io.trino.connector.CatalogHandle.createRootCatalogHandle; +import static org.assertj.core.api.Assertions.assertThat; + +class TestCatalogLifecycleListeners +{ + @Test + void testSuccessfulStaticAndDynamicCatalogLifecycleNotifications(@TempDir Path catalogDirectory) + throws IOException + { + List loadedCatalogs = new ArrayList<>(); + CatalogLifecycleListeners listeners = new CatalogLifecycleListeners( + Set.of(loadedCatalogs::add, _ -> { throw new RuntimeException("listener failure"); })); + + Files.writeString(catalogDirectory.resolve("static_catalog.properties"), "connector.name=mock\n"); + StaticCatalogManager staticManager = new StaticCatalogManager( + new TestingCatalogFactory(Set.of()), + new StaticCatalogManagerConfig().setCatalogConfigurationDir(catalogDirectory.toFile()), + MoreExecutors.directExecutor(), + listeners); + staticManager.loadInitialCatalogs(); + + InMemoryCatalogStore catalogStore = new InMemoryCatalogStore(); + catalogStore.addOrReplaceCatalog(catalogProperties("dynamic_initial")); + CoordinatorDynamicCatalogManager dynamicManager = new CoordinatorDynamicCatalogManager( + catalogStore, + new TestingCatalogFactory(Set.of()), + cacheManagerRegistry(), + MoreExecutors.directExecutor(), + listeners); + dynamicManager.loadInitialCatalogs(); + dynamicManager.createCatalog(new CatalogName("created_catalog"), new ConnectorName("mock"), Map.of(), false); + dynamicManager.createCatalog(new CatalogName("created_catalog"), new ConnectorName("mock"), Map.of(), true); + dynamicManager.dropCatalog(new CatalogName("created_catalog"), false); + dynamicManager.createCatalog(new CatalogName("created_catalog"), new ConnectorName("mock"), Map.of(), false); + + assertThat(loadedCatalogs).containsExactly( + new CatalogName("static_catalog"), + new CatalogName("dynamic_initial"), + new CatalogName("created_catalog"), + new CatalogName("created_catalog")); + + dynamicManager.stop(); + staticManager.stop(); + } + + @Test + void testFailedCatalogLoadDoesNotNotify() + { + List loadedCatalogs = new ArrayList<>(); + CatalogLifecycleListeners listeners = new CatalogLifecycleListeners(Set.of(loadedCatalogs::add)); + InMemoryCatalogStore catalogStore = new InMemoryCatalogStore(); + CatalogProperties broken = catalogProperties("broken_catalog"); + catalogStore.addOrReplaceCatalog(broken); + + CoordinatorDynamicCatalogManager manager = new CoordinatorDynamicCatalogManager( + catalogStore, + new TestingCatalogFactory(Set.of(broken.name())), + cacheManagerRegistry(), + MoreExecutors.directExecutor(), + listeners); + manager.loadInitialCatalogs(); + + assertThat(loadedCatalogs).isEmpty(); + manager.stop(); + } + + private static CatalogProperties catalogProperties(String name) + { + return new CatalogProperties(new CatalogName(name), new CatalogVersion("1"), new ConnectorName("mock"), ImmutableMap.of()); + } + + private static CacheManagerRegistry cacheManagerRegistry() + { + return new CacheManagerRegistry(OpenTelemetry.noop(), noopTracer(), new SecretsResolver(ImmutableMap.of()), new CacheManagerConfig()); + } + + private static class TestingCatalogFactory + implements CatalogFactory + { + private final Set failures; + + private TestingCatalogFactory(Set failures) + { + this.failures = Set.copyOf(failures); + } + + @Override + public void addConnectorFactory(ConnectorFactory connectorFactory) {} + + @Override + public CatalogConnector createCatalog(CatalogProperties catalogProperties) + { + if (failures.contains(catalogProperties.name())) { + throw new RuntimeException("catalog load failure"); + } + Connector connector = MockConnectorFactory.create().create(catalogProperties.name().toString(), catalogProperties.properties(), new TestingConnectorContext()); + CatalogHandle catalogHandle = createRootCatalogHandle(catalogProperties.name(), catalogProperties.version()); + ConnectorServices connectorServices = new ConnectorServices(noopTracer(), catalogHandle, connector); + return new CatalogConnector( + catalogHandle, + catalogProperties.connectorName(), + connectorServices, + connectorServices, + connectorServices, + Optional.of(catalogProperties)); + } + + @Override + public CatalogConnector createCatalog(CatalogHandle catalogHandle, ConnectorName connectorName, Connector connector) + { + throw new UnsupportedOperationException(); + } + } +} diff --git a/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java b/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java new file mode 100644 index 000000000000..96f74473cf39 --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/dispatcher/TestHogQlStatementResource.java @@ -0,0 +1,854 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.dispatcher; + +import com.google.common.collect.ImmutableList; +import io.airlift.http.client.HeaderName; +import io.airlift.http.client.HttpClient; +import io.airlift.http.client.Request; +import io.airlift.http.client.StatusResponseHandler.StatusResponse; +import io.airlift.http.client.StringResponseHandler.StringResponse; +import io.airlift.http.client.jetty.JettyHttpClient; +import io.airlift.json.JsonCodec; +import io.airlift.json.JsonCodecFactory; +import io.airlift.json.JsonMapperProvider; +import io.trino.client.Column; +import io.trino.client.QueryDataJacksonModule; +import io.trino.client.QueryResults; +import io.trino.client.ResultRowsDecoder; +import io.trino.connector.MockConnectorFactory; +import io.trino.connector.MockConnectorPlugin; +import io.trino.hogql.HogQlPhysicalCatalog; +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.plugin.tpch.TpchPlugin; +import io.trino.server.testing.TestingTrinoServer; +import io.trino.spi.connector.ColumnMetadata; +import io.trino.spi.connector.RelationColumnsMetadata; +import io.trino.spi.connector.SchemaTableName; +import io.trino.spi.type.ArrayType; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Stream; + +import static io.airlift.http.client.JsonResponseHandler.createJsonResponseHandler; +import static io.airlift.http.client.Request.Builder.prepareDelete; +import static io.airlift.http.client.Request.Builder.prepareGet; +import static io.airlift.http.client.Request.Builder.preparePost; +import static io.airlift.http.client.StaticBodyGenerator.createStaticBodyGenerator; +import static io.airlift.http.client.StatusResponseHandler.createStatusResponseHandler; +import static io.airlift.http.client.StringResponseHandler.createStringResponseHandler; +import static io.airlift.testing.Closeables.closeAll; +import static io.trino.client.ProtocolHeaders.TRINO_HEADERS; +import static io.trino.spi.type.BigintType.BIGINT; +import static io.trino.spi.type.VarcharType.createVarcharType; +import static io.trino.testing.TestingAccessControlManager.TestingPrivilegeType.SELECT_COLUMN; +import static io.trino.testing.TestingAccessControlManager.privilege; +import static jakarta.ws.rs.core.MediaType.APPLICATION_JSON; +import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.stream.Collectors.joining; +import static java.util.stream.IntStream.range; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.parallel.ExecutionMode.SAME_THREAD; + +@Execution(SAME_THREAD) +class TestHogQlStatementResource +{ + private static final HeaderName REQUEST_USER_HEADER = HeaderName.of(TRINO_HEADERS.requestUser()); + private static final HeaderName REQUEST_SESSION_HEADER = HeaderName.of(TRINO_HEADERS.requestSession()); + private static final HeaderName CONTENT_TYPE_HEADER = HeaderName.of("Content-Type"); + private static final JsonCodec QUERY_RESULTS_CODEC = new JsonCodecFactory(new JsonMapperProvider() + .withModules(Set.of(new QueryDataJacksonModule())) + .get()) + .jsonCodec(QueryResults.class); + private static final JsonCodec STRING_CODEC = JsonCodec.jsonCodec(String.class); + private static final JsonCodec PHYSICAL_CATALOG_CODEC = JsonCodec.jsonCodec(HogQlPhysicalCatalog.class); + private static final String SECRET = "do-not-echo-this-value"; + + private static HttpClient client; + private static TestingTrinoServer server; + + @BeforeAll + static void setUp() + throws Exception + { + client = new JettyHttpClient(); + server = TestingTrinoServer.builder() + .setProperties(Map.of( + "hogql.enabled", "true", + "sql.default-catalog", "tpch")) + .build(); + server.installPlugin(new TpchPlugin()); + server.createCatalog("tpch", "tpch"); + SchemaTableName physicalTable = new SchemaTableName("analytics", "events"); + server.installPlugin(new MockConnectorPlugin(MockConnectorFactory.builder() + .withName("physical_metadata_connector") + .withListSchemaNames(_ -> ImmutableList.of(physicalTable.getSchemaName())) + .withStreamRelationColumns((_, schema, relationFilter) -> { + if (schema.isPresent() && !schema.orElseThrow().equals(physicalTable.getSchemaName())) { + return ImmutableList.of().iterator(); + } + if (!relationFilter.apply(Set.of(physicalTable)).contains(physicalTable)) { + return ImmutableList.of().iterator(); + } + return ImmutableList.of(RelationColumnsMetadata.forTable( + physicalTable, + ImmutableList.of( + ColumnMetadata.builder() + .setName("event_id") + .setType(BIGINT) + .setNullable(false) + .build(), + ColumnMetadata.builder() + .setName("tags") + .setType(new ArrayType(createVarcharType(7))) + .setHidden(true) + .build()))) + .iterator(); + }) + .build())); + server.createCatalog("physical_metadata", "physical_metadata_connector"); + } + + @AfterAll + static void tearDown() + throws Exception + { + closeAll(server, client); + } + + @Test + public void testEndpointIsDisabledByDefault() + throws Exception + { + HttpClient disabledClient = new JettyHttpClient(); + TestingTrinoServer disabledServer = TestingTrinoServer.create(); + try { + StatusResponse response = disabledClient.execute( + preparePost() + .setUri(disabledServer.resolve("/v1/hogql")) + .setHeader(REQUEST_USER_HEADER, "user") + .setHeader(CONTENT_TYPE_HEADER, APPLICATION_JSON) + .setBodyGenerator(createStaticBodyGenerator(hogQlRequest("SELECT 1"), UTF_8)) + .build(), + createStatusResponseHandler()); + + assertThat(response.getStatusCode()).isEqualTo(404); + + StatusResponse physicalCatalogResponse = disabledClient.execute( + prepareGet() + .setUri(disabledServer.resolve("/v1/hogql/compatibility/physical-catalog?catalog=missing&protocolVersion=1")) + .setHeader(REQUEST_USER_HEADER, "user") + .build(), + createStatusResponseHandler()); + assertThat(physicalCatalogResponse.getStatusCode()).isEqualTo(404); + } + finally { + closeAll(disabledServer, disabledClient); + } + } + + @Test + public void testEnabledEndpointUsesStatementProtocol() + throws Exception + { + List hogqlResults = runHogQlToCompletion(hogQlRequestWithTypedValues("SELECT 1")); + assertThat(hogqlResults.getFirst().getNextUri().getPath()).startsWith("/v1/statement/queued/"); + assertThat(rows(hogqlResults)).containsExactly(ImmutableList.of(1)); + + List tableScanResults = runHogQlToCompletion(hogQlRequest("SELECT nationkey FROM tpch.tiny.nation")); + assertThat(rows(tableScanResults)).hasSize(25); + + List sqlResults = runToCompletion("/v1/statement", "SELECT 2", false); + assertThat(rows(sqlResults)).containsExactly(ImmutableList.of(2)); + } + + @Test + public void testBindsScopedVariablesAndFilters() + throws Exception + { + List results = runHogQlToCompletion(hogQlRequestWithScopedBindings( + "SELECT {variables.organization_id}, {filters.date_from}")); + + assertThat(results.getLast().getError()).isNull(); + assertThat(rows(results)).containsExactly(ImmutableList.of(42L, "2026-01-01")); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("differentialQueries") + public void testHogQlExecutionMatchesTrinoSql(String name, String hogql, String trinoSql) + throws Exception + { + List hogqlResults = runHogQlToCompletion(hogQlRequest(hogql)); + List trinoResults = runToCompletion("/v1/statement", trinoSql, false); + + assertThat(hogqlResults.getLast().getError()).describedAs("HogQL result for %s", name).isNull(); + assertThat(trinoResults.getLast().getError()).describedAs("Trino SQL result for %s", name).isNull(); + assertThat(resultColumns(hogqlResults)).isEqualTo(resultColumns(trinoResults)); + assertThat(rows(hogqlResults)).isEqualTo(rows(trinoResults)); + } + + private static Stream differentialQueries() + { + return Stream.of( + Arguments.of( + "nulls, cases, and nested casts", + "SELECT CAST(NULL AS Nullable(Int32)) AS value, CASE WHEN NULL IS NULL THEN 'yes' ELSE 'no' END AS marker", + "SELECT CAST(NULL AS integer) AS value, CASE WHEN NULL IS NULL THEN 'yes' ELSE 'no' END AS marker"), + Arguments.of( + "joins, grouping, and ordering", + "SELECT r.regionkey, count(n.nationkey) AS nations " + + "FROM tpch.tiny.region r LEFT JOIN tpch.tiny.nation n ON r.regionkey = n.regionkey " + + "GROUP BY r.regionkey ORDER BY r.regionkey", + "SELECT r.regionkey, count(n.nationkey) AS nations " + + "FROM tpch.tiny.region r LEFT JOIN tpch.tiny.nation n ON r.regionkey = n.regionkey " + + "GROUP BY r.regionkey ORDER BY r.regionkey"), + Arguments.of( + "window frames", + "SELECT nationkey, sum(nationkey) OVER (ORDER BY nationkey ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS running, " + + "rank() OVER (ORDER BY nationkey) AS ranked, row_number() OVER (ORDER BY nationkey) AS numbered " + + "FROM tpch.tiny.nation WHERE nationkey < 5 ORDER BY nationkey", + "SELECT nationkey, sum(nationkey) OVER (ORDER BY nationkey ROWS BETWEEN 1 PRECEDING AND CURRENT ROW) AS running, " + + "rank() OVER (ORDER BY nationkey) AS ranked, row_number() OVER (ORDER BY nationkey) AS numbered " + + "FROM tpch.tiny.nation WHERE nationkey < 5 ORDER BY nationkey"), + Arguments.of( + "values and set operations", + "SELECT value FROM (VALUES (2), (1), (2)) AS numbers(value) UNION SELECT 3 ORDER BY value", + "SELECT value FROM (VALUES (2), (1), (2)) AS numbers(value) UNION SELECT 3 ORDER BY value"), + Arguments.of( + "arrays, maps, and rows", + "SELECT ARRAY[1, 2, 3][2] AS array_value, mapFromArrays(['k'], [7])['k'] AS map_value, (11, 'x').1 AS row_value", + "SELECT ARRAY[1, 2, 3][2] AS array_value, MAP(ARRAY['k'], ARRAY[7])['k'] AS map_value, ROW(11, 'x')[1] AS row_value"), + Arguments.of( + "timestamp precision", + "SELECT CAST('2024-03-10 01:59:59.123456' AS Timestamp(6)) AS value", + "SELECT CAST('2024-03-10 01:59:59.123456' AS timestamp(6)) AS value"), + Arguments.of( + "interval expressions", + "SELECT INTERVAL 2 WEEK AS weeks, INTERVAL '3 months' AS months, INTERVAL nationkey DAY AS dynamic " + + "FROM tpch.tiny.nation WHERE nationkey = 2", + "SELECT 2 * INTERVAL '7' DAY AS weeks, 3 * INTERVAL '1' MONTH AS months, nationkey * INTERVAL '1' DAY AS dynamic " + + "FROM tpch.tiny.nation WHERE nationkey = 2"), + Arguments.of( + "v0 scalar and collection functions", + "SELECT abs(-2), coalesce(NULL, 'fallback'), if(true, 1, 2), lower('ABC'), upper('abc'), length('abc'), " + + "concat('a', 'b'), replace('abc', 'b', 'x'), arrayDistinct([1, 1, 2]), arraySort([2, 1]), " + + "arrayFlatten([[1], [2]]), arrayStringConcat(['a', 'b']), " + + "dateAdd('day', 1, CAST('2024-01-01' AS Date)), " + + "dateDiff('day', CAST('2024-01-01' AS Date), CAST('2024-01-03' AS Date)), " + + "dateTrunc('day', CAST('2024-01-01 12:34:56' AS Timestamp(3)))", + "SELECT abs(-2), coalesce(NULL, 'fallback'), if(true, 1, 2), lower('ABC'), upper('abc'), length('abc'), " + + "concat('a', 'b'), replace('abc', 'b', 'x'), array_distinct(ARRAY[1, 1, 2]), array_sort(ARRAY[2, 1]), " + + "flatten(ARRAY[ARRAY[1], ARRAY[2]]), array_join(ARRAY['a', 'b'], ''), " + + "date_add('day', 1, CAST('2024-01-01' AS date)), " + + "date_diff('day', CAST('2024-01-01' AS date), CAST('2024-01-03' AS date)), " + + "date_trunc('day', CAST('2024-01-01 12:34:56' AS timestamp(3)))"), + Arguments.of( + "v0 aggregate functions", + "SELECT count(), sum(nationkey), min(nationkey), max(nationkey), avg(nationkey), any(nationkey), " + + "argMin(name, nationkey), argMax(name, nationkey), array_agg(nationkey ORDER BY nationkey) " + + "FROM tpch.tiny.nation WHERE nationkey < 3", + "SELECT count(), sum(nationkey), min(nationkey), max(nationkey), avg(nationkey), arbitrary(nationkey), " + + "min_by(name, nationkey), max_by(name, nationkey), array_agg(nationkey ORDER BY nationkey) " + + "FROM tpch.tiny.nation WHERE nationkey < 3"), + Arguments.of( + "v0 value window function", + "SELECT first_value(name) OVER (ORDER BY nationkey) FROM tpch.tiny.nation WHERE nationkey < 3 ORDER BY nationkey", + "SELECT first_value(name) OVER (ORDER BY nationkey) FROM tpch.tiny.nation WHERE nationkey < 3 ORDER BY nationkey"), + Arguments.of( + "corpus temporal rewrites", + "SELECT addDays(CAST('2024-01-15 12:34:56' AS Timestamp), 2), " + + "subtractDays(CAST('2024-01-15 12:34:56' AS Timestamp), 3), " + + "addMonths(CAST('2024-01-15 12:34:56' AS Timestamp), 1), " + + "subtractMonths(CAST('2024-01-15 12:34:56' AS Timestamp), 2), " + + "subtractYears(CAST('2024-01-15 12:34:56' AS Timestamp), 1), " + + "toStartOfDay(CAST('2024-01-15 12:34:56' AS Timestamp)), " + + "toStartOfHour(CAST('2024-01-15 12:34:56' AS Timestamp)), " + + "toStartOfMonth(CAST('2024-01-15 12:34:56' AS Timestamp)), " + + "toStartOfWeek(CAST('2024-01-15 12:34:56' AS Timestamp), 1), " + + "toDayOfMonth(CAST('2024-01-15' AS Date)), toDayOfWeek(CAST('2024-01-15' AS Date)), " + + "toMonth(CAST('2024-01-15' AS Date)), toYear(CAST('2024-01-15' AS Date)), " + + "toLastDayOfMonth(CAST('2024-01-15' AS Date)), " + + "formatDateTime(CAST('2024-01-15 12:34:56' AS Timestamp), '%Y-%m-%d'), " + + "parseDateTime('2024-01-15', '%Y-%m-%d'), parseDateTimeBestEffort('2024-01-15 12:34:56')", + "SELECT date_add('day', 2, CAST('2024-01-15 12:34:56' AS timestamp(0))), " + + "date_add('day', -3, CAST('2024-01-15 12:34:56' AS timestamp(0))), " + + "date_add('month', 1, CAST('2024-01-15 12:34:56' AS timestamp(0))), " + + "date_add('month', -2, CAST('2024-01-15 12:34:56' AS timestamp(0))), " + + "date_add('year', -1, CAST('2024-01-15 12:34:56' AS timestamp(0))), " + + "date_trunc('day', CAST('2024-01-15 12:34:56' AS timestamp(0))), " + + "date_trunc('hour', CAST('2024-01-15 12:34:56' AS timestamp(0))), " + + "date_trunc('month', CAST('2024-01-15 12:34:56' AS timestamp(0))), " + + "date_trunc('week', CAST('2024-01-15 12:34:56' AS timestamp(0))), " + + "day(CAST('2024-01-15' AS date)), day_of_week(CAST('2024-01-15' AS date)), " + + "month(CAST('2024-01-15' AS date)), year(CAST('2024-01-15' AS date)), " + + "last_day_of_month(CAST('2024-01-15' AS date)), " + + "date_format(CAST('2024-01-15 12:34:56' AS timestamp(0)), '%Y-%m-%d'), " + + "date_parse('2024-01-15', '%Y-%m-%d'), TRY_CAST('2024-01-15 12:34:56' AS timestamp(3))"), + Arguments.of( + "corpus JSON regex and string rewrites", + "SELECT JSONExtractString(payload, 'name'), JSONExtractInt(payload, 'items', 0), " + + "JSONExtractFloat(payload, 'score'), JSONExtractBool(payload, 'active'), " + + "JSONExtractUInt(payload, 'items', 1), JSONExtractRaw(payload, 'object'), " + + "JSONLength(payload, 'items'), JSONHas(payload, 'object'), JSONExtractKeys(payload, 'object'), " + + "extract(sample, '([a-z]+)'), extractAll(sample, '([a-z]+)'), match(sample, '^[a-z]+'), " + + "replaceRegexpAll(sample, '[0-9]', 'x'), replaceRegexpOne(sample, '[0-9]+', 'x'), " + + "splitByString('123', sample), substringUTF8(sample, 2, 3), position(sample, '123') " + + "FROM (VALUES ('{\"name\":\"Ada\",\"items\":[2,3],\"active\":true,\"score\":1.5,\"object\":{\"k\":7}}', 'abc123abc')) AS t(payload, sample)", + "SELECT coalesce(json_extract_scalar(payload, '$[\"name\"]'), ''), " + + "coalesce(TRY_CAST(json_extract_scalar(payload, '$[\"items\"][0]') AS bigint), 0), " + + "coalesce(TRY_CAST(json_extract_scalar(payload, '$[\"score\"]') AS double), 0E0), " + + "coalesce(TRY_CAST(json_extract_scalar(payload, '$[\"active\"]') AS boolean), false), " + + "coalesce(TRY_CAST(json_extract_scalar(payload, '$[\"items\"][1]') AS bigint), 0), " + + "coalesce(json_format(json_extract(payload, '$[\"object\"]')), ''), " + + "coalesce(json_size(payload, '$[\"items\"]'), 0), json_extract(payload, '$[\"object\"]') IS NOT NULL, " + + "map_keys(coalesce(TRY_CAST(json_extract(payload, '$[\"object\"]') AS map(varchar, json)), CAST(map(ARRAY[], ARRAY[]) AS map(varchar, json)))), " + + "coalesce(regexp_extract(sample, '([a-z]+)', 1), ''), regexp_extract_all(sample, '([a-z]+)', 1), " + + "regexp_like(sample, '^[a-z]+'), regexp_replace(sample, '[0-9]', 'x'), " + + "regexp_replace(sample, '(?s)^(.*?)(([0-9]+))', '$1x'), split(sample, '123'), substring(sample, 2, 3), strpos(sample, '123') " + + "FROM (VALUES ('{\"name\":\"Ada\",\"items\":[2,3],\"active\":true,\"score\":1.5,\"object\":{\"k\":7}}', 'abc123abc')) AS t(payload, sample)"), + Arguments.of( + "corpus collection and lambda rewrites", + "SELECT arrayElement([1, 2, 3], -1), arrayFilter(x -> x > 1, [1, 2, 3]), " + + "arrayFirst(x -> x > 1, [1, 2, 3]), arrayMap(x -> x + 1, [1, 2, 3]), " + + "arraySum([1, 2, 3]), arrayMin([3, 1, 2]), arraySlice([1, 2, 3, 4], 2, 2), " + + "arrayEnumerate(['a', 'b']), range(2, 5), tupleElement(tuple('x', 7), 2), " + + "splitByChar(',', 'a,b'), has([1, 2], 2), hasAny([1, 2], [2, 3]), " + + "map('a', 1, 'b', 2)['b'], mapUpdate(map('a', 1), map('a', 2))['a']", + "SELECT element_at(ARRAY[1, 2, 3], -1), filter(ARRAY[1, 2, 3], x -> x > 1), " + + "element_at(filter(ARRAY[1, 2, 3], x -> x > 1), 1), transform(ARRAY[1, 2, 3], x -> x + 1), " + + "reduce(ARRAY[1, 2, 3], 0, (total, item) -> total + item, total -> total), " + + "array_min(ARRAY[3, 1, 2]), slice(ARRAY[1, 2, 3, 4], 2, 2), sequence(1, cardinality(ARRAY['a', 'b'])), " + + "sequence(2, 5 - 1), ROW('x', 7)[2], split('a,b', ','), " + + "coalesce(contains(ARRAY[1, 2], 2), false), arrays_overlap(ARRAY[1, 2], ARRAY[2, 3]), " + + "map(ARRAY['a', 'b'], ARRAY[1, 2])['b'], map_concat(map(ARRAY['a'], ARRAY[1]), map(ARRAY['a'], ARRAY[2]))['a']"), + Arguments.of( + "corpus arrayJoin lowering", + "SELECT id, arrayJoin(values_array) AS value " + + "FROM (VALUES (2, [3, 1]), (1, [2])) AS t(id, values_array) ORDER BY id, value", + "SELECT id, value FROM (VALUES (2, ARRAY[3, 1]), (1, ARRAY[2])) AS t(id, values_array) " + + "CROSS JOIN UNNEST(values_array) AS u(value) ORDER BY id, value"), + Arguments.of( + "corpus nested LIMIT BY lowering", + "SELECT nested.* FROM (" + + "SELECT category, value FROM (VALUES ('a', 2), ('a', 1), ('b', 4), ('b', 3)) AS t(category, value) " + + "ORDER BY value DESC LIMIT 1 BY category) AS nested ORDER BY category", + "SELECT category, value FROM (" + + "SELECT category, value, row_number() OVER (PARTITION BY category ORDER BY value DESC) AS row_number " + + "FROM (VALUES ('a', 2), ('a', 1), ('b', 4), ('b', 3)) AS t(category, value)) " + + "WHERE row_number <= 1 ORDER BY category"), + Arguments.of( + "corpus aggregate rewrites", + "SELECT countIf(active), sumIf(value, active), minIf(value, active), maxIf(value, active), " + + "avgIf(value, active), uniqExactIf(value, active), countDistinct(value), " + + "argMaxIf(value, weight, active), argMinIf(value, weight, active), " + + "quantile(0.5)(value), quantileIf(0.5)(value, active) " + + "FROM (VALUES (1, true, 10), (1, false, 20), (2, true, 30), (3, true, 40)) AS t(value, active, weight)", + "SELECT count(*) FILTER (WHERE active), sum(value) FILTER (WHERE active), min(value) FILTER (WHERE active), " + + "max(value) FILTER (WHERE active), avg(value) FILTER (WHERE active), " + + "count(DISTINCT value) FILTER (WHERE active), count(DISTINCT value), " + + "max_by(value, weight) FILTER (WHERE active), min_by(value, weight) FILTER (WHERE active), " + + "approx_percentile(value, 0.5), approx_percentile(value, 0.5) FILTER (WHERE active) " + + "FROM (VALUES (1, true, 10), (1, false, 20), (2, true, 30), (3, true, 40)) AS t(value, active, weight)"), + Arguments.of( + "corpus numeric conversion and operator rewrites", + "SELECT toFloatOrZero('bad'), toFloatOrDefault('bad', 1), toDecimal('1.25', 2), intDiv(-5, 2), " + + "toInt('4'), toIntOrZero('bad'), _toInt16('3'), " + + "toUUID('00000000-0000-0000-0000-000000000001'), roundBankers(1.25, 1), " + + "plus(4, 2), minus(4, 2), multiply(4, 2), divide(5, 2), " + + "greater(4, 2), greaterOrEquals(4, 4), lessOrEquals(2, 4), notEquals(1, 2), " + + "and(true, true, false), or(false, false, true), not(false), empty(''), notEmpty([1]), " + + "empty(mapFromArrays(['key'], [1])), empty(CAST(NULL AS Array(Int64)))", + "SELECT coalesce(TRY_CAST('bad' AS double), 0E0), coalesce(TRY_CAST('bad' AS double), CAST(1 AS double)), " + + "TRY_CAST('1.25' AS decimal(18, 2)), " + + "CAST(-5 AS bigint) / CAST(2 AS bigint) - if(CAST(-5 AS bigint) % CAST(2 AS bigint) <> 0 AND " + + "(CAST(-5 AS bigint) < 0 AND CAST(2 AS bigint) > 0 OR CAST(-5 AS bigint) > 0 AND CAST(2 AS bigint) < 0), 1, 0), " + + "CAST('4' AS bigint), coalesce(TRY_CAST('bad' AS bigint), 0), CAST('3' AS smallint), " + + "TRY_CAST('00000000-0000-0000-0000-000000000001' AS uuid), round(DOUBLE '1.25', 1), " + + "4 + 2, 4 - 2, 4 * 2, 5 / 2, 4 > 2, 4 >= 4, 2 <= 4, 1 <> 2, " + + "(true AND true) AND false, (false OR false) OR true, NOT false, " + + "coalesce(length(CAST('' AS varchar)), 0) = 0, coalesce(cardinality(ARRAY[1]), 0) > 0, " + + "coalesce(cardinality(map(ARRAY['key'], ARRAY[1])), 0) = 0, " + + "coalesce(cardinality(CAST(NULL AS array(bigint))), 0) = 0")); + } + + @Test + public void testPhysicalCatalogCompatibilityEndpoint() + { + HogQlPhysicalCatalog catalog = client.execute( + prepareGet() + .setUri(server.resolve("/v1/hogql/compatibility/physical-catalog?catalog=physical_metadata&protocolVersion=1")) + .setHeader(REQUEST_USER_HEADER, "user") + .build(), + createJsonResponseHandler(PHYSICAL_CATALOG_CODEC)); + + assertThat(catalog.protocolVersion()).isEqualTo(1); + assertThat(catalog.schemaVersion()).isEqualTo(1); + assertThat(catalog.catalog()).isEqualTo(new HogQlPhysicalCatalog.Identifier("physical_metadata", false)); + assertThat(catalog.catalogHandleVersion()).isNotBlank(); + assertThat(catalog.tables()).singleElement().satisfies(table -> { + assertThat(table.schema()).isEqualTo(new HogQlPhysicalCatalog.Identifier("analytics", false)); + assertThat(table.table()).isEqualTo(new HogQlPhysicalCatalog.Identifier("events", false)); + assertThat(table.columns()).containsExactly( + new HogQlPhysicalCatalog.Column( + new HogQlPhysicalCatalog.Identifier("event_id", false), + 1, + "bigint", + false, + false, + true), + new HogQlPhysicalCatalog.Column( + new HogQlPhysicalCatalog.Identifier("tags", false), + 2, + "array(varchar(7))", + true, + true, + false)); + }); + } + + @Test + public void testPhysicalCatalogCompatibilityEndpointFailsClosed() + { + StatusResponse missingCatalog = client.execute( + prepareGet() + .setUri(server.resolve("/v1/hogql/compatibility/physical-catalog")) + .setHeader(REQUEST_USER_HEADER, "user") + .build(), + createStatusResponseHandler()); + assertThat(missingCatalog.getStatusCode()).isEqualTo(400); + + StatusResponse missingProtocolVersion = client.execute( + prepareGet() + .setUri(server.resolve("/v1/hogql/compatibility/physical-catalog?catalog=physical_metadata")) + .setHeader(REQUEST_USER_HEADER, "user") + .build(), + createStatusResponseHandler()); + assertThat(missingProtocolVersion.getStatusCode()).isEqualTo(400); + + StatusResponse unsupportedProtocolVersion = client.execute( + prepareGet() + .setUri(server.resolve("/v1/hogql/compatibility/physical-catalog?catalog=physical_metadata&protocolVersion=2")) + .setHeader(REQUEST_USER_HEADER, "user") + .build(), + createStatusResponseHandler()); + assertThat(unsupportedProtocolVersion.getStatusCode()).isEqualTo(400); + + StatusResponse unknownCatalog = client.execute( + prepareGet() + .setUri(server.resolve("/v1/hogql/compatibility/physical-catalog?catalog=missing&protocolVersion=1")) + .setHeader(REQUEST_USER_HEADER, "user") + .build(), + createStatusResponseHandler()); + assertThat(unknownCatalog.getStatusCode()).isEqualTo(404); + } + + @Test + public void testPhysicalCatalogCompatibilityEndpointRequiresAuthentication() + { + StatusResponse response = client.execute( + prepareGet() + .setUri(server.resolve("/v1/hogql/compatibility/physical-catalog?catalog=physical_metadata&protocolVersion=1")) + .build(), + createStatusResponseHandler()); + + assertThat(response.getStatusCode()).isEqualTo(401); + } + + @Test + public void testPhysicalCatalogCompatibilityEndpointAppliesVisibilityFilters() + { + server.getAccessControl().deny(privilege("events.event_id", SELECT_COLUMN)); + try { + HogQlPhysicalCatalog catalog = client.execute( + prepareGet() + .setUri(server.resolve("/v1/hogql/compatibility/physical-catalog?catalog=physical_metadata&protocolVersion=1")) + .setHeader(REQUEST_USER_HEADER, "user") + .build(), + createJsonResponseHandler(PHYSICAL_CATALOG_CODEC)); + + assertThat(catalog.tables()).singleElement().satisfies(table -> { + assertThat(table.columns()).extracting(column -> column.name().value()).containsExactly("tags"); + assertThat(table.columns()).extracting(HogQlPhysicalCatalog.Column::ordinal).containsExactly(2); + }); + } + finally { + server.getAccessControl().reset(); + } + } + + @Test + public void testEmptyResultsAndCompilerDiagnostics() + throws Exception + { + List emptyResults = runHogQlToCompletion(hogQlRequest("SELECT 1 WHERE false")); + assertThat(rows(emptyResults)).isEmpty(); + assertThat(emptyResults.getLast().getWarnings()).isEmpty(); + + List invalidResults = runHogQlToCompletion(hogQlRequest("SELECT (")); + assertThat(invalidResults.getLast().getError().getErrorName()).isEqualTo("HOGQL_SYNTAX_ERROR"); + + List unsupportedFunction = runHogQlToCompletion(hogQlRequest("SELECT unsupportedHogQlFunction(1)")); + assertThat(unsupportedFunction.getLast().getError().getErrorName()).isEqualTo("HOGQL_RESOLUTION_ERROR"); + + List pivot = runHogQlToCompletion(hogQlRequest( + "SELECT * FROM tpch.tiny.nation PIVOT (sum(nationkey) FOR regionkey IN (0))")); + assertThat(pivot.getLast().getError().getErrorName()).isEqualTo("HOGQL_UNSUPPORTED_FEATURE"); + assertThat(pivot.getLast().getError().getMessage()).contains("PIVOT is outside the HogQL v0 profile"); + + List modifier = runHogQlToCompletion(hogQlRequestWithModifier("SELECT 1")); + assertThat(modifier.getLast().getError().getErrorName()).isEqualTo("HOGQL_UNSUPPORTED_FEATURE"); + assertThat(modifier.getLast().getError().getMessage()).contains("Modifiers are outside the HogQL v0 profile"); + } + + @Test + public void testQueuedQueryCanBeCancelled() + { + QueryResults queued = postQuery(hogQlRequest("SELECT nationkey FROM tpch.tiny.nation")); + + StatusResponse response = client.execute( + prepareDelete() + .setUri(queued.getNextUri()) + .setHeader(REQUEST_USER_HEADER, "user") + .build(), + createStatusResponseHandler()); + + assertThat(response.getStatusCode()).isEqualTo(204); + } + + @Test + public void testQueryUsesNativeExecutionTimeout() + throws Exception + { + List results = runHogQlToCompletion( + hogQlRequest("SELECT count(*) FROM tpch.tiny.lineitem a CROSS JOIN tpch.tiny.lineitem b CROSS JOIN tpch.tiny.lineitem c"), + "query_max_execution_time=1ms"); + + assertThat(results.getLast().getError().getErrorName()) + .describedAs("query error: %s", results.getLast().getError()) + .isEqualTo("EXCEEDED_TIME_LIMIT"); + } + + @Test + public void testExplainUsesNativePlanner() + throws Exception + { + List results = runHogQlToCompletion(hogQlExplainRequest( + "SELECT nationkey FROM tpch.tiny.nation", + "DISTRIBUTED", + "TEXT")); + + assertThat(results.getLast().getError()).isNull(); + assertThat(rows(results)).singleElement().satisfies(row -> + assertThat(row).singleElement().asString().contains("TableScan")); + } + + @Test + public void testExplainBindsScopedVariables() + throws Exception + { + List results = runHogQlToCompletion(hogQlExplainRequestWithScopedBinding( + "SELECT nationkey FROM tpch.tiny.nation WHERE name = {variables.name}")); + + assertThat(results.getLast().getError()).isNull(); + assertThat(rows(results)).singleElement().satisfies(row -> + assertThat(row).singleElement().asString().contains("ALGERIA")); + } + + @Test + public void testExplainExpandsSelectAliasInWhere() + throws Exception + { + List results = runHogQlToCompletion(hogQlExplainRequest( + "SELECT nationkey + 1 AS next_key FROM tpch.tiny.nation WHERE next_key > 1", + "DISTRIBUTED", + "TEXT")); + + assertThat(results.getLast().getError()).isNull(); + assertThat(rows(results)).singleElement().satisfies(row -> + assertThat(row).singleElement().asString().contains("nationkey")); + } + + @Test + public void testExplainExpandsSelectAliasContainingScopedVariable() + throws Exception + { + List results = runHogQlToCompletion(hogQlExplainRequestWithScopedBinding( + "SELECT nationkey + length({variables.name}) AS next_key FROM tpch.tiny.nation WHERE next_key > 1")); + + assertThat(results.getLast().getError()).isNull(); + assertThat(rows(results)).singleElement().satisfies(row -> + assertThat(row).singleElement().asString().contains("bigint '7'")); + } + + @Test + public void testEndpointRequiresJsonContentType() + { + StringResponse response = post(hogQlRequest("SELECT 1"), "text/plain"); + + assertThat(response.getStatusCode()).isEqualTo(415); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("invalidRequests") + public void testInvalidRequestFailsClosed(String name, String request) + { + StringResponse response = post(request, APPLICATION_JSON); + + assertThat(response.getStatusCode()).isEqualTo(400); + assertThat(response.getBody()).doesNotContain(SECRET); + assertThat(response.getBody()).hasSizeLessThan(1024); + } + + private static Stream invalidRequests() + { + String languageVersion = HogQlLanguageContract.current().languageVersion().toString(); + return Stream.of( + Arguments.of("unsupported protocol version", + """ + {"query":"SELECT '%s'","protocolVersion":2,"languageVersion":"%s"} + """.formatted(SECRET, languageVersion)), + Arguments.of("unsupported language version", + """ + {"query":"SELECT '%s'","protocolVersion":1,"languageVersion":"999.0.0"} + """.formatted(SECRET)), + Arguments.of("unknown top-level field", + """ + {"query":"SELECT 1","protocolVersion":1,"languageVersion":"%s","unknown":"%s"} + """.formatted(languageVersion, SECRET)), + Arguments.of("unknown typed-value field", + """ + {"query":"SELECT 1","protocolVersion":1,"languageVersion":"%s","parameters":{"p":{"type":"string","value":"%s","unknown":true}}} + """.formatted(languageVersion, SECRET)), + Arguments.of("ambiguous typed value", + """ + {"query":"SELECT 1","protocolVersion":1,"languageVersion":"%s","parameters":{"p":{"type":"string","unknown":"%s"}}} + """.formatted(languageVersion, SECRET)), + Arguments.of("nonpositive catalog generation", + """ + {"query":"SELECT '%s'","protocolVersion":1,"languageVersion":"%s","catalogGeneration":0} + """.formatted(SECRET, languageVersion)), + Arguments.of("unknown explain field", + """ + {"query":"SELECT 1","protocolVersion":1,"languageVersion":"%s","explain":{"type":"LOGICAL","format":"TEXT","unknown":"%s"}} + """.formatted(languageVersion, SECRET)), + Arguments.of("invalid explain type", + """ + {"query":"SELECT '%s'","protocolVersion":1,"languageVersion":"%s","explain":{"type":"UNKNOWN","format":"TEXT"}} + """.formatted(SECRET, languageVersion)), + Arguments.of("too many parameter bindings", requestWithParameterCount(languageVersion, 1_001)), + Arguments.of("too many total bindings", requestWithTotalBindingCount(languageVersion)), + Arguments.of("request exceeds byte limit", oversizedRequest(languageVersion)), + Arguments.of("malformed JSON", + """ + {"query":"SELECT 1","protocolVersion":1,"languageVersion":"%s","parameters":{"p":{"type":"string","value":"%s"}} + """.formatted(languageVersion, SECRET))); + } + + private static String requestWithParameterCount(String languageVersion, int parameterCount) + { + String parameters = range(0, parameterCount) + .mapToObj(index -> "\"p%s\":{\"type\":\"integer\",\"value\":%s}".formatted(index, index)) + .collect(joining(",")); + return """ + {"query":"SELECT 1","protocolVersion":1,"languageVersion":"%s","parameters":{%s}} + """.formatted(languageVersion, parameters); + } + + private static String requestWithTotalBindingCount(String languageVersion) + { + String parameters = range(0, 1_000) + .mapToObj(index -> "\"p%s\":{\"type\":\"integer\",\"value\":%s}".formatted(index, index)) + .collect(joining(",")); + return """ + {"query":"SELECT 1","protocolVersion":1,"languageVersion":"%s","parameters":{%s},"variables":{"extra":{"type":"integer","value":1}}} + """.formatted(languageVersion, parameters); + } + + private static String oversizedRequest(String languageVersion) + { + return """ + {"query":"SELECT '%s%s'","protocolVersion":1,"languageVersion":"%s"} + """.formatted(SECRET, "x".repeat(2 * 1024 * 1024), languageVersion); + } + + private static List runHogQlToCompletion(String request) + { + return runToCompletion("/v1/hogql", request, true, Optional.empty()); + } + + private static List runHogQlToCompletion(String request, String session) + { + return runToCompletion("/v1/hogql", request, true, Optional.of(session)); + } + + private static List runToCompletion(String path, String body, boolean json) + { + return runToCompletion(path, body, json, Optional.empty()); + } + + private static List runToCompletion(String path, String body, boolean json, Optional session) + { + ImmutableList.Builder results = ImmutableList.builder(); + Request.Builder request = preparePost() + .setUri(server.resolve(path)) + .setHeader(REQUEST_USER_HEADER, "user") + .setBodyGenerator(createStaticBodyGenerator(body, UTF_8)); + if (json) { + request.setHeader(CONTENT_TYPE_HEADER, APPLICATION_JSON); + } + session.ifPresent(value -> request.setHeader(REQUEST_SESSION_HEADER, value)); + QueryResults current = client.execute(request.build(), createJsonResponseHandler(QUERY_RESULTS_CODEC)); + results.add(current); + + while (current.getNextUri() != null) { + current = client.execute( + prepareGet() + .setUri(current.getNextUri()) + .setHeader(REQUEST_USER_HEADER, "user") + .build(), + createJsonResponseHandler(QUERY_RESULTS_CODEC)); + results.add(current); + } + return results.build(); + } + + private static QueryResults postQuery(String request) + { + return client.execute( + preparePost() + .setUri(server.resolve("/v1/hogql")) + .setHeader(REQUEST_USER_HEADER, "user") + .setHeader(CONTENT_TYPE_HEADER, APPLICATION_JSON) + .setBodyGenerator(createStaticBodyGenerator(request, UTF_8)) + .build(), + createJsonResponseHandler(QUERY_RESULTS_CODEC)); + } + + private static StringResponse post(String body, String contentType) + { + return client.execute( + preparePost() + .setUri(server.resolve("/v1/hogql")) + .setHeader(REQUEST_USER_HEADER, "user") + .setHeader(CONTENT_TYPE_HEADER, contentType) + .setBodyGenerator(createStaticBodyGenerator(body, UTF_8)) + .build(), + createStringResponseHandler()); + } + + private static String hogQlRequest(String query) + { + return """ + {"query":%s,"protocolVersion":1,"languageVersion":"%s"} + """.formatted(STRING_CODEC.toJson(query), HogQlLanguageContract.current().languageVersion()); + } + + private static String hogQlRequestWithTypedValues(String query) + { + return """ + { + "query": %s, + "protocolVersion": 1, + "languageVersion": "%s", + "parameters": {}, + "variables": {"array": {"type": "array", "value": [true, "value", null]}}, + "filters": {"object": {"type": "object", "value": {"nested": 2.5}}}, + "modifiers": {}, + "catalogGeneration": 1 + } + """.formatted(STRING_CODEC.toJson(query), HogQlLanguageContract.current().languageVersion()); + } + + private static String hogQlRequestWithScopedBindings(String query) + { + return """ + { + "query": %s, + "protocolVersion": 1, + "languageVersion": "%s", + "variables": {"organization_id": {"type": "bigint", "value": 42}}, + "filters": {"date_from": {"type": "varchar", "value": "2026-01-01"}} + } + """.formatted(STRING_CODEC.toJson(query), HogQlLanguageContract.current().languageVersion()); + } + + private static String hogQlExplainRequestWithScopedBinding(String query) + { + return """ + { + "query": %s, + "protocolVersion": 1, + "languageVersion": "%s", + "variables": {"name": {"type": "varchar", "value": "ALGERIA"}}, + "explain": {"type": "LOGICAL", "format": "TEXT"} + } + """.formatted(STRING_CODEC.toJson(query), HogQlLanguageContract.current().languageVersion()); + } + + private static String hogQlRequestWithModifier(String query) + { + return """ + { + "query": %s, + "protocolVersion": 1, + "languageVersion": "%s", + "modifiers": {"sampling": {"type": "boolean", "value": true}} + } + """.formatted(STRING_CODEC.toJson(query), HogQlLanguageContract.current().languageVersion()); + } + + private static String hogQlExplainRequest(String query, String type, String format) + { + return """ + {"query":%s,"protocolVersion":1,"languageVersion":"%s","explain":{"type":%s,"format":%s}} + """.formatted( + STRING_CODEC.toJson(query), + HogQlLanguageContract.current().languageVersion(), + STRING_CODEC.toJson(type), + STRING_CODEC.toJson(format)); + } + + private static List> rows(List results) + throws Exception + { + ImmutableList.Builder> rows = ImmutableList.builder(); + try (ResultRowsDecoder decoder = new ResultRowsDecoder()) { + for (QueryResults result : results) { + if (result.getData() != null) { + rows.addAll(decoder.toRows(result)); + } + } + } + return rows.build(); + } + + private static List resultColumns(List results) + { + return results.stream() + .map(QueryResults::getColumns) + .filter(columns -> columns != null) + .findFirst() + .orElseThrow(); + } +} diff --git a/core/trino-main/src/test/java/io/trino/execution/BenchmarkQueryPreparer.java b/core/trino-main/src/test/java/io/trino/execution/BenchmarkQueryPreparer.java new file mode 100644 index 000000000000..f84f03b4e85e --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/execution/BenchmarkQueryPreparer.java @@ -0,0 +1,65 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.execution; + +import io.trino.hogql.compiler.HogQlCompiler; +import io.trino.sql.parser.SqlParser; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.runner.RunnerException; + +import java.util.concurrent.TimeUnit; + +import static io.trino.SessionTestUtils.TEST_SESSION; +import static io.trino.jmh.Benchmarks.benchmark; + +@State(Scope.Thread) +@OutputTimeUnit(TimeUnit.MICROSECONDS) +@BenchmarkMode(Mode.AverageTime) +@Fork(1) +@Warmup(iterations = 5, time = 500, timeUnit = TimeUnit.MILLISECONDS) +@Measurement(iterations = 10, time = 500, timeUnit = TimeUnit.MILLISECONDS) +public class BenchmarkQueryPreparer +{ + private static final String QUERY = "SELECT event, properties FROM ducklake.default.events"; + + private final SqlParser sqlParser = new SqlParser(); + private final QueryPreparer sqlOnlyQueryPreparer = new QueryPreparer(sqlParser); + private final QueryPreparer hogQlCapableQueryPreparer = new QueryPreparer(sqlParser, new HogQlCompiler()); + + @Benchmark + public QueryPreparer.PreparedQuery prepareStandardSqlWithoutHogQl() + { + return sqlOnlyQueryPreparer.prepareQuery(TEST_SESSION, QUERY); + } + + @Benchmark + public QueryPreparer.PreparedQuery prepareStandardSqlWithHogQlAvailable() + { + return hogQlCapableQueryPreparer.prepareQuery(TEST_SESSION, QUERY); + } + + static void main() + throws RunnerException + { + benchmark(BenchmarkQueryPreparer.class).run(); + } +} diff --git a/core/trino-main/src/test/java/io/trino/execution/TestHogQlCompilationExecutor.java b/core/trino-main/src/test/java/io/trino/execution/TestHogQlCompilationExecutor.java new file mode 100644 index 000000000000..1c07831d0a13 --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/execution/TestHogQlCompilationExecutor.java @@ -0,0 +1,223 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.execution; + +import io.airlift.units.Duration; +import io.trino.hogql.HogQlCompilationEvent; +import io.trino.hogql.HogQlConfig; +import io.trino.hogql.compiler.HogQlCompileEnvelope; +import io.trino.hogql.compiler.HogQlCompiler; +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.spi.TrinoException; +import io.trino.sql.parser.SqlParser; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; + +import static io.trino.execution.QuerySubmission.hogQl; +import static io.trino.hogql.HogQlCompilationEvent.Outcome.INSUFFICIENT_RESOURCES; +import static io.trino.hogql.HogQlCompilationEvent.Phase.COMPILATION; +import static io.trino.hogql.HogQlCoordinatorErrorCode.HOGQL_COMPILATION_QUEUE_FULL; +import static io.trino.hogql.HogQlCoordinatorErrorCode.HOGQL_COMPILATION_TIMEOUT; +import static io.trino.testing.TestingSession.testSessionBuilder; +import static java.util.concurrent.Executors.newSingleThreadExecutor; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlCompilationExecutor +{ + @Test + public void testCompilationRunsOnDedicatedWorker() + { + HogQlCompilationExecutor executor = new HogQlCompilationExecutor(config(1, 0)); + try { + assertThat(executor.execute(() -> Thread.currentThread().getName())) + .startsWith("hogql-compilation-"); + } + finally { + executor.shutdown(); + } + } + + @Test + public void testSaturationRejectsHogQlWithoutAffectingTrinoSql() + throws Exception + { + HogQlCompilationExecutor executor = new HogQlCompilationExecutor(config(1, 0)); + CountDownLatch workerStarted = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + List events = new ArrayList<>(); + QueryPreparer queryPreparer = new QueryPreparer( + new SqlParser(), + new HogQlCompiler(), + events::add, + Optional.empty(), + executor); + + ExecutorService caller = newSingleThreadExecutor(); + try { + Future occupiedWorker = caller.submit(() -> executor.execute(() -> { + workerStarted.countDown(); + await(releaseWorker); + return null; + })); + assertThat(workerStarted.await(10, SECONDS)).isTrue(); + + try { + assertThatThrownBy(() -> queryPreparer.prepareQuery(testSessionBuilder().build(), hogQl(envelope("SELECT 1")))) + .isInstanceOfSatisfying(TrinoException.class, exception -> { + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_COMPILATION_QUEUE_FULL.toErrorCode()); + assertThat(exception).hasMessageContaining("retry"); + }); + assertThat(queryPreparer.prepareQuery(testSessionBuilder().build(), "SELECT 1").getStatement()).isNotNull(); + + assertThat(events).singleElement().satisfies(event -> { + assertThat(event.outcome()).isEqualTo(INSUFFICIENT_RESOURCES); + assertThat(event.failedPhase()).contains(COMPILATION); + }); + } + finally { + releaseWorker.countDown(); + } + + occupiedWorker.get(10, SECONDS); + } + finally { + releaseWorker.countDown(); + caller.shutdownNow(); + executor.shutdown(); + } + } + + @Test + public void testShutdownInterruptsWorkAndRejectsNewCompilations() + throws Exception + { + HogQlCompilationExecutor executor = new HogQlCompilationExecutor(config(1, 1)); + CountDownLatch workerStarted = new CountDownLatch(1); + CountDownLatch workerInterrupted = new CountDownLatch(1); + + ExecutorService caller = newSingleThreadExecutor(); + try { + Future occupiedWorker = caller.submit(() -> executor.execute(() -> { + workerStarted.countDown(); + try { + new CountDownLatch(1).await(); + } + catch (InterruptedException _) { + workerInterrupted.countDown(); + Thread.currentThread().interrupt(); + } + return null; + })); + assertThat(workerStarted.await(10, SECONDS)).isTrue(); + + executor.shutdown(); + + assertThat(workerInterrupted.await(10, SECONDS)).isTrue(); + try { + occupiedWorker.get(10, SECONDS); + } + catch (ExecutionException exception) { + assertThat(exception.getCause()) + .isInstanceOfSatisfying(TrinoException.class, cause -> + assertThat(cause.getErrorCode()).isEqualTo(HOGQL_COMPILATION_QUEUE_FULL.toErrorCode())); + } + assertThat(occupiedWorker).isDone(); + assertThat(executor.isShutdown()).isTrue(); + assertThatThrownBy(() -> executor.execute(() -> null)) + .isInstanceOfSatisfying(TrinoException.class, exception -> + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_COMPILATION_QUEUE_FULL.toErrorCode())); + } + finally { + caller.shutdownNow(); + executor.shutdown(); + } + } + + @Test + public void testCompilationTimeoutInterruptsWorker() + throws Exception + { + HogQlCompilationExecutor executor = new HogQlCompilationExecutor(config(1, 0) + .setCompilationTimeout(new Duration(10, MILLISECONDS))); + CountDownLatch workerInterrupted = new CountDownLatch(1); + try { + assertThatThrownBy(() -> executor.execute(() -> { + awaitInterruption(workerInterrupted); + return null; + })) + .isInstanceOfSatisfying(TrinoException.class, exception -> { + assertThat(exception.getErrorCode()).isEqualTo(HOGQL_COMPILATION_TIMEOUT.toErrorCode()); + assertThat(exception).hasMessageContaining("time limit"); + }); + assertThat(workerInterrupted.await(10, SECONDS)).isTrue(); + } + finally { + executor.shutdown(); + } + } + + private static HogQlConfig config(int threads, int queueCapacity) + { + return new HogQlConfig() + .setCompilationThreads(threads) + .setCompilationQueueCapacity(queueCapacity); + } + + private static HogQlCompileEnvelope envelope(String query) + { + return new HogQlCompileEnvelope( + query, + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + Map.of(), + Map.of(), + Map.of(), + Map.of(), + OptionalLong.empty()); + } + + private static void await(CountDownLatch latch) + { + try { + latch.await(); + } + catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + throw new RuntimeException(exception); + } + } + + private static void awaitInterruption(CountDownLatch interrupted) + { + try { + new CountDownLatch(1).await(); + } + catch (InterruptedException exception) { + interrupted.countDown(); + Thread.currentThread().interrupt(); + } + } +} diff --git a/core/trino-main/src/test/java/io/trino/execution/TestHogQlQueryPreparerObservability.java b/core/trino-main/src/test/java/io/trino/execution/TestHogQlQueryPreparerObservability.java new file mode 100644 index 000000000000..33bb59adbd15 --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/execution/TestHogQlQueryPreparerObservability.java @@ -0,0 +1,93 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.execution; + +import io.trino.hogql.HogQlCompilationEvent; +import io.trino.hogql.compiler.HogQlCompileEnvelope; +import io.trino.hogql.compiler.HogQlCompiler; +import io.trino.hogql.compiler.HogQlTypedValue; +import io.trino.hogql.compiler.HogQlTypedValue.StringValue; +import io.trino.sql.parser.SqlParser; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.OptionalLong; + +import static io.trino.execution.QuerySubmission.hogQl; +import static io.trino.hogql.HogQlCompilationEvent.Outcome.SUCCESS; +import static io.trino.hogql.HogQlCompilationEvent.Outcome.USER_ERROR; +import static io.trino.hogql.HogQlCompilationEvent.Phase.COMPILATION; +import static io.trino.hogql.HogQlCompilationEvent.Phase.PARAMETER_BINDING; +import static io.trino.testing.TestingSession.testSessionBuilder; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlQueryPreparerObservability +{ + private static final String SECRET = "sensitive-observability-value"; + + @Test + public void testHogQlPreparationEmitsOneRedactedEventAndTrinoDoesNot() + { + List events = new ArrayList<>(); + QueryPreparer queryPreparer = new QueryPreparer(new SqlParser(), new HogQlCompiler(), events::add); + + queryPreparer.prepareQuery(testSessionBuilder().build(), hogQl(envelope("SELECT {input}", Map.of("input", typedValue())))); + queryPreparer.prepareQuery(testSessionBuilder().build(), "SELECT '" + SECRET + "'"); + + assertThat(events).singleElement().satisfies(event -> { + assertThat(event.outcome()).isEqualTo(SUCCESS); + assertThat(event.phaseNanos()).containsKeys(COMPILATION, PARAMETER_BINDING); + assertThat(event.toString()).doesNotContain(SECRET); + }); + } + + @Test + public void testCompilerFailureRecordsRedactedOutcomeAndPhase() + { + List events = new ArrayList<>(); + QueryPreparer queryPreparer = new QueryPreparer(new SqlParser(), new HogQlCompiler(), events::add); + + assertThatThrownBy(() -> queryPreparer.prepareQuery( + testSessionBuilder().build(), + hogQl(envelope("SELECT '" + SECRET, Map.of())))) + .isInstanceOf(RuntimeException.class); + + assertThat(events).singleElement().satisfies(event -> { + assertThat(event.outcome()).isEqualTo(USER_ERROR); + assertThat(event.failedPhase()).contains(COMPILATION); + assertThat(event.toString()).doesNotContain(SECRET); + }); + } + + private static HogQlCompileEnvelope envelope(String query, Map parameters) + { + return new HogQlCompileEnvelope( + query, + HogQlCompileEnvelope.PROTOCOL_VERSION, + io.trino.hogql.parser.HogQlLanguageContract.current().languageVersion(), + parameters, + Map.of(), + Map.of(), + Map.of(), + OptionalLong.empty()); + } + + private static HogQlTypedValue typedValue() + { + return new HogQlTypedValue("varchar", new StringValue(SECRET)); + } +} diff --git a/core/trino-main/src/test/java/io/trino/execution/TestHogQlQueryPreparerSemanticCatalog.java b/core/trino-main/src/test/java/io/trino/execution/TestHogQlQueryPreparerSemanticCatalog.java new file mode 100644 index 000000000000..beba8818f44c --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/execution/TestHogQlQueryPreparerSemanticCatalog.java @@ -0,0 +1,160 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.execution; + +import io.trino.execution.QueryPreparer.PreparedQuery; +import io.trino.hogql.HogQlCompilationEvent; +import io.trino.hogql.compiler.HogQlCompileEnvelope; +import io.trino.hogql.compiler.HogQlCompiler; +import io.trino.hogql.compiler.HogQlErrorCode; +import io.trino.hogql.compiler.HogQlTypedValue; +import io.trino.hogql.compiler.HogQlTypedValue.BooleanValue; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinRequest; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.spi.TrinoException; +import io.trino.sql.parser.SqlParser; +import io.trino.sql.tree.Identifier; +import io.trino.sql.tree.Query; +import io.trino.sql.tree.QuerySpecification; +import io.trino.sql.tree.Table; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import static io.trino.execution.QuerySubmission.hogQl; +import static io.trino.hogql.HogQlCompilationObserver.NOOP; +import static io.trino.testing.TestingSession.testSessionBuilder; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlQueryPreparerSemanticCatalog +{ + private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier("analytics", false); + private static final HogQlSemanticCatalogSnapshot SNAPSHOT = new HogQlSemanticCatalogSnapshot( + HogQlSemanticCatalogSnapshot.SCHEMA_VERSION, + HogQlLanguageContract.current().languageVersion(), + CATALOG, + 7, + List.of(new LogicalTableDefinition( + "events", + new PhysicalQualifiedName(CATALOG, new PhysicalIdentifier("default", false), new PhysicalIdentifier("raw_events", false)), + List.of(new LogicalFieldDefinition("event", new PhysicalIdentifier("event_name", false), "varchar", LogicalType.STRING, false, true)), + List.of(), + List.of()))); + @Test + public void testSessionCatalogPinsSemanticSnapshotAndResolvesLogicalTable() + { + AtomicReference pinRequest = new AtomicReference<>(); + AtomicReference compilationEvent = new AtomicReference<>(); + QueryPreparer queryPreparer = new QueryPreparer( + new SqlParser(), + new HogQlCompiler(), + compilationEvent::set, + Optional.of(request -> { + pinRequest.set(request); + return new PinnedSnapshot(SNAPSHOT); + })); + + PreparedQuery preparedQuery = queryPreparer.prepareQuery( + testSessionBuilder().setCatalog("analytics").build(), + hogQl(envelope("SELECT event FROM events", OptionalLong.of(7)))); + + assertThat(pinRequest.get()).isEqualTo(new PinRequest(CATALOG, HogQlLanguageContract.current().languageVersion(), OptionalLong.of(7))); + assertThat(compilationEvent.get().dimensions().catalogGeneration()).hasValue(7); + QuerySpecification query = (QuerySpecification) ((Query) preparedQuery.getStatement()).getQueryBody(); + assertThat(((Table) query.getFrom().orElseThrow()).getName().getOriginalParts()) + .extracting(Identifier::getValue) + .containsExactly("analytics", "default", "raw_events"); + } + + @Test + public void testMissingSessionCatalogOrProviderDoesNotFetchMetadata() + { + AtomicInteger pins = new AtomicInteger(); + QueryPreparer withProvider = new QueryPreparer( + new SqlParser(), + new HogQlCompiler(), + NOOP, + Optional.of(_ -> { + pins.incrementAndGet(); + return new PinnedSnapshot(SNAPSHOT); + })); + QueryPreparer withoutProvider = new QueryPreparer(new SqlParser(), new HogQlCompiler(), NOOP, Optional.empty()); + + withProvider.prepareQuery(testSessionBuilder().build(), hogQl(envelope("SELECT 1", OptionalLong.empty()))); + withoutProvider.prepareQuery(testSessionBuilder().setCatalog("analytics").build(), hogQl(envelope("SELECT 1", OptionalLong.empty()))); + + assertThat(pins).hasValue(0); + } + + @Test + public void testV0RejectsModifiersBeforeCatalogResolution() + { + AtomicInteger pins = new AtomicInteger(); + QueryPreparer queryPreparer = new QueryPreparer( + new SqlParser(), + new HogQlCompiler(), + NOOP, + Optional.of(_ -> { + pins.incrementAndGet(); + return new PinnedSnapshot(SNAPSHOT); + })); + + assertThatThrownBy(() -> queryPreparer.prepareQuery( + testSessionBuilder().setCatalog("analytics").build(), + hogQl(envelope( + "SELECT 1", + OptionalLong.empty(), + Map.of("sampling", new HogQlTypedValue("boolean", new BooleanValue(true))))))) + .isInstanceOfSatisfying(TrinoException.class, exception -> { + assertThat(exception.getErrorCode()).isEqualTo(HogQlErrorCode.HOGQL_UNSUPPORTED_FEATURE.toErrorCode()); + assertThat(exception.getLocation()).hasValueSatisfying(location -> { + assertThat(location.lineNumber()).isEqualTo(1); + assertThat(location.columnNumber()).isEqualTo(1); + }); + assertThat(exception).hasMessageContaining("Modifiers are outside the HogQL v0 profile"); + }); + assertThat(pins).hasValue(0); + } + + private static HogQlCompileEnvelope envelope(String query, OptionalLong generation) + { + return envelope(query, generation, Map.of()); + } + + private static HogQlCompileEnvelope envelope(String query, OptionalLong generation, Map modifiers) + { + return new HogQlCompileEnvelope( + query, + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + Map.of(), + Map.of(), + Map.of(), + modifiers, + generation); + } +} diff --git a/core/trino-main/src/test/java/io/trino/execution/TestQueryPreparer.java b/core/trino-main/src/test/java/io/trino/execution/TestQueryPreparer.java index a45c0673617f..614507592331 100644 --- a/core/trino-main/src/test/java/io/trino/execution/TestQueryPreparer.java +++ b/core/trino-main/src/test/java/io/trino/execution/TestQueryPreparer.java @@ -15,15 +15,47 @@ import io.trino.Session; import io.trino.execution.QueryPreparer.PreparedQuery; +import io.trino.hogql.compiler.HogQlCompileEnvelope; +import io.trino.hogql.compiler.HogQlCompiler; +import io.trino.hogql.compiler.HogQlErrorCode; +import io.trino.hogql.compiler.HogQlTypedValue; +import io.trino.hogql.compiler.HogQlTypedValue.ArrayValue; +import io.trino.hogql.compiler.HogQlTypedValue.BooleanValue; +import io.trino.hogql.compiler.HogQlTypedValue.NullValue; +import io.trino.hogql.compiler.HogQlTypedValue.NumberValue; +import io.trino.hogql.compiler.HogQlTypedValue.ObjectValue; +import io.trino.hogql.compiler.HogQlTypedValue.StringValue; +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.sql.SqlFormatter; import io.trino.sql.parser.ParsingException; import io.trino.sql.parser.SqlParser; import io.trino.sql.tree.AllColumns; +import io.trino.sql.tree.Array; +import io.trino.sql.tree.BinaryLiteral; +import io.trino.sql.tree.Cast; +import io.trino.sql.tree.Expression; +import io.trino.sql.tree.FunctionCall; +import io.trino.sql.tree.GenericLiteral; import io.trino.sql.tree.QualifiedName; +import io.trino.sql.tree.Query; +import io.trino.sql.tree.Row; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.OptionalLong; +import java.util.stream.Stream; import static io.trino.SessionTestUtils.TEST_SESSION; +import static io.trino.execution.QuerySubmission.hogQl; +import static io.trino.execution.QuerySubmission.trino; import static io.trino.spi.StandardErrorCode.INVALID_PARAMETER_USAGE; import static io.trino.spi.StandardErrorCode.NOT_FOUND; +import static io.trino.spi.StandardErrorCode.NOT_SUPPORTED; import static io.trino.sql.QueryUtil.selectList; import static io.trino.sql.QueryUtil.simpleQuery; import static io.trino.sql.QueryUtil.table; @@ -31,17 +63,205 @@ import static io.trino.testing.assertions.TrinoExceptionAssert.assertTrinoExceptionThrownBy; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.params.provider.Arguments.arguments; public class TestQueryPreparer { private static final SqlParser SQL_PARSER = new SqlParser(); private static final QueryPreparer QUERY_PREPARER = new QueryPreparer(SQL_PARSER); + private static final QueryPreparer HOGQL_QUERY_PREPARER = new QueryPreparer(SQL_PARSER, new HogQlCompiler()); @Test public void testSelectStatement() { PreparedQuery preparedQuery = QUERY_PREPARER.prepareQuery(TEST_SESSION, "SELECT * FROM foo"); + PreparedQuery submittedQuery = QUERY_PREPARER.prepareQuery(TEST_SESSION, trino("SELECT * FROM foo")); assertThat(preparedQuery.getStatement()).isEqualTo(simpleQuery(selectList(new AllColumns()), table(QualifiedName.of("foo")))); + assertThat(submittedQuery.getStatement()).isEqualTo(preparedQuery.getStatement()); + assertThat(preparedQuery.getSessionPropertyOverrides()).isEmpty(); + assertThat(submittedQuery.getSessionPropertyOverrides()).isEmpty(); + } + + @Test + public void testHogQlStatement() + { + HogQlCompileEnvelope envelope = envelope("SELECT 1"); + QuerySubmission submission = hogQl(envelope); + PreparedQuery preparedQuery = HOGQL_QUERY_PREPARER.prepareQuery(TEST_SESSION, submission); + assertThat(preparedQuery.getStatement()).isInstanceOf(Query.class); + assertThat(preparedQuery.getSessionPropertyOverrides()).isEmpty(); + assertThat(submission.hogQlEnvelope()).containsSame(envelope); + } + + @Test + public void testHogQlTypedParametersUseThePreparedValueBoundary() + { + Map parameters = new LinkedHashMap<>(); + parameters.put("json_value", typedValue("json", new ObjectValue(Map.of("enabled", new BooleanValue(true), "items", new ArrayValue(List.of(new NumberValue("1"), NullValue.NULL)))))); + parameters.put("row_value", typedValue("row(label varchar, count bigint)", new ObjectValue(Map.of("count", new NumberValue("2"), "label", new StringValue("synthetic-row"))))); + parameters.put("map_value", typedValue("map(varchar, bigint)", new ObjectValue(Map.of("second", NullValue.NULL, "first", new NumberValue("1"))))); + parameters.put("array_value", typedValue("array(bigint)", new ArrayValue(List.of(new NumberValue("1"), NullValue.NULL, new NumberValue("2"))))); + parameters.put("uuid_value", typedValue("uuid", new StringValue("018f6b9d-89f4-7e8a-8f5d-4c621e5d4a33"))); + parameters.put("timestamp_value", typedValue("timestamp(3)", new StringValue("2026-08-27 12:34:56.789"))); + parameters.put("date_value", typedValue("date", new StringValue("2026-08-27"))); + parameters.put("varchar_value", typedValue("varchar", new StringValue("synthetic-text"))); + parameters.put("decimal_value", typedValue("decimal(4, 2)", new NumberValue("12.30"))); + parameters.put("double_value", typedValue("double", new NumberValue("1.25"))); + parameters.put("bigint_value", typedValue("bigint", new NumberValue("42"))); + parameters.put("boolean_value", typedValue("boolean", new BooleanValue(true))); + + PreparedQuery preparedQuery = HOGQL_QUERY_PREPARER.prepareQuery( + TEST_SESSION, + hogQl(envelope( + "SELECT {boolean_value}, {bigint_value}, {double_value}, {decimal_value}, {varchar_value}, {date_value}, {timestamp_value}, {uuid_value}, {array_value}, {map_value}, {row_value}, {json_value}, {varchar_value}", + parameters))); + + assertThat(preparedQuery.getParameters()).hasSize(13); + assertThat(preparedQuery.getParameters()) + .extracting(expression -> expression.getLocation()) + .containsExactlyElementsOf(ParameterExtractor.extractParameters(preparedQuery.getStatement()).stream() + .map(parameter -> parameter.getLocation()) + .toList()); + assertThat(preparedQuery.getParameters().subList(0, 11)).allMatch(Cast.class::isInstance); + assertThat(preparedQuery.getParameters().subList(0, 11)) + .extracting(expression -> ((Cast) expression).getType()) + .containsExactly( + SQL_PARSER.createType("boolean"), + SQL_PARSER.createType("bigint"), + SQL_PARSER.createType("double"), + SQL_PARSER.createType("decimal(4, 2)"), + SQL_PARSER.createType("varchar"), + SQL_PARSER.createType("date"), + SQL_PARSER.createType("timestamp(3)"), + SQL_PARSER.createType("uuid"), + SQL_PARSER.createType("array(bigint)"), + SQL_PARSER.createType("map(varchar, bigint)"), + SQL_PARSER.createType("row(label varchar, count bigint)")); + assertThat(((Cast) preparedQuery.getParameters().get(8)).getExpression()).isInstanceOf(Array.class); + assertThat(((Cast) preparedQuery.getParameters().get(9)).getExpression()).isInstanceOf(FunctionCall.class); + assertThat(((Cast) preparedQuery.getParameters().get(10)).getExpression()).isInstanceOf(Row.class); + assertThat(preparedQuery.getParameters().get(11)) + .isInstanceOfSatisfying(GenericLiteral.class, json -> assertThat(json.getValue()).isEqualTo("{\"enabled\":true,\"items\":[1,null]}")); + assertThat(preparedQuery.getParameters().get(12)).isEqualTo(preparedQuery.getParameters().get(4)); + assertThat(SqlFormatter.formatSql(preparedQuery.getStatement())) + .doesNotContain("synthetic-text", "synthetic-row", "018f6b9d") + .contains("?"); + } + + @ParameterizedTest + @MethodSource("validHogQlLiteralParameters") + public void testHogQlLiteralParametersPreserveStockAstValues(HogQlTypedValue typedValue, Class literalType, String expectedValue) + { + PreparedQuery preparedQuery = HOGQL_QUERY_PREPARER.prepareQuery( + TEST_SESSION, + hogQl(envelope("SELECT {input}", Map.of("input", typedValue)))); + + assertThat(preparedQuery.getParameters()).hasSize(1); + Expression parameter = preparedQuery.getParameters().getFirst(); + Expression literal; + if (typedValue.type().equalsIgnoreCase("json")) { + literal = parameter; + } + else { + assertThat(parameter).isInstanceOf(Cast.class); + Cast cast = (Cast) parameter; + assertThat(cast.getType()).isEqualTo(SQL_PARSER.createType(typedValue.type())); + literal = cast.getExpression(); + } + assertThat(literal).isInstanceOf(literalType); + String literalValue = switch (literal) { + case GenericLiteral genericLiteral -> genericLiteral.getValue(); + case BinaryLiteral binaryLiteral -> binaryLiteral.toHexString(); + default -> throw new AssertionError("unexpected literal type"); + }; + assertThat(literalValue).isEqualTo(expectedValue); + assertThat(SqlFormatter.formatSql(preparedQuery.getStatement())) + .contains("?") + .doesNotContain(expectedValue); + } + + @ParameterizedTest + @MethodSource("invalidHogQlTypedParameters") + public void testHogQlTypedParameterErrorsAreStableAndRedacted(HogQlTypedValue typedValue, String redactedFragment) + { + assertTrinoExceptionThrownBy(() -> HOGQL_QUERY_PREPARER.prepareQuery( + TEST_SESSION, + hogQl(envelope("SELECT {input}", Map.of("input", typedValue))))) + .hasErrorCode(HogQlErrorCode.HOGQL_BINDING_ERROR) + .hasMessage("line 1:8: Invalid HogQL parameter binding: input") + .satisfies(exception -> assertThat(exception.getMessage()).doesNotContain(redactedFragment)); + } + + @Test + public void testHogQlSubmissionFailsClosedWithoutCompiler() + { + assertTrinoExceptionThrownBy(() -> QUERY_PREPARER.prepareQuery(TEST_SESSION, hogQl(envelope("SELECT 1")))) + .hasErrorCode(NOT_SUPPORTED) + .hasMessage("HogQL query submission is disabled"); + } + + private static HogQlCompileEnvelope envelope(String query) + { + return envelope(query, Map.of()); + } + + private static HogQlCompileEnvelope envelope(String query, Map parameters) + { + return new HogQlCompileEnvelope( + query, + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + parameters, + Map.of(), + Map.of(), + Map.of(), + OptionalLong.empty()); + } + + private static Stream invalidHogQlTypedParameters() + { + String sensitiveValue = "sensitive-binding-value"; + return Stream.of( + arguments(typedValue(sensitiveValue, new StringValue(sensitiveValue)), sensitiveValue), + arguments(typedValue("boolean", new StringValue(sensitiveValue)), sensitiveValue), + arguments(typedValue("tinyint", new NumberValue("128")), "128"), + arguments(typedValue("decimal(3, 1)", new NumberValue("123.4")), "123.4"), + arguments(typedValue("uuid", new StringValue(sensitiveValue)), sensitiveValue), + arguments(typedValue("array(bigint)", new ArrayValue(List.of(new StringValue(sensitiveValue)))), sensitiveValue), + arguments(typedValue("map(bigint, varchar)", new ObjectValue(Map.of("field", new StringValue(sensitiveValue)))), sensitiveValue), + arguments(typedValue("row(label varchar)", new ObjectValue(Map.of("unexpected", new StringValue(sensitiveValue)))), sensitiveValue), + arguments(typedValue("time(13)", new StringValue("12:34:56.123456789012")), "12:34:56.123456789012"), + arguments(typedValue("time(12) with time zone", new StringValue("12:34:56.123456789012+15:00")), "+15:00"), + arguments(typedValue("time(12) with time zone", new StringValue("12:34:56.123456789012 America/Toronto")), "America/Toronto"), + arguments(typedValue("timestamp(12) with time zone", new StringValue("2026-03-08 02:30:00.123456789012 America/Toronto")), "America/Toronto"), + arguments(typedValue("timestamp(12) with time zone", new StringValue("2026-08-27 12:34:56.123456789012 Invalid/Zone")), "Invalid/Zone"), + arguments(typedValue("timestamp(12)", new StringValue("2026-08-27 12:34:56.1234567890123")), "1234567890123"), + arguments(typedValue("date", new StringValue("2026-02-29")), "2026-02-29"), + arguments(typedValue("uuid", new StringValue("1-1-1-1-1")), "1-1-1-1-1"), + arguments(typedValue("ipaddress", new StringValue("host.example.com")), "host.example.com"), + arguments(typedValue("varbinary", new StringValue("ABC")), "ABC")); + } + + private static Stream validHogQlLiteralParameters() + { + return Stream.of( + arguments(typedValue("time(0)", new StringValue("00:00")), GenericLiteral.class, "00:00"), + arguments(typedValue("time(10)", new StringValue("12:34:56.1234567890")), GenericLiteral.class, "12:34:56.1234567890"), + arguments(typedValue("time(12) with time zone", new StringValue("12:34:56.123456789012 +05:45")), GenericLiteral.class, "12:34:56.123456789012 +05:45"), + arguments(typedValue("timestamp(0)", new StringValue("2026-08-27 12:34:56")), GenericLiteral.class, "2026-08-27 12:34:56"), + arguments(typedValue("timestamp(10)", new StringValue("2026-08-27 12:34:56.1234567890")), GenericLiteral.class, "2026-08-27 12:34:56.1234567890"), + arguments(typedValue("timestamp(11) with time zone", new StringValue("2026-08-27 12:34:56.12345678901 +05:45")), GenericLiteral.class, "2026-08-27 12:34:56.12345678901 +05:45"), + arguments(typedValue("timestamp(12) with time zone", new StringValue("2026-08-27 12:34:56.123456789012 America/Toronto")), GenericLiteral.class, "2026-08-27 12:34:56.123456789012 America/Toronto"), + arguments(typedValue("date", new StringValue("2000-02-29")), GenericLiteral.class, "2000-02-29"), + arguments(typedValue("uuid", new StringValue("00000000-0000-0000-0000-000000000000")), GenericLiteral.class, "00000000-0000-0000-0000-000000000000"), + arguments(typedValue("ipaddress", new StringValue("64:ff9b::10.0.0.1")), GenericLiteral.class, "64:ff9b::10.0.0.1"), + arguments(typedValue("varbinary", new StringValue("00 ff 10")), BinaryLiteral.class, "00FF10"), + arguments(typedValue("json", new ObjectValue(Map.of("escaped", new StringValue("line\n\u2603"), "number", new NumberValue("12345678901234567890.000000000001"), "null", NullValue.NULL))), GenericLiteral.class, "{\"escaped\":\"line\\n☃\",\"null\":null,\"number\":12345678901234567890.000000000001}")); + } + + private static HogQlTypedValue typedValue(String type, HogQlTypedValue.Value value) + { + return new HogQlTypedValue(type, value); } @Test diff --git a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlCompilationObservability.java b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlCompilationObservability.java new file mode 100644 index 000000000000..422777a044b2 --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlCompilationObservability.java @@ -0,0 +1,160 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import io.trino.hogql.HogQlCompilationEvent.Dimensions; +import io.trino.hogql.HogQlCompilationEvent.Outcome; +import io.trino.hogql.compiler.HogQlCompileEnvelope; +import io.trino.hogql.compiler.HogQlTypedValue; +import io.trino.hogql.compiler.HogQlTypedValue.StringValue; +import io.trino.spi.TrinoException; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.Supplier; +import java.util.stream.Stream; + +import static io.trino.hogql.HogQlCompilationEvent.Outcome.EXTERNAL_ERROR; +import static io.trino.hogql.HogQlCompilationEvent.Outcome.INTERNAL_ERROR; +import static io.trino.hogql.HogQlCompilationEvent.Outcome.SUCCESS; +import static io.trino.hogql.HogQlCompilationEvent.Outcome.USER_ERROR; +import static io.trino.hogql.HogQlCompilationEvent.Phase.BIND; +import static io.trino.hogql.HogQlCompilationEvent.Phase.LOWER; +import static io.trino.hogql.HogQlCompilationEvent.Phase.PARSE; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_CATALOG_NOT_READY; +import static io.trino.hogql.compiler.HogQlErrorCode.HOGQL_SYNTAX_ERROR; +import static io.trino.hogql.parser.HogQlLanguageContract.current; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchRuntimeException; + +public class TestHogQlCompilationObservability +{ + private static final String QUERY_SECRET = "literal-query-secret"; + private static final String VALUE_SECRET = "typed-value-secret"; + private static final String NAME_SECRET = "semantic-name-secret"; + + @Test + public void testEventContainsOnlyRedactedDimensionsAndPhaseTimings() + { + AtomicLong ticker = new AtomicLong(); + List events = new ArrayList<>(); + HogQlCompilationTracker tracker = new HogQlCompilationTracker(events::add, Dimensions.fromEnvelope(envelope()), ticker::get); + + assertThat(tracker.observe(PARSE, () -> { + ticker.addAndGet(11); + return "parsed"; + })).isEqualTo("parsed"); + tracker.observe(BIND, () -> { + ticker.addAndGet(13); + return null; + }); + ticker.addAndGet(17); + tracker.succeeded(); + + assertThat(events).singleElement().satisfies(event -> { + assertThat(event.dimensions()).isEqualTo(new Dimensions( + HogQlCompileEnvelope.PROTOCOL_VERSION, + current().languageVersion(), + 1, + 1, + 1, + 1, + OptionalLong.of(41))); + assertThat(event.outcome()).isEqualTo(SUCCESS); + assertThat(event.failedPhase()).isEmpty(); + assertThat(event.totalNanos()).isEqualTo(41); + assertThat(event.phaseNanos()).containsExactlyInAnyOrderEntriesOf(Map.of(PARSE, 11L, BIND, 13L)); + assertThat(event.toString()).doesNotContain(QUERY_SECRET, VALUE_SECRET, NAME_SECRET); + }); + } + + @ParameterizedTest + @MethodSource("failures") + public void testFailureOutcomeAndPhaseAreStable(Supplier failureSupplier, Outcome expectedOutcome) + { + List events = new ArrayList<>(); + HogQlCompilationTracker tracker = new HogQlCompilationTracker(events::add, Dimensions.fromEnvelope(envelope()), () -> 0); + + RuntimeException failure = catchRuntimeException(() -> tracker.observe(LOWER, () -> { + throw failureSupplier.get(); + })); + tracker.failed(failure); + + assertThat(events).singleElement().satisfies(event -> { + assertThat(event.outcome()).isEqualTo(expectedOutcome); + assertThat(event.failedPhase()).contains(LOWER); + assertThat(event.toString()).doesNotContain(QUERY_SECRET, VALUE_SECRET, NAME_SECRET); + }); + } + + @Test + public void testResolvedCatalogGenerationReplacesRequestedGeneration() + { + List events = new ArrayList<>(); + HogQlCompilationTracker tracker = new HogQlCompilationTracker(events::add, Dimensions.fromEnvelope(envelope()), () -> 0); + + tracker.catalogGeneration(OptionalLong.of(43)); + tracker.succeeded(); + + assertThat(events).singleElement() + .extracting(event -> event.dimensions().catalogGeneration()) + .isEqualTo(OptionalLong.of(43)); + } + + @Test + public void testStatsRecordOutcomes() + { + HogQlCompilationStats stats = new HogQlCompilationStats(); + Dimensions dimensions = Dimensions.fromEnvelope(envelope()); + + stats.compilationCompleted(new HogQlCompilationEvent(dimensions, SUCCESS, Optional.empty(), 100, Map.of(PARSE, 10L, BIND, 20L, LOWER, 30L))); + stats.compilationCompleted(new HogQlCompilationEvent(dimensions, USER_ERROR, Optional.of(PARSE), 50, Map.of(PARSE, 40L))); + + assertThat(stats.getCompletedCompilations().getTotalCount()).isEqualTo(2); + assertThat(stats.getSuccessfulCompilations().getTotalCount()).isEqualTo(1); + assertThat(stats.getUserErrorFailures().getTotalCount()).isEqualTo(1); + assertThat(stats.getInternalErrorFailures().getTotalCount()).isZero(); + assertThat(stats.getLastCatalogGeneration()).isEqualTo(41); + } + + private static Stream failures() + { + return Stream.of( + Arguments.of((Supplier) () -> new TrinoException(HOGQL_SYNTAX_ERROR, "synthetic syntax failure"), USER_ERROR), + Arguments.of((Supplier) () -> new TrinoException(HOGQL_CATALOG_NOT_READY, "synthetic catalog failure"), EXTERNAL_ERROR), + Arguments.of((Supplier) () -> new IllegalStateException("synthetic internal failure"), INTERNAL_ERROR)); + } + + private static HogQlCompileEnvelope envelope() + { + HogQlTypedValue typedValue = new HogQlTypedValue("varchar", new StringValue(VALUE_SECRET)); + return new HogQlCompileEnvelope( + "SELECT '" + QUERY_SECRET + "'", + HogQlCompileEnvelope.PROTOCOL_VERSION, + current().languageVersion(), + Map.of(NAME_SECRET + "-parameter", typedValue), + Map.of(NAME_SECRET + "-variable", typedValue), + Map.of(NAME_SECRET + "-filter", typedValue), + Map.of(NAME_SECRET + "-modifier", typedValue), + OptionalLong.of(41)); + } +} diff --git a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlConfig.java b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlConfig.java new file mode 100644 index 000000000000..9d62fb36efd6 --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlConfig.java @@ -0,0 +1,53 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import io.airlift.units.Duration; +import org.junit.jupiter.api.Test; + +import java.util.Map; + +import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static java.util.concurrent.TimeUnit.SECONDS; + +public class TestHogQlConfig +{ + @Test + public void testDefaults() + { + assertRecordedDefaults(recordDefaults(HogQlConfig.class) + .setEnabled(false) + .setCompilationThreads(2) + .setCompilationQueueCapacity(32) + .setCompilationTimeout(new Duration(10, SECONDS))); + } + + @Test + public void testExplicitPropertyMappings() + { + assertFullMapping( + Map.of( + "hogql.enabled", "true", + "hogql.compilation-threads", "4", + "hogql.compilation-queue-capacity", "128", + "hogql.compilation-timeout", "3s"), + new HogQlConfig() + .setEnabled(true) + .setCompilationThreads(4) + .setCompilationQueueCapacity(128) + .setCompilationTimeout(new Duration(3, SECONDS))); + } +} diff --git a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlExchangeRateConversionEngine.java b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlExchangeRateConversionEngine.java new file mode 100644 index 000000000000..0451fe30a05a --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlExchangeRateConversionEngine.java @@ -0,0 +1,173 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import io.airlift.http.client.testing.TestingHttpClient; +import io.airlift.http.client.testing.TestingResponse; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshot; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshot.ExchangeRate; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotJsonDecoder; +import io.trino.metadata.InternalFunctionBundle; +import io.trino.spi.type.Int128; +import io.trino.sql.query.QueryAssertions; +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.time.LocalDate; +import java.util.List; +import java.util.OptionalLong; +import java.util.concurrent.atomic.AtomicInteger; + +import static com.google.common.net.MediaType.JSON_UTF_8; +import static io.airlift.http.client.HttpStatus.OK; +import static io.airlift.http.client.testing.TestingResponse.contentType; +import static io.airlift.slice.Slices.utf8Slice; +import static io.trino.spi.type.Decimals.encodeScaledValue; +import static io.trino.spi.type.DecimalType.createDecimalType; +import static io.trino.spi.type.SqlDecimal.decimal; +import static org.assertj.core.api.Assertions.assertThat; + +final class TestHogQlExchangeRateConversionEngine +{ + private static final LocalDate FIRST_DATE = LocalDate.parse("2024-01-01"); + private static final LocalDate SECOND_DATE = LocalDate.parse("2024-01-10"); + + private final HogQlExchangeRateConversionEngine engine = new HogQlExchangeRateConversionEngine(new HogQlExchangeRateSnapshot( + 1, + 1, + 17, + "USD", + 10, + List.of( + rate("EUR", FIRST_DATE, "0.9049"), + rate("EUR", SECOND_DATE, "0.95"), + rate("JPY", FIRST_DATE, "108.1234"), + rate("USD", FIRST_DATE, "1"), + rate("ZZZ", FIRST_DATE, "0")))); + + @Test + void testFloorLookupUsesLatestRateNotAfterDate() + { + assertThat(engine.rate("EUR", FIRST_DATE.minusDays(1))).isEmpty(); + assertThat(engine.rate("EUR", FIRST_DATE)).contains(new BigDecimal("0.9049000000")); + assertThat(engine.rate("EUR", SECOND_DATE.minusDays(1))).contains(new BigDecimal("0.9049000000")); + assertThat(engine.rate("EUR", SECOND_DATE.plusYears(1))).contains(new BigDecimal("0.9500000000")); + } + + @Test + void testConversionMatchesPostHogDecimalSemantics() + { + assertDecimal(engine.convert("USD", "EUR", new BigDecimal("100"), FIRST_DATE), "90.4900000000"); + assertDecimal(engine.convert("EUR", "USD", new BigDecimal("90.49"), FIRST_DATE), "100.0000000000"); + assertDecimal(engine.convert("AAA", "AAA", new BigDecimal("1.23456789019"), FIRST_DATE), "1.2345678901"); + assertDecimal(engine.convert("USD", "JPY", new BigDecimal("-2.5"), FIRST_DATE), "-270.3085000000"); + } + + @Test + void testMissingOrZeroRateReturnsZero() + { + assertDecimal(engine.convert("MISSING", "EUR", BigDecimal.TEN, FIRST_DATE), "0.0000000000"); + assertDecimal(engine.convert("USD", "MISSING", BigDecimal.TEN, FIRST_DATE), "0.0000000000"); + assertDecimal(engine.convert("ZZZ", "EUR", BigDecimal.TEN, FIRST_DATE), "0.0000000000"); + assertDecimal(engine.convert("EUR", "USD", BigDecimal.TEN, FIRST_DATE.minusDays(1)), "0.0000000000"); + } + + @Test + void testRuntimeStateAndCompilerPinsReuseCachedGeneration() + { + String snapshot = """ + {"protocolVersion":1,"schemaVersion":1,"generation":17,"baseCurrency":"USD","decimalScale":10,"rates":[ + {"currency":"EUR","effectiveDate":"2024-01-01","unscaledRate":"9049000000"}, + {"currency":"USD","effectiveDate":"2024-01-01","unscaledRate":"10000000000"}]} + """; + AtomicInteger requests = new AtomicInteger(); + TestingHttpClient client = new TestingHttpClient(_ -> { + requests.incrementAndGet(); + return new TestingResponse( + OK, + contentType(JSON_UTF_8), + snapshot.getBytes(StandardCharsets.UTF_8)); + }); + HogQlExchangeRateHttpTransport transport = new HogQlExchangeRateHttpTransport( + URI.create("https://duckgres.example/"), + client, + () -> "test-token"); + HogQlExchangeRateManager manager = new HogQlExchangeRateManager( + new HogQlSemanticCatalogConfig(), + transport, + new HogQlExchangeRateSnapshotJsonDecoder()); + try { + manager.pin(OptionalLong.empty()); + manager.pin(OptionalLong.empty()); + HogQlExchangeRateFunction.State state = new HogQlExchangeRateFunction.State(manager); + Int128 result = state.convert( + 17, + utf8Slice("USD"), + utf8Slice("EUR"), + encodeScaledValue(new BigDecimal("100"), 10), + FIRST_DATE.toEpochDay()); + + assertDecimal(new BigDecimal(result.toBigInteger(), 10), "90.4900000000"); + assertThat(requests).hasValue(1); + } + finally { + manager.shutdown(); + } + } + + @Test + void testRegisteredScalarFunctionExecutesPinnedConversion() + { + String snapshot = """ + {"protocolVersion":1,"schemaVersion":1,"generation":17,"baseCurrency":"USD","decimalScale":10,"rates":[ + {"currency":"EUR","effectiveDate":"2024-01-01","unscaledRate":"9049000000"}, + {"currency":"USD","effectiveDate":"2024-01-01","unscaledRate":"10000000000"}]} + """; + TestingHttpClient client = new TestingHttpClient(_ -> new TestingResponse( + OK, + contentType(JSON_UTF_8), + snapshot.getBytes(StandardCharsets.UTF_8))); + HogQlExchangeRateManager manager = new HogQlExchangeRateManager( + new HogQlSemanticCatalogConfig(), + new HogQlExchangeRateHttpTransport(URI.create("https://duckgres.example/"), client, () -> "test-token"), + new HogQlExchangeRateSnapshotJsonDecoder()); + try { + manager.pin(OptionalLong.empty()); + try (QueryAssertions assertions = new QueryAssertions()) { + assertions.addFunctions(new InternalFunctionBundle(new HogQlExchangeRateFunction(manager))); + + assertThat(assertions.expression( + "hogql_convert_currency(17, 'USD', 'EUR', DECIMAL '100.0000000000', DATE '2024-01-01')")) + .hasType(createDecimalType(38, 10)) + .isEqualTo(decimal("90.4900000000", createDecimalType(38, 10))); + } + } + finally { + manager.shutdown(); + } + } + + private static ExchangeRate rate(String currency, LocalDate date, String value) + { + return new ExchangeRate(currency, date.toString(), new BigDecimal(value).movePointRight(10).toBigIntegerExact().toString()); + } + + private static void assertDecimal(BigDecimal actual, String expected) + { + assertThat(actual.scale()).isEqualTo(10); + assertThat(actual).isEqualByComparingTo(expected); + } +} diff --git a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlExchangeRateHttpTransport.java b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlExchangeRateHttpTransport.java new file mode 100644 index 000000000000..e8507c837cc4 --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlExchangeRateHttpTransport.java @@ -0,0 +1,165 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import com.google.common.collect.ImmutableListMultimap; +import io.airlift.http.client.HttpStatus; +import io.airlift.http.client.Request; +import io.airlift.http.client.Response; +import io.airlift.http.client.testing.TestingHttpClient; +import io.airlift.http.client.testing.TestingResponse; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateException; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateException.Failure; +import io.trino.hogql.compiler.catalog.HogQlExchangeRateSnapshotLoader.LoadRequest; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; + +import static com.google.common.net.MediaType.JSON_UTF_8; +import static io.airlift.http.client.HeaderNames.ACCEPT; +import static io.airlift.http.client.HttpStatus.CONFLICT; +import static io.airlift.http.client.HttpStatus.INTERNAL_SERVER_ERROR; +import static io.airlift.http.client.HttpStatus.NOT_FOUND; +import static io.airlift.http.client.HttpStatus.OK; +import static io.airlift.http.client.testing.TestingResponse.contentType; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlExchangeRateHttpTransport +{ + private static final URI BASE_URI = URI.create("https://duckgres.example/control-plane/"); + private static final String SECRET = "secret-metadata-response"; + private static final String AUTHENTICATION_TOKEN = "test-authentication-token"; + + @Test + public void testConstructsAuthenticatedLatestAndExactRequests() + { + List requests = new ArrayList<>(); + TestingHttpClient httpClient = new TestingHttpClient(request -> { + requests.add(request); + return response(OK, "{}"); + }); + HogQlExchangeRateHttpTransport transport = new HogQlExchangeRateHttpTransport(BASE_URI, httpClient, () -> AUTHENTICATION_TOKEN); + + assertThat(transport.load(LoadRequest.latest()).toCompletableFuture().join()).isEqualTo("{}".getBytes(StandardCharsets.UTF_8)); + assertThat(transport.load(LoadRequest.pinned(42)).toCompletableFuture().join()).isEqualTo("{}".getBytes(StandardCharsets.UTF_8)); + + assertThat(requests).extracting(request -> request.getUri().toASCIIString()).containsExactly( + "https://duckgres.example/control-plane/v1/hogql/compatibility/exchange-rates?protocolVersion=1", + "https://duckgres.example/control-plane/v1/hogql/compatibility/exchange-rates?protocolVersion=1&generation=42"); + assertThat(requests).allSatisfy(request -> { + assertThat(request.getMethod()).isEqualTo("GET"); + assertThat(request.getHeader(ACCEPT)).isEqualTo(JSON_UTF_8.withoutParameters().toString()); + assertThat(request.getHeader("X-Duckgres-Internal-Secret")).isEqualTo(AUTHENTICATION_TOKEN); + }); + } + + @Test + public void testReadsAuthenticationTokenForEveryRequest() + { + List requests = new ArrayList<>(); + List tokens = new ArrayList<>(List.of("first-token", "second-token")); + TestingHttpClient httpClient = new TestingHttpClient(request -> { + requests.add(request); + return response(OK, "{}"); + }); + HogQlExchangeRateHttpTransport transport = new HogQlExchangeRateHttpTransport(BASE_URI, httpClient, tokens::removeFirst); + + transport.load(LoadRequest.latest()).toCompletableFuture().join(); + transport.load(LoadRequest.latest()).toCompletableFuture().join(); + + assertThat(requests).extracting(request -> request.getHeader("X-Duckgres-Internal-Secret")) + .containsExactly("first-token", "second-token"); + } + + @Test + public void testRejectsOversizedOrNonJsonResponses() + { + HogQlExchangeRateHttpTransport oversized = new HogQlExchangeRateHttpTransport( + BASE_URI, + new TestingHttpClient(_ -> response(OK, "12345")), + 4, + () -> AUTHENTICATION_TOKEN); + assertUnavailable(oversized, LoadRequest.latest()); + + TestingResponse plainText = new TestingResponse(OK, ImmutableListMultimap.of(), "plain text".getBytes(StandardCharsets.UTF_8)); + HogQlExchangeRateHttpTransport nonJson = new HogQlExchangeRateHttpTransport( + BASE_URI, + new TestingHttpClient(_ -> plainText), + () -> AUTHENTICATION_TOKEN); + assertUnavailable(nonJson, LoadRequest.latest()); + } + + @Test + public void testRejectsInvalidAuthenticationWithoutDisclosure() + { + HogQlExchangeRateHttpTransport transport = new HogQlExchangeRateHttpTransport( + BASE_URI, + new TestingHttpClient(_ -> response(OK, "{}")), + () -> "line-one\nline-two"); + + assertThatThrownBy(() -> transport.load(LoadRequest.latest()).toCompletableFuture().join()) + .cause() + .isInstanceOfSatisfying(HogQlExchangeRateException.class, exception -> { + assertThat(exception.failure()).isEqualTo(Failure.UNAVAILABLE); + assertThat(exception).hasMessageNotContaining("line-one"); + }); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("statusFailures") + public void testMapsHttpStatusWithoutResponseDisclosure(String name, HttpStatus status, LoadRequest request, Failure failure) + { + HogQlExchangeRateHttpTransport transport = new HogQlExchangeRateHttpTransport( + BASE_URI, + new TestingHttpClient(_ -> response(status, SECRET)), + () -> AUTHENTICATION_TOKEN); + + assertThatThrownBy(() -> transport.load(request).toCompletableFuture().join()) + .cause() + .isInstanceOfSatisfying(HogQlExchangeRateException.class, exception -> { + assertThat(exception.failure()).isEqualTo(failure); + assertThat(exception).hasMessageNotContaining(SECRET); + assertThat(exception.getMessage()).hasSizeLessThan(128); + }); + } + + private static Stream statusFailures() + { + return Stream.of( + Arguments.of("latest missing", NOT_FOUND, LoadRequest.latest(), Failure.UNAVAILABLE), + Arguments.of("exact missing", NOT_FOUND, LoadRequest.pinned(7), Failure.GENERATION_MISMATCH), + Arguments.of("generation conflict", CONFLICT, LoadRequest.pinned(7), Failure.GENERATION_MISMATCH), + Arguments.of("server failure", INTERNAL_SERVER_ERROR, LoadRequest.latest(), Failure.UNAVAILABLE)); + } + + private static void assertUnavailable(HogQlExchangeRateHttpTransport transport, LoadRequest request) + { + assertThatThrownBy(() -> transport.load(request).toCompletableFuture().join()) + .cause() + .isInstanceOfSatisfying(HogQlExchangeRateException.class, exception -> assertThat(exception.failure()).isEqualTo(Failure.UNAVAILABLE)); + } + + private static Response response(HttpStatus status, String body) + { + return new TestingResponse(status, contentType(JSON_UTF_8), body.getBytes(StandardCharsets.UTF_8)); + } +} diff --git a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogConfig.java b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogConfig.java new file mode 100644 index 000000000000..3d5e647394bf --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogConfig.java @@ -0,0 +1,89 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import io.airlift.units.DataSize; +import io.airlift.units.Duration; +import jakarta.validation.constraints.AssertTrue; +import org.junit.jupiter.api.Test; + +import java.net.URI; +import java.util.Map; + +import static io.airlift.configuration.testing.ConfigAssertions.assertFullMapping; +import static io.airlift.configuration.testing.ConfigAssertions.assertRecordedDefaults; +import static io.airlift.configuration.testing.ConfigAssertions.recordDefaults; +import static io.airlift.testing.ValidationAssertions.assertFailsValidation; +import static io.airlift.units.DataSize.Unit.MEGABYTE; +import static java.util.concurrent.TimeUnit.MINUTES; +import static java.util.concurrent.TimeUnit.SECONDS; + +public class TestHogQlSemanticCatalogConfig +{ + @Test + public void testDefaults() + { + assertRecordedDefaults(recordDefaults(HogQlSemanticCatalogConfig.class) + .setUri(null) + .setAuthenticationTokenFile(null) + .setMaximumEntries(100) + .setRefreshAfter(new Duration(1, MINUTES)) + .setExpireAfter(new Duration(5, MINUTES)) + .setFailureBackoff(new Duration(10, SECONDS)) + .setLoaderThreads(4) + .setLoaderQueueCapacity(64) + .setRequestTimeout(new Duration(10, SECONDS)) + .setMaximumResponseSize(DataSize.of(8, MEGABYTE))); + } + + @Test + public void testExplicitPropertyMappings() + { + Map properties = Map.of( + "hogql.semantic-catalog.uri", "https://duckgres.example/metadata", + "hogql.semantic-catalog.authentication-token-file", "/run/secrets/hogql-catalog-token", + "hogql.semantic-catalog.maximum-entries", "17", + "hogql.semantic-catalog.refresh-after", "2m", + "hogql.semantic-catalog.expire-after", "9m", + "hogql.semantic-catalog.failure-backoff", "20s", + "hogql.semantic-catalog.loader-threads", "3", + "hogql.semantic-catalog.loader-queue-capacity", "41", + "hogql.semantic-catalog.request-timeout", "7s", + "hogql.semantic-catalog.maximum-response-size", "4MB"); + + HogQlSemanticCatalogConfig expected = new HogQlSemanticCatalogConfig() + .setUri(URI.create("https://duckgres.example/metadata")) + .setAuthenticationTokenFile("/run/secrets/hogql-catalog-token") + .setMaximumEntries(17) + .setRefreshAfter(new Duration(2, MINUTES)) + .setExpireAfter(new Duration(9, MINUTES)) + .setFailureBackoff(new Duration(20, SECONDS)) + .setLoaderThreads(3) + .setLoaderQueueCapacity(41) + .setRequestTimeout(new Duration(7, SECONDS)) + .setMaximumResponseSize(DataSize.of(4, MEGABYTE)); + + assertFullMapping(properties, expected); + } + + @Test + public void testUriRequiresAuthenticationTokenFile() + { + assertFailsValidation( + new HogQlSemanticCatalogConfig().setUri(URI.create("https://duckgres.example/metadata")), + "authenticationConfigured", + "hogql.semantic-catalog.authentication-token-file is required when hogql.semantic-catalog.uri is set", + AssertTrue.class); + } +} diff --git a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogGuiceWiring.java b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogGuiceWiring.java new file mode 100644 index 000000000000..54a780cd084b --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogGuiceWiring.java @@ -0,0 +1,74 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import com.google.inject.Key; +import com.google.inject.TypeLiteral; +import io.trino.connector.CatalogLifecycleListener; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotCache; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider; +import io.trino.server.testing.TestingTrinoServer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TestHogQlSemanticCatalogGuiceWiring +{ + private static final Key> OPTIONAL_PROVIDER_KEY = + Key.get(new TypeLiteral<>() {}); + private static final Key> CATALOG_LISTENERS_KEY = + Key.get(new TypeLiteral<>() {}); + + @TempDir + public Path temporaryDirectory; + + @Test + public void testOptionalProviderFollowsHogQlAndCatalogConfiguration() + throws IOException + { + Path tokenFile = temporaryDirectory.resolve("catalog-token"); + Files.writeString(tokenFile, "test-token"); + + try (TestingTrinoServer server = TestingTrinoServer.builder() + .setProperties(Map.of("hogql.enabled", "false")) + .build()) { + assertThat(server.getInstance(OPTIONAL_PROVIDER_KEY)).isEmpty(); + assertThat(server.getInstance(CATALOG_LISTENERS_KEY)).isEmpty(); + } + + try (TestingTrinoServer server = TestingTrinoServer.builder() + .setProperties(Map.of( + "hogql.enabled", "true", + "hogql.semantic-catalog.uri", "https://duckgres.example/metadata", + "hogql.semantic-catalog.authentication-token-file", tokenFile.toString())) + .build()) { + Optional provider = server.getInstance(OPTIONAL_PROVIDER_KEY); + HogQlSemanticCatalogManager manager = server.getInstance(Key.get(HogQlSemanticCatalogManager.class)); + HogQlSemanticCatalogSnapshotCache cache = server.getInstance(Key.get(HogQlSemanticCatalogSnapshotCache.class)); + + assertThat(provider).isPresent(); + assertThat(server.getInstance(CATALOG_LISTENERS_KEY)).hasOnlyElementsOfType(HogQlSemanticCatalogPrewarmListener.class); + assertThat(server.getInstance(OPTIONAL_PROVIDER_KEY)).containsSame(provider.orElseThrow()); + assertThat(cache).isSameAs(manager.cache()); + } + } +} diff --git a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogHttpTransport.java b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogHttpTransport.java new file mode 100644 index 000000000000..56949a298e78 --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogHttpTransport.java @@ -0,0 +1,179 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import io.airlift.http.client.HttpStatus; +import io.airlift.http.client.Request; +import io.airlift.http.client.Response; +import io.airlift.http.client.testing.TestingHttpClient; +import io.airlift.http.client.testing.TestingResponse; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogException; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogException.Failure; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotLoader.LoadRequest; +import io.trino.hogql.parser.HogQlLanguageVersion; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; + +import static com.google.common.net.MediaType.JSON_UTF_8; +import static io.airlift.http.client.HeaderNames.ACCEPT; +import static io.airlift.http.client.HttpStatus.CONFLICT; +import static io.airlift.http.client.HttpStatus.INTERNAL_SERVER_ERROR; +import static io.airlift.http.client.HttpStatus.NOT_FOUND; +import static io.airlift.http.client.HttpStatus.OK; +import static io.airlift.http.client.testing.TestingResponse.contentType; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlSemanticCatalogHttpTransport +{ + private static final HogQlLanguageVersion LANGUAGE_VERSION = HogQlLanguageVersion.valueOf("1.0.0"); + private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier("Sales & Growth/2026", true); + private static final URI BASE_URI = URI.create("https://duckgres.example/control-plane/"); + private static final String SECRET = "secret-metadata-response"; + private static final String AUTHENTICATION_TOKEN = "test-authentication-token"; + + @Test + public void testConstructsEncodedLatestAndPinnedUris() + { + List requests = new ArrayList<>(); + TestingHttpClient httpClient = new TestingHttpClient(request -> { + requests.add(request); + return response(OK, "{}"); + }); + HogQlSemanticCatalogHttpTransport transport = new HogQlSemanticCatalogHttpTransport(BASE_URI, httpClient, () -> AUTHENTICATION_TOKEN); + + assertThat(transport.load(LoadRequest.latest(CATALOG, LANGUAGE_VERSION)).toCompletableFuture().join()) + .isEqualTo("{}".getBytes(StandardCharsets.UTF_8)); + assertThat(transport.load(LoadRequest.pinned(CATALOG, LANGUAGE_VERSION, 42)).toCompletableFuture().join()) + .isEqualTo("{}".getBytes(StandardCharsets.UTF_8)); + + assertThat(requests).extracting(request -> request.getUri().toASCIIString()).containsExactly( + "https://duckgres.example/control-plane/v1/hogql/compatibility/semantic-catalog?protocolVersion=1&languageVersion=1.0.0&catalog=Sales%20%26%20Growth/2026&catalogDelimited=true", + "https://duckgres.example/control-plane/v1/hogql/compatibility/semantic-catalog?protocolVersion=1&languageVersion=1.0.0&catalog=Sales%20%26%20Growth/2026&catalogDelimited=true&generation=42"); + assertThat(requests).allSatisfy(request -> { + assertThat(request.getMethod()).isEqualTo("GET"); + assertThat(request.getHeader(ACCEPT)).isEqualTo(JSON_UTF_8.withoutParameters().toString()); + assertThat(request.getHeader("X-Duckgres-Internal-Secret")).isEqualTo(AUTHENTICATION_TOKEN); + }); + } + + @Test + public void testReadsAuthenticationTokenForEveryRequest() + { + List requests = new ArrayList<>(); + List tokens = new ArrayList<>(List.of("first-token", "second-token")); + TestingHttpClient httpClient = new TestingHttpClient(request -> { + requests.add(request); + return response(OK, "{}"); + }); + HogQlSemanticCatalogHttpTransport transport = new HogQlSemanticCatalogHttpTransport(BASE_URI, httpClient, () -> tokens.removeFirst()); + + transport.load(LoadRequest.latest(CATALOG, LANGUAGE_VERSION)).toCompletableFuture().join(); + transport.load(LoadRequest.latest(CATALOG, LANGUAGE_VERSION)).toCompletableFuture().join(); + + assertThat(requests).extracting(request -> request.getHeader("X-Duckgres-Internal-Secret")) + .containsExactly("first-token", "second-token"); + } + + @Test + public void testReadsRotatingAuthenticationTokenFileForEveryRequest(@TempDir Path temporaryDirectory) + throws Exception + { + Path tokenFile = temporaryDirectory.resolve("hogql-catalog-token"); + Files.writeString(tokenFile, "first-token\n"); + List requests = new ArrayList<>(); + TestingHttpClient httpClient = new TestingHttpClient(request -> { + requests.add(request); + return response(OK, "{}"); + }); + HogQlSemanticCatalogHttpTransport transport = new HogQlSemanticCatalogHttpTransport( + new HogQlSemanticCatalogConfig() + .setUri(BASE_URI) + .setAuthenticationTokenFile(tokenFile.toString()), + httpClient); + + transport.load(LoadRequest.latest(CATALOG, LANGUAGE_VERSION)).toCompletableFuture().join(); + Files.writeString(tokenFile, "second-token"); + transport.load(LoadRequest.latest(CATALOG, LANGUAGE_VERSION)).toCompletableFuture().join(); + + assertThat(requests).extracting(request -> request.getHeader("X-Duckgres-Internal-Secret")) + .containsExactly("first-token", "second-token"); + } + + @ParameterizedTest + @MethodSource("invalidAuthenticationTokens") + public void testRejectsInvalidAuthenticationTokensWithoutDisclosure(String token) + { + HogQlSemanticCatalogHttpTransport transport = new HogQlSemanticCatalogHttpTransport( + BASE_URI, + new TestingHttpClient(_ -> response(OK, "{}")), + () -> token); + + assertThatThrownBy(() -> transport.load(LoadRequest.latest(CATALOG, LANGUAGE_VERSION)).toCompletableFuture().join()) + .cause() + .isInstanceOfSatisfying(HogQlSemanticCatalogException.class, exception -> { + assertThat(exception.failure()).isEqualTo(Failure.UNAVAILABLE); + if (!token.isEmpty()) { + assertThat(exception).hasMessageNotContaining(token); + } + }); + } + + @ParameterizedTest(name = "{0}") + @MethodSource("statusFailures") + public void testMapsHttpStatusWithoutResponseDisclosure(String name, HttpStatus status, LoadRequest request, Failure failure) + { + TestingHttpClient httpClient = new TestingHttpClient(_ -> response(status, SECRET)); + HogQlSemanticCatalogHttpTransport transport = new HogQlSemanticCatalogHttpTransport(BASE_URI, httpClient, () -> AUTHENTICATION_TOKEN); + + assertThatThrownBy(() -> transport.load(request).toCompletableFuture().join()) + .cause() + .isInstanceOfSatisfying(HogQlSemanticCatalogException.class, exception -> { + assertThat(exception.failure()).isEqualTo(failure); + assertThat(exception).hasMessageNotContaining(SECRET); + assertThat(exception.getMessage()).hasSizeLessThan(128); + }); + } + + private static Stream statusFailures() + { + return Stream.of( + Arguments.of("latest missing", NOT_FOUND, LoadRequest.latest(CATALOG, LANGUAGE_VERSION), Failure.UNAVAILABLE), + Arguments.of("pinned missing", NOT_FOUND, LoadRequest.pinned(CATALOG, LANGUAGE_VERSION, 7), Failure.GENERATION_MISMATCH), + Arguments.of("generation conflict", CONFLICT, LoadRequest.pinned(CATALOG, LANGUAGE_VERSION, 7), Failure.GENERATION_MISMATCH), + Arguments.of("server failure", INTERNAL_SERVER_ERROR, LoadRequest.latest(CATALOG, LANGUAGE_VERSION), Failure.UNAVAILABLE)); + } + + private static Stream invalidAuthenticationTokens() + { + return Stream.of("", " ", "line-one\nline-two"); + } + + private static Response response(HttpStatus status, String body) + { + return new TestingResponse(status, contentType(JSON_UTF_8), body.getBytes(StandardCharsets.UTF_8)); + } +} diff --git a/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogManager.java b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogManager.java new file mode 100644 index 000000000000..49e97bdce043 --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/hogql/TestHogQlSemanticCatalogManager.java @@ -0,0 +1,248 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.hogql; + +import io.airlift.units.Duration; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotLoader; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotLoader.LoadRequest; +import io.trino.hogql.parser.HogQlLanguageVersion; +import io.trino.spi.catalog.CatalogName; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import static io.trino.testing.assertions.Assert.assertEventually; +import static java.util.concurrent.TimeUnit.SECONDS; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +public class TestHogQlSemanticCatalogManager +{ + private static final HogQlLanguageVersion LANGUAGE_VERSION = HogQlLanguageVersion.valueOf("1.0.0"); + private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier("ducklake", false); + + @Test + public void testColdAndExpiredReadsStayLocalWhileRefreshIsAsynchronous() + { + AtomicLong ticker = new AtomicLong(); + AtomicInteger loads = new AtomicInteger(); + CompletableFuture firstLoadStarted = new CompletableFuture<>(); + CompletableFuture secondLoadStarted = new CompletableFuture<>(); + CompletableFuture firstLoad = new CompletableFuture<>(); + CompletableFuture secondLoad = new CompletableFuture<>(); + HogQlSemanticCatalogSnapshotLoader loader = _ -> switch (loads.incrementAndGet()) { + case 1 -> { + firstLoadStarted.complete(null); + yield firstLoad; + } + case 2 -> { + secondLoadStarted.complete(null); + yield secondLoad; + } + default -> throw new IllegalStateException("unexpected semantic catalog load"); + }; + HogQlSemanticCatalogManager manager = new HogQlSemanticCatalogManager(config(), loader, LANGUAGE_VERSION, ticker::get); + try { + Optional cold = manager.cache().currentSnapshot(CATALOG); + assertThat(cold).isEmpty(); + firstLoadStarted.join(); + CompletableFuture firstRefresh = manager.prewarm(CATALOG).toCompletableFuture(); + firstLoad.complete(snapshot(1)); + assertThat(firstRefresh.join().generation()).isEqualTo(1); + assertThat(manager.cache().currentSnapshot(CATALOG)).get().extracting(HogQlSemanticCatalogSnapshot::generation).isEqualTo(1L); + + ticker.set(3 * SECONDS.toNanos(1)); + Optional expired = manager.cache().currentSnapshot(CATALOG); + assertThat(expired).isEmpty(); + secondLoadStarted.join(); + CompletableFuture secondRefresh = manager.prewarm(CATALOG).toCompletableFuture(); + secondLoad.complete(snapshot(2)); + assertThat(secondRefresh.join().generation()).isEqualTo(2); + assertThat(loads).hasValue(2); + } + finally { + manager.shutdown(); + } + } + + @Test + public void testColdExactGenerationUsesPinnedLoadRequest() + { + CompletableFuture requestedLoad = new CompletableFuture<>(); + CompletableFuture metadata = new CompletableFuture<>(); + HogQlSemanticCatalogManager manager = new HogQlSemanticCatalogManager( + config(), + request -> { + requestedLoad.complete(request); + return metadata; + }, + LANGUAGE_VERSION, + System::nanoTime); + try { + assertThat(manager.cache().currentSnapshot(CATALOG, OptionalLong.of(7))).isEmpty(); + assertThat(requestedLoad.join()).isEqualTo(LoadRequest.pinned(CATALOG, LANGUAGE_VERSION, 7)); + + metadata.complete(snapshot(7)); + assertEventually(() -> assertThat(manager.cache().currentSnapshot(CATALOG, OptionalLong.of(7))) + .get() + .extracting(HogQlSemanticCatalogSnapshot::generation) + .isEqualTo(7L)); + } + finally { + metadata.completeExceptionally(new IllegalStateException("test shutdown")); + manager.shutdown(); + } + } + + @Test + public void testShutdownTerminatesDedicatedLoaderExecutor() + { + HogQlSemanticCatalogManager manager = new HogQlSemanticCatalogManager( + config(), + _ -> CompletableFuture.completedFuture(snapshot(1)), + LANGUAGE_VERSION, + System::nanoTime); + + assertThat(manager.isLoaderExecutorShutdown()).isFalse(); + manager.shutdown(); + assertThat(manager.isLoaderExecutorShutdown()).isTrue(); + } + + @Test + public void testCatalogLifecyclePrewarmsWithoutWaitingForMetadata() + { + PhysicalIdentifier unusualCatalog = new PhysicalIdentifier("sales-data", true); + AtomicInteger loads = new AtomicInteger(); + CompletableFuture loadStarted = new CompletableFuture<>(); + CompletableFuture metadata = new CompletableFuture<>(); + HogQlSemanticCatalogManager manager = new HogQlSemanticCatalogManager( + config(), + request -> { + assertThat(request.catalog()).isEqualTo(unusualCatalog); + loads.incrementAndGet(); + loadStarted.complete(null); + return metadata; + }, + LANGUAGE_VERSION, + System::nanoTime); + try { + CompletableFuture listenerInvocation = CompletableFuture.runAsync( + () -> new HogQlSemanticCatalogPrewarmListener(manager).catalogLoaded(new CatalogName("sales-data"))); + + assertThat(listenerInvocation).succeedsWithin(10, SECONDS); + loadStarted.join(); + assertThat(loads).hasValue(1); + assertThat(manager.cache().currentSnapshot(unusualCatalog)).isEmpty(); + } + finally { + metadata.complete(new HogQlSemanticCatalogSnapshot(2, LANGUAGE_VERSION, unusualCatalog, 1, List.of())); + manager.shutdown(); + } + } + + @Test + public void testRejectedLoaderWorkIsBackedOffBeforeRetry() + { + AtomicLong ticker = new AtomicLong(); + PhysicalIdentifier runningCatalog = new PhysicalIdentifier("running", false); + PhysicalIdentifier queuedCatalog = new PhysicalIdentifier("queued", false); + PhysicalIdentifier rejectedCatalog = new PhysicalIdentifier("rejected", false); + CompletableFuture runningStarted = new CompletableFuture<>(); + CompletableFuture releaseRunning = new CompletableFuture<>(); + CompletableFuture rejectedStarted = new CompletableFuture<>(); + CompletableFuture rejectedLoad = new CompletableFuture<>(); + AtomicInteger rejectedLoads = new AtomicInteger(); + HogQlSemanticCatalogSnapshotLoader loader = request -> switch (request.catalog().value()) { + case "running" -> { + runningStarted.complete(null); + releaseRunning.join(); + yield CompletableFuture.completedFuture(snapshot(runningCatalog, 1)); + } + case "queued" -> CompletableFuture.completedFuture(snapshot(queuedCatalog, 1)); + case "rejected" -> { + rejectedLoads.incrementAndGet(); + rejectedStarted.complete(null); + yield rejectedLoad; + } + default -> throw new IllegalStateException("unexpected catalog load"); + }; + HogQlSemanticCatalogConfig config = config() + .setMaximumEntries(4) + .setFailureBackoff(new Duration(10, SECONDS)) + .setLoaderQueueCapacity(1); + HogQlSemanticCatalogManager manager = new HogQlSemanticCatalogManager(config, loader, LANGUAGE_VERSION, ticker::get); + try { + CompletionStage runningRefresh = manager.prewarm(runningCatalog); + runningStarted.join(); + CompletionStage queuedRefresh = manager.prewarm(queuedCatalog); + CompletionStage rejectedRefresh = manager.prewarm(rejectedCatalog); + + assertThatThrownBy(rejectedRefresh.toCompletableFuture()::join) + .cause() + .isInstanceOf(RejectedExecutionException.class); + assertThat(rejectedLoads).hasValue(0); + assertThat(manager.cache().currentSnapshot(rejectedCatalog)).isEmpty(); + + releaseRunning.complete(null); + runningRefresh.toCompletableFuture().join(); + queuedRefresh.toCompletableFuture().join(); + + ticker.set(SECONDS.toNanos(10) - 1); + assertThat(manager.cache().currentSnapshot(rejectedCatalog)).isEmpty(); + assertThat(rejectedLoads).hasValue(0); + + ticker.set(SECONDS.toNanos(10)); + assertThat(manager.cache().currentSnapshot(rejectedCatalog)).isEmpty(); + rejectedStarted.join(); + assertThat(rejectedLoads).hasValue(1); + rejectedLoad.complete(snapshot(rejectedCatalog, 1)); + assertThat(manager.cache().currentSnapshot(rejectedCatalog)).get().extracting(HogQlSemanticCatalogSnapshot::generation).isEqualTo(1L); + } + finally { + releaseRunning.complete(null); + rejectedLoad.completeExceptionally(new IllegalStateException("test shutdown")); + manager.shutdown(); + } + } + + private static HogQlSemanticCatalogConfig config() + { + return new HogQlSemanticCatalogConfig() + .setMaximumEntries(2) + .setRefreshAfter(new Duration(1, SECONDS)) + .setExpireAfter(new Duration(2, SECONDS)) + .setFailureBackoff(new Duration(0, SECONDS)) + .setLoaderThreads(1) + .setLoaderQueueCapacity(2); + } + + private static HogQlSemanticCatalogSnapshot snapshot(long generation) + { + return snapshot(CATALOG, generation); + } + + private static HogQlSemanticCatalogSnapshot snapshot(PhysicalIdentifier catalog, long generation) + { + return new HogQlSemanticCatalogSnapshot(2, LANGUAGE_VERSION, catalog, generation, List.of()); + } +} diff --git a/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlCorrelatedSubquery.java b/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlCorrelatedSubquery.java new file mode 100644 index 000000000000..62efdb631bb7 --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlCorrelatedSubquery.java @@ -0,0 +1,122 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.sql.planner; + +import io.trino.hogql.compiler.HogQlCompilationResult; +import io.trino.hogql.compiler.HogQlCompileEnvelope; +import io.trino.hogql.compiler.HogQlCompiler; +import io.trino.hogql.compiler.HogQlSemanticCatalogContext; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.spi.TrinoException; +import io.trino.sql.SqlFormatter; +import io.trino.sql.planner.assertions.BasePlanTest; +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; + +import static io.trino.spi.StandardErrorCode.COLUMN_NOT_FOUND; +import static io.trino.testing.TestingHandles.TEST_CATALOG_NAME; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; +import static org.assertj.core.api.Assertions.catchThrowableOfType; + +public class TestHogQlCorrelatedSubquery + extends BasePlanTest +{ + private final HogQlCompiler compiler = new HogQlCompiler(); + + @Test + public void testAnalyzerBindsCorrelatedOuterColumn() + { + assertThatCode(() -> plan(compile( + "SELECT o.orderkey FROM orders o " + + "WHERE o.custkey IN (" + + "SELECT c.custkey FROM customer c WHERE c.custkey = o.custkey)"))) + .doesNotThrowAnyException(); + } + + @Test + public void testAnalyzerBindsCorrelatedOuterColumnInScalarSubquery() + { + assertThatCode(() -> plan(compile( + "SELECT o.orderkey, (" + + "SELECT c.custkey FROM customer c WHERE c.custkey = o.custkey) " + + "FROM orders o"))) + .doesNotThrowAnyException(); + } + + @Test + public void testAnalyzerBindsResolvedCorrelatedLogicalField() + { + assertThatCode(() -> plan(compileSemantic( + "SELECT e.event FROM events e WHERE e.personId IN (" + + "SELECT p.personId FROM persons p WHERE p.personId = e.personId)"))) + .doesNotThrowAnyException(); + } + + @Test + public void testAnalyzerPlansLeftAnyJoinLateralLimit() + { + assertThatCode(() -> plan(compile( + "SELECT r.regionkey, n.nationkey FROM region r " + + "LEFT ANY JOIN nation n ON r.regionkey = n.regionkey"))) + .doesNotThrowAnyException(); + } + + @Test + public void testAnalyzerRejectsMissingCorrelatedOuterColumn() + { + AssertionError failure = catchThrowableOfType( + AssertionError.class, + () -> plan(compile( + "SELECT o.orderkey FROM orders o " + + "WHERE o.custkey IN (" + + "SELECT c.custkey FROM customer c WHERE c.custkey = missing.custkey)"))); + + assertThat(failure.getCause()) + .isInstanceOfSatisfying(TrinoException.class, cause -> { + assertThat(cause.getErrorCode()).isEqualTo(COLUMN_NOT_FOUND.toErrorCode()); + assertThat(cause).hasMessageContaining("Column 'missing.custkey' cannot be resolved"); + }); + } + + private String compile(String hogql) + { + return SqlFormatter.formatSql(compiler.compile(hogql)); + } + + private String compileSemantic(String hogql) + { + PhysicalIdentifier catalog = new PhysicalIdentifier(TEST_CATALOG_NAME, false); + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext( + catalog, + _ -> new PinnedSnapshot(TestHogQlProjectionPruning.testingSnapshot())); + HogQlCompilationResult result = compiler.compile( + new HogQlCompileEnvelope( + hogql, + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + Map.of(), + Map.of(), + Map.of(), + Map.of(), + OptionalLong.of(1)), + Optional.of(context)); + return SqlFormatter.formatSql(result.statement()); + } +} diff --git a/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlPivot.java b/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlPivot.java new file mode 100644 index 000000000000..9d0088cbe04e --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlPivot.java @@ -0,0 +1,53 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.sql.planner; + +import io.trino.hogql.compiler.HogQlCompiler; +import io.trino.sql.SqlFormatter; +import io.trino.sql.planner.assertions.BasePlanTest; +import io.trino.sql.planner.optimizations.PlanNodeSearcher; +import io.trino.sql.planner.plan.AggregationNode; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static io.trino.sql.planner.LogicalPlanner.Stage.CREATED; +import static org.assertj.core.api.Assertions.assertThat; + +public class TestHogQlPivot + extends BasePlanTest +{ + private final HogQlCompiler compiler = new HogQlCompiler(); + + @Test + public void testLowersToStockFilteredAggregations() + { + String hogql = "SELECT custkey, filled_total, open_total FROM orders " + + "PIVOT (sum(totalprice) AS total FOR orderstatus IN ('F' AS filled, 'O' AS open) GROUP BY custkey)"; + + Plan plan = plan(SqlFormatter.formatSql(compiler.compile(hogql)), CREATED); + + List aggregations = PlanNodeSearcher.searchFrom(plan.getRoot()) + .whereIsInstanceOfAny(AggregationNode.class) + .findAll().stream() + .map(AggregationNode.class::cast) + .toList(); + assertThat(aggregations).singleElement().satisfies(aggregation -> { + assertThat(aggregation.getGroupingKeys()).hasSize(1); + assertThat(aggregation.getAggregations()).hasSize(2); + assertThat(aggregation.getAggregations().values()) + .allSatisfy(value -> assertThat(value.getFilter()).isPresent()); + }); + } +} diff --git a/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java b/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java new file mode 100644 index 000000000000..0dff9426030b --- /dev/null +++ b/core/trino-main/src/test/java/io/trino/sql/planner/TestHogQlProjectionPruning.java @@ -0,0 +1,326 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.sql.planner; + +import com.google.common.collect.ImmutableMap; +import io.trino.hogql.compiler.HogQlCompilationResult; +import io.trino.hogql.compiler.HogQlCompileEnvelope; +import io.trino.hogql.compiler.HogQlCompiler; +import io.trino.hogql.compiler.HogQlSemanticCatalogContext; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ActionReference; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ArgumentReferenceRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.ExpressionArgument; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.FieldReferenceRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.JoinKey; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LazyProjectionDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LazyTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalFieldDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalTableDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LogicalType; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralEncoding; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.LiteralRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.OperatorRecipe; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalIdentifier; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PhysicalQualifiedName; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PredicateRepresentation; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.PropertyStorage; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipCardinality; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.RelationshipDefinition; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.SemanticOperator; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshot.TypedLiteral; +import io.trino.hogql.compiler.catalog.HogQlSemanticCatalogSnapshotProvider.PinnedSnapshot; +import io.trino.hogql.parser.HogQlLanguageContract; +import io.trino.sql.SqlFormatter; +import io.trino.sql.planner.assertions.BasePlanTest; +import io.trino.sql.planner.optimizations.PlanNodeSearcher; +import io.trino.sql.planner.plan.FilterNode; +import io.trino.sql.planner.plan.JoinNode; +import io.trino.sql.planner.plan.PlanNode; +import io.trino.sql.planner.plan.SetOperationNode; +import io.trino.sql.planner.plan.TableScanNode; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; + +import static io.trino.SystemSessionProperties.ENABLE_DYNAMIC_FILTERING; +import static io.trino.SystemSessionProperties.JOIN_REORDERING_STRATEGY; +import static io.trino.sql.planner.LogicalPlanner.Stage.CREATED; +import static io.trino.sql.planner.plan.JoinType.INNER; +import static io.trino.testing.TestingHandles.TEST_CATALOG_NAME; +import static org.assertj.core.api.Assertions.assertThat; + +public class TestHogQlProjectionPruning + extends BasePlanTest +{ + private static final PhysicalIdentifier CATALOG = new PhysicalIdentifier(TEST_CATALOG_NAME, false); + private static final HogQlSemanticCatalogSnapshot SNAPSHOT = testingSnapshot(); + + private final HogQlCompiler compiler = new HogQlCompiler(); + + public TestHogQlProjectionPruning() + { + super(ImmutableMap.of( + ENABLE_DYNAMIC_FILTERING, "false", + JOIN_REORDERING_STRATEGY, "NONE")); + } + + @Test + public void testUnusedLazyProjectionDoesNotReachStockPlan() + { + Plan plan = plan(compile("SELECT sub.event FROM (SELECT e.personProfile.*, e.* FROM events e) sub")); + + assertThat(nodes(plan, TableScanNode.class)).hasSize(1); + assertThat(nodes(plan, JoinNode.class)).isEmpty(); + } + + @Test + public void testUsedLazyProjectionRemainsVisibleToStockPredicatePushdown() + { + Plan plan = plan(compile( + "SELECT sub.name FROM (SELECT e.personProfile.*, e.* FROM events e) sub " + + "WHERE sub.name = 'Customer#000000001'")); + + assertThat(nodes(plan, TableScanNode.class)).hasSize(2); + List joins = nodes(plan, JoinNode.class); + assertThat(joins).hasSize(1); + JoinNode join = joins.getFirst(); + assertThat(join.getType()).isEqualTo(INNER); + assertThat(PlanNodeSearcher.searchFrom(join.getRight()) + .whereIsInstanceOfAny(FilterNode.class) + .findFirst()).isPresent(); + } + + @Test + public void testUnusedLazyProjectionIsPrunedThroughChainedCtes() + { + Plan plan = plan(compile( + "WITH first_source(aliasedName, aliasedEvent, aliasedPersonId) AS (SELECT e.personProfile.*, e.* FROM events e), " + + "second_source AS (SELECT * FROM first_source) " + + "SELECT aliasedEvent FROM second_source")); + + assertThat(nodes(plan, TableScanNode.class)).hasSize(1); + assertThat(nodes(plan, JoinNode.class)).isEmpty(); + } + + @Test + public void testDemandedLazyProjectionRemainsInPlanThroughChainedCtes() + { + Plan plan = plan(compile( + "WITH first_source(aliasedName, aliasedEvent, aliasedPersonId) AS (SELECT e.personProfile.*, e.* FROM events e), " + + "second_source AS (SELECT * FROM first_source) " + + "SELECT aliasedName FROM second_source WHERE aliasedName = 'Customer#000000001'")); + + assertThat(nodes(plan, TableScanNode.class)).hasSize(2); + assertThat(nodes(plan, JoinNode.class)).hasSize(1); + } + + @Test + public void testUnusedLazyProjectionIsPrunedThroughDerivedCteChain() + { + Plan plan = plan(compile( + "WITH first_source AS (SELECT e.personProfile.*, e.* FROM events e), " + + "second_source AS (SELECT derived.* FROM (SELECT * FROM first_source) " + + "derived(aliasedName, aliasedEvent, aliasedPersonId)) " + + "SELECT aliasedEvent FROM second_source")); + + assertThat(nodes(plan, TableScanNode.class)).hasSize(1); + assertThat(nodes(plan, JoinNode.class)).isEmpty(); + } + + @Test + public void testDemandedLazyProjectionRemainsThroughDerivedCteChain() + { + Plan plan = plan(compile( + "WITH first_source AS (SELECT e.personProfile.*, e.* FROM events e), " + + "second_source AS (SELECT derived.* FROM (SELECT * FROM first_source) " + + "derived(aliasedName, aliasedEvent, aliasedPersonId)) " + + "SELECT aliasedName FROM second_source WHERE aliasedName = 'Customer#000000001'")); + + assertThat(nodes(plan, TableScanNode.class)).hasSize(2); + assertThat(nodes(plan, JoinNode.class)).hasSize(1); + } + + @Test + public void testVarcharJsonPropertyPredicateReachesStockPlanner() + { + Plan plan = plan(compile("SELECT properties.plan FROM persons WHERE properties.plan = 'pro'")); + + assertThat(nodes(plan, TableScanNode.class)).hasSize(1); + assertThat(nodes(plan, FilterNode.class)).hasSize(1); + } + + @Test + public void testV0ActionPredicateReachesStockPlanner() + { + Plan plan = plan(compileV0("SELECT event FROM events WHERE matchesAction(42)")); + + assertThat(nodes(plan, TableScanNode.class)).hasSize(1); + assertThat(nodes(plan, FilterNode.class)).hasSize(1); + } + + @ParameterizedTest + @ValueSource(strings = {"UNION ALL", "INTERSECT ALL", "EXCEPT ALL"}) + public void testSetOperationDemandIsMappedByBranchPosition(String operator) + { + Plan plan = plan(compile( + "WITH source AS (" + + "SELECT CAST(e.event AS String) AS selected, e.personProfile.name AS discarded FROM events e " + + operator + " " + + "SELECT e.personProfile.name AS rightSelected, CAST(e.event AS String) AS rightDiscarded FROM events e) " + + "SELECT selected FROM source"), CREATED); + + List setOperations = nodes(plan, SetOperationNode.class); + assertThat(setOperations).hasSize(1); + SetOperationNode setOperation = setOperations.getFirst(); + assertThat(nodes(setOperation.getSources().getFirst(), TableScanNode.class)).hasSize(1); + assertThat(nodes(setOperation.getSources().getFirst(), JoinNode.class)).isEmpty(); + assertThat(nodes(setOperation.getSources().getLast(), TableScanNode.class)).hasSize(2); + assertThat(nodes(setOperation.getSources().getLast(), JoinNode.class)).hasSize(1); + assertThat(nodes(plan, TableScanNode.class)).hasSize(3); + assertThat(nodes(plan, JoinNode.class)).hasSize(1); + } + + private String compile(String query) + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + HogQlCompilationResult result = compiler.compile(new HogQlCompileEnvelope( + query, + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + Map.of(), + Map.of(), + Map.of(), + Map.of(), + OptionalLong.of(1)), Optional.of(context)); + return SqlFormatter.formatSql(result.statement()); + } + + private String compileV0(String query) + { + HogQlSemanticCatalogContext context = new HogQlSemanticCatalogContext(CATALOG, _ -> new PinnedSnapshot(SNAPSHOT)); + HogQlCompilationResult result = compiler.compileV0(new HogQlCompileEnvelope( + query, + HogQlCompileEnvelope.PROTOCOL_VERSION, + HogQlLanguageContract.current().languageVersion(), + Map.of(), + Map.of(), + Map.of(), + Map.of(), + OptionalLong.of(1)), Optional.of(context)); + return SqlFormatter.formatSql(result.statement()); + } + + private static List nodes(Plan plan, Class type) + { + return nodes(plan.getRoot(), type); + } + + private static List nodes(PlanNode root, Class type) + { + return PlanNodeSearcher.searchFrom(root) + .whereIsInstanceOfAny(type) + .findAll().stream() + .map(type::cast) + .toList(); + } + + static HogQlSemanticCatalogSnapshot testingSnapshot() + { + LogicalTableDefinition events = new LogicalTableDefinition( + "events", + physicalName("orders"), + List.of( + field("event", "orderkey", "bigint", LogicalType.INTEGER), + field("personId", "custkey", "bigint", LogicalType.INTEGER)), + List.of(), + List.of(new RelationshipDefinition( + "person", + "persons", + RelationshipCardinality.MANY_TO_ONE, + List.of(new JoinKey("personId", "personId"))))); + LogicalTableDefinition persons = new LogicalTableDefinition( + "persons", + physicalName("customer"), + List.of( + field("personId", "custkey", "bigint", LogicalType.INTEGER), + field("name", "name", "varchar", LogicalType.STRING), + field("properties", "comment", "varchar", LogicalType.STRING)), + List.of(new PropertyDefinition( + "properties", + "properties", + PropertyStorage.JSON_OBJECT, + LogicalType.STRING, + true, + Optional.of("varchar"), + Optional.of("varchar"), + Optional.of(new OperatorRecipe( + SemanticOperator.JSON_OBJECT_LOOKUP, + List.of( + new ArgumentReferenceRecipe(ExpressionArgument.PROPERTY_SOURCE), + new ArgumentReferenceRecipe(ExpressionArgument.PROPERTY_KEY)))))), + List.of()); + LazyTableDefinition personProfile = new LazyTableDefinition( + "events", + "personProfile", + List.of("person"), + List.of(new LazyProjectionDefinition( + "name", + "varchar", + LogicalType.STRING, + false, + true, + new FieldReferenceRecipe("persons", "name")))); + return new HogQlSemanticCatalogSnapshot( + HogQlSemanticCatalogSnapshot.PROTOCOL_VERSION, + HogQlSemanticCatalogSnapshot.SCHEMA_VERSION, + HogQlLanguageContract.current().languageVersion(), + CATALOG, + 1, + List.of(events, persons), + List.of(), + List.of(), + List.of(), + List.of(), + List.of(), + List.of(), + List.of(personProfile), + List.of(new ActionReference( + "Synthetic order", + "42", + "events", + new PredicateRepresentation(new OperatorRecipe( + SemanticOperator.EQUAL, + List.of( + new FieldReferenceRecipe("events", "event"), + new LiteralRecipe(new TypedLiteral("bigint", LiteralEncoding.INTEGER, "1"))))))), + List.of()); + } + + private static LogicalFieldDefinition field(String name, String physicalName, String type, LogicalType logicalType) + { + return new LogicalFieldDefinition(name, new PhysicalIdentifier(physicalName, false), type, logicalType, false, true); + } + + private static PhysicalQualifiedName physicalName(String table) + { + return new PhysicalQualifiedName(CATALOG, new PhysicalIdentifier("tiny", false), new PhysicalIdentifier(table, false)); + } +} diff --git a/docs/src/main/sphinx/admin.md b/docs/src/main/sphinx/admin.md index 52edb4e4c26c..c6dee22d0f10 100644 --- a/docs/src/main/sphinx/admin.md +++ b/docs/src/main/sphinx/admin.md @@ -12,6 +12,7 @@ admin/tuning admin/jmx admin/opentelemetry admin/openmetrics +admin/hogql admin/properties admin/spill admin/resource-groups @@ -64,6 +65,7 @@ admin/properties ``` * [Properties reference overview](admin/properties) +* [](admin/hogql) * [](admin/properties-general) * [](admin/properties-client-protocol) * [](admin/properties-http-server) diff --git a/docs/src/main/sphinx/admin/hogql.md b/docs/src/main/sphinx/admin/hogql.md new file mode 100644 index 000000000000..f772ff7886e0 --- /dev/null +++ b/docs/src/main/sphinx/admin/hogql.md @@ -0,0 +1,229 @@ +# Native HogQL queries + +The PostHog Trino distribution can accept read-only HogQL through +`POST /v1/hogql`. The coordinator parses and compiles the query to the standard +Trino SQL AST before analysis and planning. Workers, connectors, access control, +and the optimizer receive standard Trino plans. + +Set the following coordinator property to enable the endpoint: + +```properties +hogql.enabled=true +``` + +Compilation runs on a dedicated fixed-size executor. Configure +`hogql.compilation-threads`, `hogql.compilation-queue-capacity`, and +`hogql.compilation-timeout`; saturation and timeout fail with retryable +insufficient-resource errors without affecting standard SQL submission. + +The executable profile supports read-only queries over logical tables declared +by the pinned manifest, including ordered star expansion, properties, lazy +relationships, and declarative actions. Physical tables remain available +through normal Trino names. Unsupported functions and semantic definitions +fail before Trino analysis. + +The endpoint accepts a JSON request envelope. Identity, catalog, schema, time +zone, source, tags, and allowed session properties continue to use the standard +Trino request headers. + +```json +{ + "query": "SELECT event FROM events WHERE person_id = {person_id}", + "protocolVersion": 1, + "languageVersion": "1.0.0", + "parameters": { + "person_id": {"type": "uuid", "value": "00000000-0000-0000-0000-000000000000"} + }, + "variables": {}, + "filters": {}, + "modifiers": {} +} +``` + +Responses use the standard Trino query results protocol, including result +paging and cancellation. The coordinator rejects request bodies larger than 2 +MiB and more than 1,000 total entries across `parameters`, `variables`, +`filters`, and `modifiers`. + +## Modifiers + +Modifiers are defined by the pinned semantic catalog manifest. An explicit +modifier causes the coordinator to pin a manifest even when the query does not +otherwise use semantic catalog definitions. Unknown modifiers and values whose +type does not exactly match the declared type are rejected before execution. + +Each declared modifier has one behavior: + +| Behavior | Result | +| --- | --- | +| `TRINO_SESSION_PROPERTY` | Applies the explicit value, or the manifest default when omitted, as a query-scoped Trino session property. Normal session-property validation and access control still apply. | +| `COMPILER` | Uses the compiler's default behavior when omitted. Explicit values are rejected until that modifier has a compiler handler. | +| `SAFE_NOOP` | Accepts a type-valid explicit value or default and does not change the query. | +| `UNSUPPORTED` | Does nothing when omitted and rejects explicit use. | + +Modifier values stay outside the generated statement AST. The coordinator +decodes them through the typed-value boundary and carries the resulting session +overrides separately during query preparation. + +To return a native Trino plan instead of executing the query, add an `explain` +object: + +```json +{ + "query": "SELECT event FROM events", + "protocolVersion": 1, + "languageVersion": "1.0.0", + "explain": {"type": "DISTRIBUTED", "format": "TEXT"} +} +``` + +The supported plan types are `LOGICAL`, `DISTRIBUTED`, `VALIDATE`, and `IO`. +The supported formats are `TEXT`, `GRAPHVIZ`, and `JSON`. Execution timeouts +continue to use Trino session properties such as `query_max_execution_time`. + +## Compilation capacity + +HogQL compilation runs on coordinator threads that are separate from standard +query dispatch work. The following properties set the maximum concurrent and +queued compilation work: + +| Property | Default | Description | +| --- | --- | --- | +| `hogql.compilation-threads` | `2` | Number of coordinator threads dedicated to HogQL compilation. The value must be at least `1`. | +| `hogql.compilation-queue-capacity` | `32` | Maximum compilations waiting for a compiler thread. Set this to `0` to reject requests instead of queuing them when every compiler thread is busy. | + +When the workers and queue are full, the coordinator fails the HogQL query with +`HOGQL_COMPILATION_QUEUE_FULL`. This is an insufficient-resources error, so the +caller can retry the query. Standard Trino SQL does not use this executor or +queue. + +## Semantic catalog manifests + +Logical tables, fields, quoted physical names, and star expansion use an +immutable semantic catalog manifest published by Duckgres. Configure its base +URI on the coordinator: + +```properties +hogql.semantic-catalog.uri=https://duckgres.example +``` + +The coordinator refreshes manifests asynchronously and compiles each query +against one pinned generation. A query does not wait for a remote metadata +request. A missing, expired, malformed, or mismatched manifest fails the query +with a catalog readiness or compatibility error. + +The compatibility request path is +`/v1/hogql/compatibility/semantic-catalog`. The coordinator identifies the +selected Trino catalog, HogQL language version, and optional requested +generation. Manifests contain typed identifiers and definitions, not SQL text. + +Qualified stars on relations without a manifest schema compile to standard +Trino `relation.*`. `EXCLUDE` needs the manifest's ordered, star-visible field +list, and fails with a source-located compatibility error when that schema is +unavailable. Duckgres should publish physical tables as physical-derived +semantic definitions from the physical catalog inventory when those tables +need HogQL star exclusion. + +Set `hogql.semantic-catalog.authentication-token-file` to a file containing a +Duckgres read-only or admin token. Trino reads the file for every manifest +request so token rotation does not require a coordinator restart. Surrounding +whitespace is stripped; blank tokens, oversized tokens, and embedded CR or LF +are rejected. The token is sent as `X-Duckgres-Internal-Secret` and is never +included in error messages. + +Clients can set `catalogGeneration` in the request envelope to require an exact +immutable generation. Exact-generation cache entries never fall back to the +latest manifest. A missing, expired, or mismatched generation fails closed. + +## Function profile + +The original MVP function set remains supported: + +| Family | HogQL names | +| --- | --- | +| Numeric, conditional, and strings | `abs`, `coalesce`, `if`, `lower`, `upper`, `length`, `concat`, `replace` | +| Collections | `map`, `arraySort`, `arrayDistinct`, `arrayFlatten`, `arrayStringConcat` | +| Date and time | `dateAdd`, `dateDiff`, `dateTrunc` | +| Aggregates | `count`, `sum`, `min`, `max`, `avg`, `any`, `argMin`, `argMax`, `array_agg` | +| Windows | `first_value`, `rank`, `row_number` | + +The Team 2 materialized-view compatibility extension adds the exact scalar, +aggregate, JSON, collection, date/time, regular-expression, conversion, and +window rewrites exercised by its frozen query corpus. The in-tree +`HogQlV0FunctionRegistry` is the source of truth for accepted names, arities, +determinism, and lowering strategy. Catalog function declarations do not +silently broaden this fail-closed profile. + +Declarative actions resolve by name or ID from the pinned manifest and lower +to optimizer-visible predicates or relation membership. Cohorts, saved +queries, explicit modifiers without an enabled behavior, HogQLX, +PIVOT/UNPIVOT execution, unsupported ClickHouse clauses, and complete +type/function parity remain outside this endpoint profile. Unsupported +constructs return typed compatibility errors; they do not fall through as +Trino functions. + +`any`, `argMin`, and `argMax` are nondeterministic when more than one input can +satisfy the selection rule. Differential validation uses unique selection keys +or invariant-based comparisons for those calls. + +## Physical catalog inventory + +An authenticated client can read connector-facing table and column metadata +from `GET /v1/hogql/compatibility/physical-catalog?catalog=&protocolVersion=1`. +The endpoint is available only on coordinators when `hogql.enabled=true`. + +The versioned response contains `protocolVersion` and `schemaVersion` before +the structured catalog, schema, table, and column identifiers. Each column +includes its one-based ordinal, exact Trino type signature, nullability, hidden +state, and star visibility. Hidden columns are reported when the caller can see +them, but remain excluded from star expansion. The ordinal is the connector's +original column position. Column visibility filtering can therefore leave gaps +in the returned ordinals. + +The `catalogHandleVersion` identifies the active Trino catalog registration. +It detects a catalog replacement during the request, but it is not the +connector's metadata snapshot or a semantic catalog generation. Duckgres +assigns its own monotonically increasing generation when it publishes the +inventory. + +The endpoint applies the same catalog, table, and column visibility filters as +Trino metadata listings. It reads one connector transaction, so DuckLake +metadata is pinned to one snapshot. The request fails if the catalog is dropped +or replaced while the inventory is read. Results are limited to 10,000 tables, +10,000 columns per table, 100,000 columns in total, and 8 million characters of +identifier and type text. + +Use normal Trino HTTP authentication and request headers. The endpoint does not +accept a separate credential or an existing transaction ID. + +## Semantic catalog properties + +| Property | Default | Description | +| --- | --- | --- | +| `hogql.semantic-catalog.maximum-entries` | `100` | Maximum catalog snapshots retained by one coordinator. | +| `hogql.semantic-catalog.authentication-token-file` | none | Required when the semantic catalog URI is set. File containing a rotating Duckgres read-only or admin token. | +| `hogql.semantic-catalog.refresh-after` | `1m` | Age at which a cached manifest is refreshed in the background. | +| `hogql.semantic-catalog.expire-after` | `5m` | Age after which a cached manifest cannot be used. | +| `hogql.semantic-catalog.failure-backoff` | `10s` | Minimum delay before retrying a failed refresh. | +| `hogql.semantic-catalog.loader-threads` | `4` | Number of background manifest loader threads. | +| `hogql.semantic-catalog.loader-queue-capacity` | `64` | Maximum queued background manifest loads. | +| `hogql.semantic-catalog.request-timeout` | `10s` | Timeout for a background compatibility request. | +| `hogql.semantic-catalog.maximum-response-size` | `8MB` | Maximum accepted manifest response size. | + +Keep `refresh-after` below `expire-after`. Size the entry count, loader threads, +and queue capacity for the number of catalogs assigned to one coordinator. + +## Image identity and rollback + +Release images use tags of the form +`-ducklake.-hogql.`. Inspect the immutable digest and +the `io.posthog.trino.*` OCI labels to verify the source/DuckLake revision, +compiler build, server and CLI artifact digests, language version, and semantic +catalog protocol/schema. + +To roll back, first stop routing clients to `/v1/hogql`. Set +`hogql.enabled=false` and roll the coordinators; standard `/v1/statement` +queries remain on the unchanged Trino path. Restore the previous immutable +image digest if needed. Semantic catalog content is immutable, so restore prior +content by publishing it as a new higher generation rather than mutating an +existing generation. diff --git a/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/DuckLakePlugin.java b/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/DuckLakePlugin.java index 92a423ce7df6..9e19f8b03fd1 100644 --- a/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/DuckLakePlugin.java +++ b/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/DuckLakePlugin.java @@ -14,9 +14,14 @@ package io.trino.plugin.ducklake; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import io.trino.plugin.ducklake.function.CityHash64Function; +import io.trino.plugin.ducklake.function.FormatReadableTimeDeltaFunction; import io.trino.spi.Plugin; import io.trino.spi.connector.ConnectorFactory; +import java.util.Set; + public class DuckLakePlugin implements Plugin { @@ -25,4 +30,10 @@ public Iterable getConnectorFactories() { return ImmutableList.of(new DuckLakeConnectorFactory()); } + + @Override + public Set> getFunctions() + { + return ImmutableSet.of(CityHash64Function.class, FormatReadableTimeDeltaFunction.class); + } } diff --git a/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/CityHash64Function.java b/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/CityHash64Function.java new file mode 100644 index 000000000000..49c13c671f6f --- /dev/null +++ b/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/CityHash64Function.java @@ -0,0 +1,35 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.ducklake.function; + +import io.airlift.slice.Slice; +import io.trino.spi.function.Description; +import io.trino.spi.function.LiteralParameters; +import io.trino.spi.function.ScalarFunction; +import io.trino.spi.function.SqlType; +import io.trino.spi.type.Int128; + +public final class CityHash64Function +{ + private CityHash64Function() {} + + @Description("Returns the ClickHouse CityHash 1.0.2 hash of a string") + @ScalarFunction("cityhash64") + @LiteralParameters("x") + @SqlType("decimal(20,0)") + public static Int128 cityHash64(@SqlType("varchar(x)") Slice value) + { + return Int128.valueOf(0, ClickHouseCityHash64.hash(value)); + } +} diff --git a/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/ClickHouseCityHash64.java b/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/ClickHouseCityHash64.java new file mode 100644 index 000000000000..0ded7eebd9df --- /dev/null +++ b/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/ClickHouseCityHash64.java @@ -0,0 +1,178 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.ducklake.function; + +import io.airlift.slice.Slice; + +final class ClickHouseCityHash64 +{ + private static final long K0 = 0xC3A5C85C97CB3127L; + private static final long K1 = 0xB492B66FBE98F273L; + private static final long K2 = 0x9AE16A3B2F90404FL; + private static final long K3 = 0xC949D7C7509E6557L; + private static final long HASH_128_TO_64_MULTIPLIER = 0x9DDFEA08EB382D69L; + + private ClickHouseCityHash64() {} + + public static long hash(Slice value) + { + int length = value.length(); + if (length <= 16) { + return hashLength0To16(value, length); + } + if (length <= 32) { + return hashLength17To32(value, length); + } + if (length <= 64) { + return hashLength33To64(value, length); + } + + long x = fetch64(value, 0); + long y = fetch64(value, length - 16) ^ K1; + long z = fetch64(value, length - 56) ^ K0; + LongPair v = weakHashLength32WithSeeds(value, length - 64, length, y); + LongPair w = weakHashLength32WithSeeds(value, length - 32, length * K1, K0); + z += shiftMix(v.high()) * K1; + x = rotateRight(z + x, 39) * K1; + y = rotateRight(y, 33) * K1; + + int offset = 0; + int remaining = (length - 1) & ~63; + while (remaining > 0) { + x = rotateRight(x + y + v.low() + fetch64(value, offset + 16), 37) * K1; + y = rotateRight(y + v.high() + fetch64(value, offset + 48), 42) * K1; + x ^= w.high(); + y ^= v.low(); + z = rotateRight(z ^ w.low(), 33); + v = weakHashLength32WithSeeds(value, offset, v.high() * K1, x + w.low()); + w = weakHashLength32WithSeeds(value, offset + 32, z + w.high(), y); + long previousX = x; + x = z; + z = previousX; + offset += 64; + remaining -= 64; + } + + return hash16( + hash16(v.low(), w.low()) + shiftMix(y) * K1 + z, + hash16(v.high(), w.high()) + x); + } + + private static long hashLength0To16(Slice value, int length) + { + if (length > 8) { + long a = fetch64(value, 0); + long b = fetch64(value, length - 8); + return hash16(a, rotateRight(b + length, length)) ^ b; + } + if (length >= 4) { + long a = fetch32(value, 0); + return hash16(length + (a << 3), fetch32(value, length - 4)); + } + if (length > 0) { + int a = value.getUnsignedByte(0); + int b = value.getUnsignedByte(length >>> 1); + int c = value.getUnsignedByte(length - 1); + int y = a + (b << 8); + int z = length + (c << 2); + return shiftMix((Integer.toUnsignedLong(y) * K2) ^ (Integer.toUnsignedLong(z) * K3)) * K2; + } + return K2; + } + + private static long hashLength17To32(Slice value, int length) + { + long a = fetch64(value, 0) * K1; + long b = fetch64(value, 8); + long c = fetch64(value, length - 8) * K2; + long d = fetch64(value, length - 16) * K0; + return hash16( + rotateRight(a - b, 43) + rotateRight(c, 30) + d, + a + rotateRight(b ^ K3, 20) - c + length); + } + + private static long hashLength33To64(Slice value, int length) + { + long z = fetch64(value, 24); + long a = fetch64(value, 0) + (length + fetch64(value, length - 16)) * K0; + long b = rotateRight(a + z, 52); + long c = rotateRight(a, 37); + a += fetch64(value, 8); + c += rotateRight(a, 7); + a += fetch64(value, 16); + + long vLow = a + z; + long vHigh = b + rotateRight(a, 31) + c; + a = fetch64(value, 16) + fetch64(value, length - 32); + z = fetch64(value, length - 8); + b = rotateRight(a + z, 52); + c = rotateRight(a, 37); + a += fetch64(value, length - 24); + c += rotateRight(a, 7); + a += fetch64(value, length - 16); + + long wLow = a + z; + long wHigh = b + rotateRight(a, 31) + c; + long r = shiftMix((vLow + wHigh) * K2 + (wLow + vHigh) * K0); + return shiftMix(r * K0 + vHigh) * K2; + } + + private static LongPair weakHashLength32WithSeeds(Slice value, int offset, long a, long b) + { + long w = fetch64(value, offset); + long x = fetch64(value, offset + 8); + long y = fetch64(value, offset + 16); + long z = fetch64(value, offset + 24); + a += w; + b = rotateRight(b + a + z, 21); + long c = a; + a += x + y; + b += rotateRight(a, 44); + return new LongPair(a + z, b + c); + } + + private static long hash16(long low, long high) + { + long a = (low ^ high) * HASH_128_TO_64_MULTIPLIER; + a ^= a >>> 47; + long b = (high ^ a) * HASH_128_TO_64_MULTIPLIER; + b ^= b >>> 47; + return b * HASH_128_TO_64_MULTIPLIER; + } + + private static long shiftMix(long value) + { + return value ^ (value >>> 47); + } + + private static long rotateRight(long value, int shift) + { + return Long.rotateRight(value, shift); + } + + private static long fetch32(Slice value, int offset) + { + return value.getUnsignedByte(offset) + | ((long) value.getUnsignedByte(offset + 1) << 8) + | ((long) value.getUnsignedByte(offset + 2) << 16) + | ((long) value.getUnsignedByte(offset + 3) << 24); + } + + private static long fetch64(Slice value, int offset) + { + return fetch32(value, offset) | (fetch32(value, offset + 4) << 32); + } + + private record LongPair(long low, long high) {} +} diff --git a/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/FormatReadableTimeDeltaFunction.java b/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/FormatReadableTimeDeltaFunction.java new file mode 100644 index 000000000000..b11af437efd5 --- /dev/null +++ b/plugin/trino-ducklake/src/main/java/io/trino/plugin/ducklake/function/FormatReadableTimeDeltaFunction.java @@ -0,0 +1,230 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.ducklake.function; + +import io.airlift.slice.Slice; +import io.trino.spi.TrinoException; +import io.trino.spi.function.Description; +import io.trino.spi.function.ScalarFunction; +import io.trino.spi.function.SqlType; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +import static io.airlift.slice.Slices.utf8Slice; +import static io.trino.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; +import static io.trino.spi.type.StandardTypes.DOUBLE; +import static io.trino.spi.type.StandardTypes.VARCHAR; + +public final class FormatReadableTimeDeltaFunction +{ + private FormatReadableTimeDeltaFunction() {} + + @Description("Formats a numeric duration in seconds using ClickHouse time delta wording") + @ScalarFunction("format_readable_time_delta") + @SqlType(VARCHAR) + public static Slice formatReadableTimeDelta(@SqlType(DOUBLE) double value) + { + return utf8Slice(format(value, Unit.YEARS, Unit.SECONDS)); + } + + @Description("Formats a numeric duration in seconds using ClickHouse time delta wording") + @ScalarFunction("format_readable_time_delta") + @SqlType(VARCHAR) + public static Slice formatReadableTimeDelta( + @SqlType(DOUBLE) double value, + @SqlType(VARCHAR) Slice maximumUnit) + { + Unit maximum = parseUnit(maximumUnit, "maximum"); + Unit minimum = maximum.ordinal() < Unit.SECONDS.ordinal() ? Unit.NANOSECONDS : Unit.SECONDS; + return utf8Slice(format(value, maximum, minimum)); + } + + @Description("Formats a numeric duration in seconds using ClickHouse time delta wording") + @ScalarFunction("format_readable_time_delta") + @SqlType(VARCHAR) + public static Slice formatReadableTimeDelta( + @SqlType(DOUBLE) double value, + @SqlType(VARCHAR) Slice maximumUnit, + @SqlType(VARCHAR) Slice minimumUnit) + { + Unit maximum = parseUnit(maximumUnit, "maximum"); + Unit minimum = parseUnit(minimumUnit, "minimum"); + if (minimum.ordinal() > maximum.ordinal()) { + throw new TrinoException( + INVALID_FUNCTION_ARGUMENT, + "Minimum unit (%s) must not be greater than maximum unit (%s)" + .formatted(minimum.name().toLowerCase(Locale.ENGLISH), maximum.name().toLowerCase(Locale.ENGLISH))); + } + return utf8Slice(format(value, maximum, minimum)); + } + + private static String format(double value, Unit maximum, Unit minimum) + { + if (Double.isNaN(value)) { + return "nan"; + } + if (value == Double.POSITIVE_INFINITY) { + return "inf"; + } + if (value == Double.NEGATIVE_INFINITY) { + return "-inf"; + } + + boolean negative = value < 0; + value = Math.abs(value); + double wholeSeconds = Math.floor(value); + long fractionalNanoseconds = roundedFractionalNanoseconds(value - wholeSeconds); + List parts = new ArrayList<>(); + + for (int index = maximum.ordinal(); index >= minimum.ordinal(); index--) { + Unit unit = Unit.values()[index]; + if (wholeSeconds + 1.0 == wholeSeconds) { + double units = Math.floor(wholeSeconds * unit.scaleMultiplier() / unit.secondsMultiplier()); + parts.add(formatDouble(units) + " " + unit.singular() + "s"); + break; + } + + long units; + if (unit.scale() == 0) { + units = (long) Math.floor(wholeSeconds / unit.secondsMultiplier()); + if (units == 0 && (unit != Unit.SECONDS || !parts.isEmpty())) { + continue; + } + wholeSeconds -= units * unit.secondsMultiplier(); + } + else { + long scaleMultiplier = unit.scaleMultiplier(); + units = (long) wholeSeconds * scaleMultiplier; + wholeSeconds = 0; + long divisor = pow10(9 - unit.scale()); + units += fractionalNanoseconds / divisor; + fractionalNanoseconds %= divisor; + if (units == 0 && (unit != minimum || !parts.isEmpty())) { + continue; + } + } + parts.add(units + " " + unit.singular() + (units == 1 ? "" : "s")); + } + + String result = joinParts(parts); + return negative ? "-" + result : result; + } + + private static long roundedFractionalNanoseconds(double fractional) + { + return Math.round(fractional * 1_000_000_000) % 1_000_000_000; + } + + private static String joinParts(List parts) + { + if (parts.isEmpty()) { + return ""; + } + if (parts.size() == 1) { + return parts.getFirst(); + } + if (parts.size() == 2) { + return parts.getFirst() + " and " + parts.getLast(); + } + return String.join(", ", parts.subList(0, parts.size() - 1)) + " and " + parts.getLast(); + } + + private static String formatDouble(double value) + { + if (Double.isInfinite(value)) { + return "inf"; + } + if (value < 1e21) { + return BigDecimal.valueOf(value).stripTrailingZeros().toPlainString(); + } + String formatted = Double.toString(value).replace("E+", "e").replace('E', 'e'); + int exponent = formatted.indexOf('e'); + if (exponent > 1 && formatted.substring(0, exponent).endsWith(".0")) { + return formatted.substring(0, exponent - 2) + formatted.substring(exponent); + } + return formatted; + } + + private static long pow10(int exponent) + { + return switch (exponent) { + case 0 -> 1; + case 3 -> 1_000; + case 6 -> 1_000_000; + case 9 -> 1_000_000_000; + default -> throw new IllegalArgumentException("unsupported decimal exponent: " + exponent); + }; + } + + private static Unit parseUnit(Slice unit, String bound) + { + String value = unit.toStringUtf8(); + for (Unit candidate : Unit.values()) { + if (candidate.name().toLowerCase(Locale.ENGLISH).equals(value)) { + return candidate; + } + } + throw new TrinoException( + INVALID_FUNCTION_ARGUMENT, + "Unexpected value of %s unit argument (%s); expected nanoseconds, microseconds, milliseconds, seconds, minutes, hours, days, months, or years" + .formatted(bound, value)); + } + + private enum Unit + { + NANOSECONDS(1, 9, "nanosecond"), + MICROSECONDS(1, 6, "microsecond"), + MILLISECONDS(1, 3, "millisecond"), + SECONDS(1, 0, "second"), + MINUTES(60, 0, "minute"), + HOURS(3_600, 0, "hour"), + DAYS(86_400, 0, "day"), + MONTHS(2_635_200, 0, "month"), + YEARS(31_536_000, 0, "year"); + + private final long secondsMultiplier; + private final int scale; + private final String singular; + + Unit(long secondsMultiplier, int scale, String singular) + { + this.secondsMultiplier = secondsMultiplier; + this.scale = scale; + this.singular = singular; + } + + public long secondsMultiplier() + { + return secondsMultiplier; + } + + public int scale() + { + return scale; + } + + public long scaleMultiplier() + { + return pow10(scale); + } + + public String singular() + { + return singular; + } + } +} diff --git a/plugin/trino-ducklake/src/test/java/io/trino/plugin/ducklake/function/TestCityHash64Function.java b/plugin/trino-ducklake/src/test/java/io/trino/plugin/ducklake/function/TestCityHash64Function.java new file mode 100644 index 000000000000..7561fc237ef7 --- /dev/null +++ b/plugin/trino-ducklake/src/test/java/io/trino/plugin/ducklake/function/TestCityHash64Function.java @@ -0,0 +1,89 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.ducklake.function; + +import io.airlift.slice.Slices; +import io.trino.plugin.ducklake.DuckLakePlugin; +import io.trino.spi.type.DecimalType; +import io.trino.sql.query.QueryAssertions; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +import java.util.Map; + +import static io.trino.spi.type.DecimalType.createDecimalType; +import static io.trino.spi.type.SqlDecimal.decimal; +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; + +@TestInstance(PER_CLASS) +final class TestCityHash64Function +{ + private QueryAssertions assertions; + + @BeforeAll + void init() + { + assertions = new QueryAssertions(); + assertions.addPlugin(new DuckLakePlugin()); + } + + @AfterAll + void teardown() + { + assertions.close(); + assertions = null; + } + + @Test + void testClickHouseCompatibility() + { + Map expectedHashes = Map.ofEntries( + Map.entry("", "11160318154034397263"), + Map.entry("a", "2603192927274642682"), + Map.entry("abc", "4220206313085259313"), + Map.entry("abcd", "17823623939509273229"), + Map.entry("12345678", "7177601938557627951"), + Map.entry("123456789", "12390271160407166709"), + Map.entry("1234567890abcdef", "10283158570132023530"), + Map.entry("1234567890abcdefg", "4637769396780685212"), + Map.entry("1234567890abcdef1234567890abcdef", "5983709264297605835"), + Map.entry("1234567890abcdef1234567890abcdefg", "1434880753424560409"), + Map.entry("x".repeat(64), "6437053381938498259"), + Map.entry("x".repeat(65), "5260653789997849295"), + Map.entry("x".repeat(128), "1142585757146322653"), + Map.entry("y".repeat(129), "15472941566376925510"), + Map.entry("z".repeat(200), "14427141409027721128"), + Map.entry("q".repeat(257), "14786733977854578775"), + Map.entry("Moscow", "12507901496292878638"), + Map.entry("Grüße", "17364701079286602353")); + + expectedHashes.forEach((input, expected) -> assertThat(ClickHouseCityHash64.hash(Slices.utf8Slice(input))) + .isEqualTo(Long.parseUnsignedLong(expected))); + } + + @Test + void testFunctionRegistrationAndUnsignedResult() + { + DecimalType resultType = createDecimalType(20, 0); + assertThat(assertions.function("cityHash64", "'Moscow'")) + .hasType(resultType) + .isEqualTo(decimal("12507901496292878638", resultType)); + + assertThat(assertions.function("cityHash64", "CAST(NULL AS varchar)")) + .isNull(resultType); + } +} diff --git a/plugin/trino-ducklake/src/test/java/io/trino/plugin/ducklake/function/TestFormatReadableTimeDeltaFunction.java b/plugin/trino-ducklake/src/test/java/io/trino/plugin/ducklake/function/TestFormatReadableTimeDeltaFunction.java new file mode 100644 index 000000000000..4ee7ce956aa8 --- /dev/null +++ b/plugin/trino-ducklake/src/test/java/io/trino/plugin/ducklake/function/TestFormatReadableTimeDeltaFunction.java @@ -0,0 +1,116 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.trino.plugin.ducklake.function; + +import io.airlift.slice.Slices; +import io.trino.plugin.ducklake.DuckLakePlugin; +import io.trino.spi.TrinoException; +import io.trino.sql.query.QueryAssertions; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; + +import java.util.Map; + +import static io.trino.spi.type.VarcharType.VARCHAR; +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; + +@TestInstance(PER_CLASS) +final class TestFormatReadableTimeDeltaFunction +{ + private QueryAssertions assertions; + + @BeforeAll + void init() + { + assertions = new QueryAssertions(); + assertions.addPlugin(new DuckLakePlugin()); + } + + @AfterAll + void teardown() + { + assertions.close(); + assertions = null; + } + + @Test + void testClickHouseCompatibility() + { + Map expectedResults = Map.ofEntries( + Map.entry("5.5", "5 seconds"), + Map.entry("330.0", "5 minutes and 30 seconds"), + Map.entry("475200.0", "5 days and 12 hours"), + Map.entry("CAST(14256000 AS double)", "5 months, 12 days and 12 hours"), + Map.entry("CAST(173448000 AS double)", "5 years, 5 months and 30 days"), + Map.entry("-34261261.0", "-1 year, 1 month, 1 day, 1 hour, 1 minute and 1 second"), + Map.entry("CAST(0 AS double)", "0 seconds"), + Map.entry("CAST('Infinity' AS double)", "inf"), + Map.entry("CAST('-Infinity' AS double)", "-inf"), + Map.entry("nan()", "nan"), + Map.entry("1e100", "3.1709791983764585e92 years"), + Map.entry("CAST(1152921504606846976 AS double)", "36558901084 years")); + + expectedResults.forEach((input, expected) -> assertThat(assertions.function("format_readable_time_delta", input)) + .hasType(VARCHAR) + .isEqualTo(expected)); + } + + @Test + void testMaximumAndMinimumUnits() + { + assertThat(assertions.function("format_readable_time_delta", "475200.0", "'minutes'")) + .isEqualTo("7920 minutes"); + assertThat(assertions.function("format_readable_time_delta", "67.79797979", "'milliseconds'")) + .isEqualTo("67797 milliseconds, 979 microseconds and 790 nanoseconds"); + assertThat(assertions.function("format_readable_time_delta", "3601.000000003", "'hours'", "'microseconds'")) + .isEqualTo("1 hour and 1 second"); + assertThat(assertions.function("format_readable_time_delta", "1.0005", "'milliseconds'")) + .isEqualTo("1000 milliseconds and 500 microseconds"); + assertThat(assertions.function("format_readable_time_delta", "CAST(0 AS double)", "'milliseconds'")) + .isEqualTo("0 nanoseconds"); + assertThat(assertions.function("format_readable_time_delta", "CAST(0 AS double)", "'years'", "'years'")) + .isEqualTo(""); + } + + @Test + void testInvalidUnits() + { + assertThatThrownBy(() -> FormatReadableTimeDeltaFunction.formatReadableTimeDelta( + 1.0, + Slices.utf8Slice("second"))) + .isInstanceOf(TrinoException.class) + .hasMessageContaining("Unexpected value of maximum unit argument"); + assertThatThrownBy(() -> FormatReadableTimeDeltaFunction.formatReadableTimeDelta( + 1.0, + Slices.utf8Slice("seconds"), + Slices.utf8Slice("hours"))) + .isInstanceOf(TrinoException.class) + .hasMessageContaining("must not be greater than maximum unit"); + } + + @Test + void testNullPropagation() + { + assertThat(assertions.function("format_readable_time_delta", "CAST(NULL AS double)")) + .isNull(VARCHAR); + assertThat(assertions.function("format_readable_time_delta", "1.0", "CAST(NULL AS varchar)")) + .isNull(VARCHAR); + assertThat(assertions.function("format_readable_time_delta", "1.0", "'hours'", "CAST(NULL AS varchar)")) + .isNull(VARCHAR); + } +} diff --git a/pom.xml b/pom.xml index 82d64d8b4849..5511ddb43c17 100644 --- a/pom.xml +++ b/pom.xml @@ -32,6 +32,8 @@ client/trino-client client/trino-jdbc core/trino-grammar + core/trino-hogql-compiler + core/trino-hogql-parser core/trino-main core/trino-parser core/trino-server @@ -1121,6 +1123,18 @@ ${project.version} + + io.trino + trino-hogql-compiler + ${project.version} + + + + io.trino + trino-hogql-parser + ${project.version} + + io.trino trino-hudi @@ -1790,6 +1804,12 @@ + + org.antlr + antlr4 + ${dep.antlr.version} + + org.antlr antlr4-runtime