Introduce -Xstatic-dfa

.gitignore

@@ -15,6 +15,7 @@ build/
 dist/
 lib/
 user.build.properties
+.factorypath
 
 # MacOSX files
 .DS_Store

AGENTS.md

@@ -0,0 +1,93 @@
+# AGENTS.md
+
+## What this repo is
+
+This is **sdf-labs/antlr4** (`origin`), a **fork of antlr/antlr4** (`upstream`) maintained
+for [dbt-fusion](https://github.com/dbt-labs/dbt-fusion). The fork's reason for existing is
+the **Rust target**: the Rust runtime crate (`runtime/Rust`, published as `dbt-antlr4`) and
+its tool codegen. Most upstream targets (Java, C++, Go, etc.) are inherited and rarely
+touched here — nearly all fork work happens in `runtime/Rust` and the tool's Rust codegen.
+
+Two version numbers, kept in lockstep since 2.0.0 but set in different places:
+- Tool/Java/Maven version: `pom.xml` `2.0.0` (the shaded tool jar is named
+  `dbt-antlr4-<version>-complete.jar` via `shadedArtifactId` in `tool/pom.xml`;
+  the Maven coordinate remains `org.antlr:antlr4`)
+- Rust runtime crate version: `runtime/Rust/Cargo.toml` (`dbt-antlr4`, `2.0.0`)
+
+
+## Build (Maven, builds the tool that generates parsers)
+
+ANTLR is written in itself, so the build has a self-dependency. Use `install`, not `compile`:
+
+```bash
+export MAVEN_OPTS="-Xmx1G"     # required on Linux
+mvn install -DskipTests        # produces tool/target/dbt-antlr4-<version>-complete.jar
+```
+
+Maven modules (`pom.xml`): `runtime/Java`, `tool`, `antlr4-maven-plugin`, `tool-testsuite`,
+`runtime-testsuite`. JDK 21, Maven 3.8+.
+
+## Rust target: the important workflow
+
+The Rust runtime is a Cargo crate, **not** part of the Maven build. To work on it:
+
+```bash
+cd runtime/Rust
+cargo build
+cargo test          # runs tests against the checked-in generated parsers in tests/gen/
+```
+
+Cargo features (off by default): `recursion-limit`, `arena-allocation-limit` — these gate
+fork-specific safety limits added in recent commits. The `stacker` dep is optional.
+
+### Regenerating the Rust test parsers (critical, easy to get wrong)
+
+`runtime/Rust/tests/gen/` contains **checked-in generated parsers**. If you change the Rust
+codegen template you must regenerate them, and this requires the freshly-built tool jar:
+
+```bash
+# 1. build the tool jar first (from repo root):
+mvn install -DskipTests
+# 2. regenerate (script hardcodes TOOL_VERSION — update it if pom.xml version changed):
+cd runtime/Rust && ./gen_test_grammars.sh
+```
+
+`gen_test_grammars.sh` has `TOOL_VERSION="2.0.0"` hardcoded and reads
+`tool/target/dbt-antlr4-${TOOL_VERSION}-complete.jar`. Keep it in sync with `pom.xml`.
+
+### Single source of truth for the Rust codegen template
+
+`runtime/Rust/templates/Rust.stg` is a **symlink** to
+`tool/resources/org/antlr/v4/tool/templates/codegen/Rust/Rust.stg`. Edit only the real file
+under `tool/resources/...`; do not replace the symlink with a copy.
+
+Rust tool target Java class: `tool/src/org/antlr/v4/codegen/target/RustTarget.java`.
+
+## Runtime test suite (cross-target, JVM-driven)
+
+`runtime-testsuite` tests every target by launching processes to compile/run generated code.
+Tests are defined as text descriptor files (not Java), see
+`doc/antlr-project-testing.md`. Run a single target from the `runtime-testsuite` dir:
+
+```bash
+cd runtime-testsuite
+export MAVEN_OPTS="-Xmx1G"
+mvn -Dtest='rust.**' test       # or java.**, cpp.**, go.** ...
+```
+
+Rust runner lives at `runtime-testsuite/test/org/antlr/v4/test/runtime/rust/` and shells out
+to `cargo`. Note: Rust is wired into `runtime-testsuite` but is **not** in the CI matrix in
+`.github/workflows/hosted.yml` — CI does not test Rust, so verify Rust changes locally.
+
+Two Rust suites exist: `rust.RustRuntimeTests` (adaptive prediction, all descriptors) and
+`rust.RustStaticDFARuntimeTests` (same corpus generated with `-Xstatic-dfa`; a differential
+test of table-driven static prediction — skips `showDiagnosticErrors`/`traceATN` descriptors,
+which introspect the adaptive engine that static tables bypass).
+
+Tool-only unit tests: `cd tool-testsuite && mvn test`.
+
+## Conventions
+
+- Commits require DCO sign-off: `git commit -s` (lowercase `-s`). See `CONTRIBUTING.md`.
+- `.editorconfig` defines the Java style; `runtime/Rust/rustfmt.toml` (edition 2021) the Rust style.
+- Contributor conduct rules: `ANTLR-HOUSE-RULES.md`.

antlr4-maven-plugin/pom.xml

@@ -8,7 +8,7 @@
   <parent>
     <groupId>org.antlr</groupId>
     <artifactId>antlr4-master</artifactId>
-    <version>2.0.0</version>
+    <version>2.0.2</version>
   </parent>
   <artifactId>antlr4-maven-plugin</artifactId>
   <packaging>maven-plugin</packaging>

pom.xml

@@ -13,7 +13,7 @@
 	</parent>
 	<groupId>org.antlr</groupId>
 	<artifactId>antlr4-master</artifactId>
-	<version>2.0.0</version>
+	<version>2.0.2</version>
 	<packaging>pom</packaging>
 
 	<name>ANTLR 4</name>

runtime-testsuite/pom.xml

@@ -10,7 +10,7 @@
 	<parent>
 		<groupId>org.antlr</groupId>
 		<artifactId>antlr4-master</artifactId>
-		<version>2.0.0</version>
+		<version>2.0.2</version>
 	</parent>
 	<artifactId>antlr4-runtime-testsuite</artifactId>
 	<name>ANTLR 4 Runtime Tests (4th generation)</name>

runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeRunner.java

@@ -143,8 +143,20 @@ private static class InitializationStatus {
 		cacheDirectory = new File(System.getProperty("java.io.tmpdir"), "ANTLR-runtime-testsuite-cache").getAbsolutePath();
 	}
 
+	/**
+	 * Identifies a distinct runner configuration sharing a language runtime
+	 * (e.g. Rust with and without -Xstatic-dfa). Defaults to the language
+	 * itself. Configurations get separate on-disk caches and separate
+	 * initialization locks so that two configurations of one language can
+	 * never clean or rebuild a cache directory out from under the other,
+	 * even when run concurrently in the same or in separate JVMs.
+	 */
+	protected String getRuntimeConfigurationName() {
+		return getLanguage();
+	}
+
 	protected final String getCachePath() {
-		return getCachePath(getLanguage());
+		return getCachePath(getRuntimeConfigurationName());
 	}
 
 	public static String getCachePath(String language) {
@@ -268,15 +280,15 @@ protected void addExtraRecognizerParameters(ST template) {
 	}
 
 	private boolean initAntlrRuntimeIfRequired(RunOptions runOptions) {
-		String language = getLanguage();
+		String configurationName = getRuntimeConfigurationName();
 		InitializationStatus status;
 
-		// Create initialization status for every runtime with lock object
+		// Create initialization status for every runtime configuration with lock object
 		synchronized (runtimeInitializationStatuses) {
-			status = runtimeInitializationStatuses.get(language);
+			status = runtimeInitializationStatuses.get(configurationName);
 			if (status == null) {
 				status = new InitializationStatus();
-				runtimeInitializationStatuses.put(language, status);
+				runtimeInitializationStatuses.put(configurationName, status);
 			}
 		}
 

runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeTests.java

@@ -41,6 +41,14 @@
 public abstract class RuntimeTests {
 	protected abstract RuntimeRunner createRuntimeRunner();
 
+	/**
+	 * Subclasses may exclude descriptors that do not apply to their
+	 * configuration (e.g. the Rust static-DFA suite skips descriptors that
+	 * introspect the adaptive prediction engine). Skipped descriptors are
+	 * not materialized as tests at all.
+	 */
+	protected boolean skipDescriptor(RuntimeTestDescriptor descriptor) { return false; }
+
 	private final static HashMap<String, RuntimeTestDescriptor[]> testDescriptors = new HashMap<>();
 	private final static Map<String, STGroup> cachedTargetTemplates = new HashMap<>();
 	private final static StringRenderer rendered = new StringRenderer();
@@ -96,6 +104,9 @@ public List<DynamicNode> runtimeTests() {
 			ArrayList<DynamicNode> descriptorTests = new ArrayList<>();
 			RuntimeTestDescriptor[] descriptors = testDescriptors.get(group);
 			for (RuntimeTestDescriptor descriptor : descriptors) {
+				if (skipDescriptor(descriptor)) {
+					continue;
+				}
 				descriptorTests.add(dynamicTest(descriptor.name, descriptor.uri, () -> {
 					try (RuntimeRunner runner = createRuntimeRunner()) {
 						String errorMessage = test(descriptor, runner);

runtime-testsuite/test/org/antlr/v4/test/runtime/rust/RustRuntimeTests.java

@@ -8,7 +8,19 @@
 
 import org.antlr.v4.test.runtime.RuntimeRunner;
 import org.antlr.v4.test.runtime.RuntimeTests;
+import org.junit.jupiter.api.parallel.Execution;
+import org.junit.jupiter.api.parallel.ExecutionMode;
 
+/**
+ * Runs the descriptor corpus against the Rust target with adaptive
+ * prediction. Executed {@link ExecutionMode#SAME_THREAD} so this suite never
+ * overlaps {@link RustStaticDFARuntimeTests} in the forked JVM: with
+ * concurrent classes, surefire 2.22.0 mis-attributes the two suites'
+ * dynamic tests to whichever class reports first (706/1-style splits), and
+ * any failure is reported against the wrong suite. Tests within the suite
+ * still run concurrently via the {@code @TestFactory}'s own annotation.
+ */
+@Execution(ExecutionMode.SAME_THREAD)
 public class RustRuntimeTests extends RuntimeTests {
 	@Override
 	protected RuntimeRunner createRuntimeRunner() {

runtime-testsuite/test/org/antlr/v4/test/runtime/rust/RustStaticDFARunner.java

@@ -0,0 +1,39 @@
+/*
+ * Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+package org.antlr.v4.test.runtime.rust;
+
+import org.antlr.v4.test.runtime.RunOptions;
+
+import java.util.List;
+
+/**
+ * A {@link RustRunner} that generates the recognizers with
+ * {@code -Xstatic-dfa}, so every statically-decidable decision is driven by
+ * a precomputed prediction table instead of {@code adaptive_predict}. Used
+ * by {@link RustStaticDFARuntimeTests} to run the descriptor corpus as a
+ * large differential test: expected outputs were authored against adaptive
+ * prediction, so any deviation of the table-driven path fails the test.
+ */
+public class RustStaticDFARunner extends RustRunner {
+	/**
+	 * Keep this configuration's build cache separate from the adaptive
+	 * suite's: both otherwise share {@code <tmp>/ANTLR-runtime-testsuite-cache/Rust},
+	 * where {@code cargo clean} during one suite's runtime initialization can
+	 * delete the rlib out from under the other suite's compiles when they
+	 * overlap (same JVM with concurrent classes, or concurrent Maven JVMs).
+	 */
+	@Override
+	protected String getRuntimeConfigurationName() {
+		return "Rust-static-dfa";
+	}
+
+	@Override
+	protected List<String> getTargetToolOptions(RunOptions ro) {
+		List<String> options = super.getTargetToolOptions(ro);
+		options.add("-Xstatic-dfa");
+		return options;
+	}
+}

runtime-testsuite/test/org/antlr/v4/test/runtime/rust/RustStaticDFARuntimeTests.java

@@ -0,0 +1,47 @@
+/*
+ * Copyright (c) 2012-2022 The ANTLR Project. All rights reserved.
+ * Use of this file is governed by the BSD 3-clause license that
+ * can be found in the LICENSE.txt file in the project root.
+ */
+
+package org.antlr.v4.test.runtime.rust;
+
+import org.antlr.v4.test.runtime.RuntimeRunner;
+import org.antlr.v4.test.runtime.RuntimeTestDescriptor;
+import org.antlr.v4.test.runtime.RuntimeTests;
+import org.junit.jupiter.api.parallel.Execution;
+import org.junit.jupiter.api.parallel.ExecutionMode;
+
+/**
+ * Runs the full runtime-test descriptor corpus against the Rust target with
+ * {@code -Xstatic-dfa} enabled (see {@link RustStaticDFARunner}). Since all
+ * expected outputs were authored against {@code adaptive_predict}, this
+ * suite is a broad differential test of the table-driven static prediction
+ * path: parse trees, matched text, and error messages must be identical.
+ *
+ * <p>Descriptors that introspect the adaptive prediction engine itself are
+ * excluded: ambiguity/full-context diagnostics ({@code showDiagnosticErrors})
+ * and ATN traces ({@code traceATN}) are reported from inside
+ * {@code adaptive_predict}, which statically-decided decisions bypass by
+ * design (a trusted exact ambiguity, for example, is resolved to its minimum
+ * alternative by the table without a runtime ambiguity report).</p>
+ *
+ * <p>Executed {@link ExecutionMode#SAME_THREAD} so this suite never overlaps
+ * {@link RustRuntimeTests} in the forked JVM (see that class for why), and
+ * its runner uses a separate build-cache directory
+ * ({@link RustStaticDFARunner#getRuntimeConfigurationName()}) so the two
+ * suites cannot clobber each other's compiled runtime even across
+ * concurrent Maven JVMs.</p>
+ */
+@Execution(ExecutionMode.SAME_THREAD)
+public class RustStaticDFARuntimeTests extends RuntimeTests {
+	@Override
+	protected RuntimeRunner createRuntimeRunner() {
+		return new RustStaticDFARunner();
+	}
+
+	@Override
+	protected boolean skipDescriptor(RuntimeTestDescriptor descriptor) {
+		return descriptor.showDiagnosticErrors || descriptor.traceATN;
+	}
+}

runtime/Java/pom.xml

@@ -9,7 +9,7 @@
 	<parent>
 		<groupId>org.antlr</groupId>
 		<artifactId>antlr4-master</artifactId>
-		<version>2.0.0</version>
+		<version>2.0.2</version>
 		<relativePath>../../pom.xml</relativePath>
 	</parent>
 	<artifactId>antlr4-runtime</artifactId>