[FEATURE rust-parser] Rust/WASM template parser using pest.rs

.github/workflows/ci-jobs.yml

@@ -37,8 +37,6 @@ jobs:
         run: pnpm build:types
       - name: Check internal types
         run: pnpm type-check:internals
-      - name: Check @handlebars/parser types
-        run: pnpm type-check:handlebars
       - name: Check published types
         run: pnpm type-check:types
 
@@ -222,7 +220,7 @@ jobs:
           SHOULD_TRANSPILE_FOR_NODE: true
         run: pnpm build
       - name: test
-        run: pnpm test:node && pnpm --filter "@handlebars/parser" test
+        run: pnpm test:node
 
   blueprint-test:
     name: Blueprint Tests

.github/workflows/glimmer-syntax-prettier-smoke-test.yml

@@ -17,7 +17,6 @@ on:
       - "packages/@glimmer/interfaces/**"
       - "packages/@glimmer/util/**"
       - "packages/@glimmer/wire-format/**"
-      - "packages/@handlebars/parser/**"
   pull_request:
     paths:
       - ".github/workflows/glimmer-syntax-prettier-smoke-test.yml"
@@ -27,7 +26,6 @@ on:
       - "packages/@glimmer/interfaces/**"
       - "packages/@glimmer/util/**"
       - "packages/@glimmer/wire-format/**"
-      - "packages/@handlebars/parser/**"
   workflow_dispatch:
 
 permissions:

.gitignore

@@ -22,7 +22,7 @@ dist-prod
 lib/*/tests/all.js
 lib/*/tests/qunit*
 lib/bundler/man
-pkg
+/pkg
 rdoc
 spade-boot.js
 spec/reports
@@ -55,3 +55,7 @@ npm-debug.log
 # couple of the files. Once it is, we can switch this over to just ignoring
 # `types/stable` entirely.
 types/stable
+
+# Rust build artifacts (WASM output in pkg/ is committed)
+packages/@glimmer/syntax/target/
+packages/@glimmer/syntax/Cargo.lock

.prettierignore

@@ -14,3 +14,9 @@ package.json
 pnpm-lock.yaml
 internal-docs/**/*.md
 tracerbench-testing/
+packages/@glimmer/syntax/target/
+packages/@glimmer/syntax/pkg/
+packages/@glimmer/syntax/Cargo.lock
+packages/@glimmer/syntax/src/
+*.pest
+*.wasm

eslint.config.mjs

@@ -28,8 +28,9 @@ export default [
       '**/type-tests/',
       'internal-docs/guides/**',
       'packages/@glimmer-workspace/**',
-      'packages/@handlebars/parser/lib/parser.js',
-      'packages/@handlebars/parser/src/**',
+      'packages/@glimmer/syntax/pkg/**',
+      'packages/@glimmer/syntax/src/**',
+      'packages/@glimmer/syntax/target/**',
       'tracerbench-testing/',
     ],
   },
@@ -171,33 +172,6 @@ export default [
       'ember-internal/no-const-outside-module-scope': 'error',
     },
   },
-  {
-    files: ['packages/@handlebars/**/*.js'],
-
-    languageOptions: {
-      ecmaVersion: 2017,
-      sourceType: 'module',
-    },
-
-    rules: {
-      'ember-internal/require-yuidoc-access': 'off',
-      'ember-internal/no-const-outside-module-scope': 'off',
-      'disable-features/disable-async-await': 'off',
-      'disable-features/disable-generator-functions': 'off',
-      'no-implicit-coercion': 'off',
-      'no-unused-vars': 'off',
-      'import/namespace': 'off',
-    },
-  },
-  {
-    files: ['packages/@handlebars/parser/spec/**/*.js'],
-
-    languageOptions: {
-      globals: {
-        ...globals.mocha,
-      },
-    },
-  },
   {
     files: [
       'packages/*/tests/**/*.[jt]s',

internal-docs/guides/development/build-constraints.md

@@ -137,8 +137,6 @@ The build system has specific rules for what gets inlined vs treated as external
 - TypeScript helper library (`tslib`)
 
 **Always External:**
-- `@handlebars/parser`
-- `simple-html-tokenizer`
 - `babel-plugin-debug-macros`
 - Other `@glimmer/*` packages (to avoid duplication)
 - `@simple-dom/*` packages

package.json

@@ -55,7 +55,6 @@
     "test:browserstack": "node bin/run-browserstack-tests.js",
     "test:wip": "vite build --mode development --minify false && testem ci",
     "type-check:internals": "tsc --noEmit",
-    "type-check:handlebars": "tsc --noEmit --project packages/@handlebars/parser/tsconfig.json",
     "type-check:types": "tsc --noEmit --project type-tests",
     "type-check": "npm-run-all type-check:*"
   },
@@ -76,8 +75,7 @@
     "inflection": "^2.0.1",
     "route-recognizer": "^0.3.4",
     "semver": "^7.5.2",
-    "silent-error": "^1.1.1",
-    "simple-html-tokenizer": "^0.5.11"
+    "silent-error": "^1.1.1"
   },
   "devDependencies": {
     "@aws-sdk/client-s3": "^3.731.0",

packages/@glimmer-workspace/integration-tests/lib/snapshot.ts

@@ -1,10 +1,8 @@
 import type { Nullable, SimpleElement, SimpleNode } from '@glimmer/interfaces';
-import type { EndTag, Token } from 'simple-html-tokenizer';
 import { COMMENT_NODE, TEXT_NODE } from '@glimmer/constants';
-import { castToSimple, unwrap } from '@glimmer/debug-util';
-import { tokenize } from 'simple-html-tokenizer';
+import { unwrap } from '@glimmer/debug-util';
 
-import { replaceHTML, toInnerHTML } from './dom/simple-utils';
+import { toInnerHTML } from './dom/simple-utils';
 
 export type IndividualSnapshot = 'up' | 'down' | SimpleNode;
 export type NodesSnapshot = IndividualSnapshot[];
@@ -13,6 +11,40 @@ export function snapshotIsNode(snapshot: IndividualSnapshot): snapshot is Simple
   return snapshot !== 'up' && snapshot !== 'down';
 }
 
+// -- HTML equivalence (replaces simple-html-tokenizer) ----------------------
+//
+// Compares two HTML fragments for semantic equivalence by normalizing
+// attribute order via the browser's DOM parser. Ember ID attributes
+// are normalized to a stable counter so order-independent tests pass.
+
+function normalizeHTML(html: string): string {
+  const container = document.createElement('div');
+  container.innerHTML = html;
+  sortAttributes(container);
+  return container.innerHTML;
+}
+
+function sortAttributes(element: Element): void {
+  for (const child of Array.from(element.children)) {
+    if (child.attributes.length > 1) {
+      const attrs = Array.from(child.attributes).map((a) => [a.name, a.value] as const);
+      attrs.sort((a, b) => (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0));
+      for (const [name] of attrs) {
+        child.removeAttribute(name);
+      }
+      for (const [name, value] of attrs) {
+        child.setAttribute(name, value);
+      }
+    }
+    sortAttributes(child);
+  }
+}
+
+function cleanEmberIds(html: string): string {
+  let id = 0;
+  return html.replace(/ember(\d+|\*)/gu, () => `ember${++id}`);
+}
+
 export function equalTokens(
   testFragment: SimpleElement | string | null,
   testHTML: SimpleElement | string,
@@ -22,41 +54,17 @@ export function equalTokens(
     throw new Error(`Unexpectedly passed null to equalTokens`);
   }
 
-  const fragTokens = generateTokens(testFragment);
-  const htmlTokens = generateTokens(testHTML);
-
-  cleanEmberIds(fragTokens.tokens);
-  cleanEmberIds(htmlTokens.tokens);
-
-  const equiv = QUnit.equiv(fragTokens.tokens, htmlTokens.tokens);
-
-  if (equiv && fragTokens.html !== htmlTokens.html) {
-    QUnit.assert.deepEqual(
-      fragTokens.tokens,
-      htmlTokens.tokens,
-      message || 'expected tokens to match'
-    );
-  } else {
-    QUnit.assert.pushResult({
-      result: QUnit.equiv(fragTokens.tokens, htmlTokens.tokens),
-      actual: fragTokens.html,
-      expected: htmlTokens.html,
-      message: message || 'expected tokens to match',
-    });
-  }
-
-  // QUnit.assert.deepEqual(fragTokens.tokens, htmlTokens.tokens, msg);
-}
-
-function cleanEmberIds(tokens: Token[]) {
-  let id = 0;
+  const fragHTML = typeof testFragment === 'string' ? testFragment : toInnerHTML(testFragment);
+  const expectedHTML = typeof testHTML === 'string' ? testHTML : toInnerHTML(testHTML);
 
-  tokens.forEach((token) => {
-    const idAttr = 'attributes' in token && token.attributes.filter((a) => a[0] === 'id')[0];
+  const normalizedFrag = cleanEmberIds(normalizeHTML(fragHTML));
+  const normalizedExpected = cleanEmberIds(normalizeHTML(expectedHTML));
 
-    if (idAttr) {
-      idAttr[1] = idAttr[1].replace(/ember(\d+|\*)/u, `ember${++id}`);
-    }
+  QUnit.assert.pushResult({
+    result: normalizedFrag === normalizedExpected,
+    actual: fragHTML,
+    expected: expectedHTML,
+    message: message || 'expected tokens to match',
   });
 }
 
@@ -86,49 +94,6 @@ export function generateSnapshot(element: SimpleElement): SimpleNode[] {
   return snapshot;
 }
 
-function generateTokens(divOrHTML: SimpleElement | string): { tokens: Token[]; html: string } {
-  let div: SimpleElement;
-  if (typeof divOrHTML === 'string') {
-    div = castToSimple(document.createElement('div'));
-    replaceHTML(div, divOrHTML);
-  } else {
-    div = divOrHTML;
-  }
-
-  let tokens = tokenize(toInnerHTML(div), {});
-
-  tokens = tokens.reduce((tokens, token) => {
-    // eslint-disable-next-line @typescript-eslint/no-unsafe-enum-comparison
-    if (token.type === 'StartTag') {
-      if (token.attributes) {
-        token.attributes.sort((a, b) => {
-          if (a[0] > b[0]) {
-            return 1;
-          }
-          if (a[0] < b[0]) {
-            return -1;
-          }
-          return 0;
-        });
-      }
-
-      if (token.selfClosing) {
-        token.selfClosing = false;
-        tokens.push(token);
-        tokens.push({ type: 'EndTag', tagName: token.tagName } as EndTag);
-      } else {
-        tokens.push(token);
-      }
-    } else {
-      tokens.push(token);
-    }
-
-    return tokens;
-  }, new Array<Token>());
-
-  return { tokens, html: toInnerHTML(div) };
-}
-
 export function equalSnapshots(a: SimpleNode[], b: SimpleNode[]) {
   QUnit.assert.strictEqual(a.length, b.length, 'Same number of nodes');
   for (let i = 0; i < b.length; i++) {

packages/@glimmer-workspace/integration-tests/package.json

@@ -36,8 +36,7 @@
     "@simple-dom/serializer": "^1.4.0",
     "@simple-dom/void-map": "^1.4.0",
     "js-reporters": "^2.1.0",
-    "qunit": "^2.24.1",
-    "simple-html-tokenizer": "^0.5.11"
+    "qunit": "^2.24.1"
   },
   "devDependencies": {
     "@ember/runloop": "workspace:*",

packages/@glimmer/syntax/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "glimmer-template-parser"
+version = "0.1.0"
+edition = "2024"
+description = "A Rust-based PEG parser for Glimmer/Handlebars templates"
+
+[lib]
+crate-type = ["cdylib", "rlib"]
+
+[dependencies]
+pest = "2"
+pest_derive = "2"
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+wasm-bindgen = "0.2"
+serde-wasm-bindgen = "0.6"
+
+[profile.release]
+opt-level = "z"
+lto = true

packages/@glimmer/syntax/build.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$SCRIPT_DIR"
+
+echo "🦀 Building Glimmer template parser (WASM)..."
+
+# Clean previous builds
+rm -rf pkg/standalone pkg/wasm-bytes.mjs
+
+# Build only the web (standalone) target. We use a universal wrapper
+# (pkg/universal.mjs) that inlines the WASM bytes as base64, so there's
+# no need for separate Node/bundler targets.
+echo "  → Building web target..."
+wasm-pack build --target web --out-dir pkg/standalone 2>&1 | grep -v "^warning:" || true
+
+# Remove files that cause build:types issues
+rm -f pkg/standalone/.gitignore
+rm -f pkg/standalone/glimmer_template_parser_bg.wasm.d.ts
+
+# Generate the base64-encoded WASM bytes module for the universal wrapper
+echo "  → Generating wasm-bytes.mjs..."
+node -e "
+const fs = require('fs');
+const wasm = fs.readFileSync('pkg/standalone/glimmer_template_parser_bg.wasm');
+const base64 = wasm.toString('base64');
+const content = 'export const WASM_BYTES_BASE64 = ' + JSON.stringify(base64) + ';\n';
+fs.writeFileSync('pkg/wasm-bytes.mjs', content);
+console.log('    wasm-bytes.mjs: ' + (content.length / 1024).toFixed(1) + 'KB');
+"
+
+echo "✅ Build complete!"
+ls -lh pkg/standalone/glimmer_template_parser_bg.wasm pkg/wasm-bytes.mjs pkg/universal.mjs