Add recursive descent XQuery parser (opt-in via feature flag)

exist-core/src/main/antlr/org/exist/xquery/parser/XQuery.g

@@ -2669,6 +2669,7 @@ options {
 		$setType(XML_COMMENT);
 	}
 	|
+	{ !inStringConstructor }?
 	( XML_PI_START )
 	=> XML_PI { $setType(XML_PI); }
 	|

exist-core/src/main/java/org/exist/dom/persistent/SortedNodeSet.java

@@ -86,20 +86,28 @@ public void addAll(final NodeSet other) {
         try(final DBBroker broker = pool.get(Optional.ofNullable(user))) {
 
             final XQueryContext context = new XQueryContext(pool);
-            final XQueryLexer lexer = new XQueryLexer(context, new StringReader(sortExpr));
-            final XQueryParser parser = new XQueryParser(lexer);
-            final XQueryTreeParser treeParser = new XQueryTreeParser(context);
-            parser.xpath();
-            if(parser.foundErrors()) {
-                //TODO : error ?
-                LOG.debug(parser.getErrorMessage());
-            }
-            final AST ast = parser.getAST();
-            LOG.debug("generated AST: {}", ast.toStringTree());
-            final PathExpr expr = new PathExpr(context);
-            treeParser.xpath(ast, expr);
-            if(treeParser.foundErrors()) {
-                LOG.debug(treeParser.getErrorMessage());
+            final PathExpr expr;
+            if (org.exist.xquery.XQuery.useRdParser()) {
+                final org.exist.xquery.parser.next.XQueryParser rdParser =
+                        new org.exist.xquery.parser.next.XQueryParser(context, sortExpr);
+                final Expression rootExpr = rdParser.parse();
+                expr = rootExpr instanceof PathExpr ? (PathExpr) rootExpr : new PathExpr(context);
+                if (!(rootExpr instanceof PathExpr)) { expr.add(rootExpr); }
+            } else {
+                expr = new PathExpr(context);
+                final XQueryLexer lexer = new XQueryLexer(context, new StringReader(sortExpr));
+                final XQueryParser parser = new XQueryParser(lexer);
+                final XQueryTreeParser treeParser = new XQueryTreeParser(context);
+                parser.xpath();
+                if (parser.foundErrors()) {
+                    LOG.debug(parser.getErrorMessage());
+                }
+                final AST ast = parser.getAST();
+                LOG.debug("generated AST: {}", ast.toStringTree());
+                treeParser.xpath(ast, expr);
+                if (treeParser.foundErrors()) {
+                    LOG.debug(treeParser.getErrorMessage());
+                }
             }
             expr.analyze(new AnalyzeContextInfo());
             for(final SequenceIterator i = other.iterate(); i.hasNext(); ) {

exist-core/src/main/java/org/exist/source/AbstractSource.java

@@ -89,6 +89,9 @@ public QName isModule() throws IOException {
      * @param is the input stream
      * @return The guessed encoding.
      */
+    // TODO(rd-parser): DeclScanner is a lightweight ANTLR 2 pre-scanner that extracts
+    // version/encoding declarations without full parsing. The rd parser may need an
+    // equivalent lightweight method (e.g., XQueryParser.scanVersionDecl).
     protected static String guessXQueryEncoding(final InputStream is) {
         final XQueryLexer lexer = new XQueryLexer(null, new InputStreamReader(is));
         final DeclScanner scanner = new DeclScanner(lexer);

exist-core/src/main/java/org/exist/xquery/GeneralComparison.java

@@ -128,17 +128,17 @@ public GeneralComparison( XQueryContext context, Expression left, Expression rig
         this.relation   = relation;
         this.truncation = truncation;
 
-        if( ( left instanceof PathExpr ) && ( ( ( PathExpr )left ).getLength() == 1 ) ) {
+        if( isSimplifiablePathExpr( left ) ) {
             left                  = ( ( PathExpr )left ).getExpression( 0 );
             didLeftSimplification = true;
         }
-        add( left );
+        addOperand( left );
 
-        if( ( right instanceof PathExpr ) && ( ( ( PathExpr )right ).getLength() == 1 ) ) {
+        if( isSimplifiablePathExpr( right ) ) {
             right                  = ( ( PathExpr )right ).getExpression( 0 );
             didRightSimplification = true;
         }
-        add( right );
+        addOperand( right );
 
         //TODO : should we also use simplify() here ? -pb
         if( didLeftSimplification ) {
@@ -150,6 +150,26 @@ public GeneralComparison( XQueryContext context, Expression left, Expression rig
         }
     }
 
+    /**
+     * Check if an expression is a plain PathExpr container that can be safely unwrapped.
+     * Function, BinaryOp, and other PathExpr subclasses that use steps for their own
+     * purposes must NOT be unwrapped — doing so would replace the expression with its
+     * operands/arguments.
+     */
+    private static boolean isSimplifiablePathExpr( final Expression expr ) {
+        return expr instanceof PathExpr
+                && expr.getClass() == PathExpr.class
+                && ( ( PathExpr )expr ).getLength() == 1;
+    }
+
+    /**
+     * Add an operand expression using the Expression overload (not PathExpr)
+     * to prevent flattening of Function/BinaryOp subclasses of PathExpr.
+     */
+    private void addOperand( final Expression expr ) {
+        steps.add( expr );
+    }
+
     /* (non-Javadoc)
      * @see org.exist.xquery.BinaryOp#analyze(org.exist.xquery.AnalyzeContextInfo)
      */
@@ -1065,7 +1085,8 @@ private AtomicValue convertForValueComparison(final AtomicValue value, final int
             /*
              *  d. Otherwise, a type error is raised [err:XPTY0004].
              */
-            throw new XPathException(this, ErrorCodes.XPTY0004, "Incompatible primitive types");
+            throw new XPathException(this, ErrorCodes.XPTY0004,
+                    "Incompatible primitive types: " + Type.getTypeName(thisType) + " vs " + Type.getTypeName(otherType));
         }
 
         return value;

exist-core/src/main/java/org/exist/xquery/Predicate.java

@@ -72,7 +72,9 @@ public Predicate(final XQueryContext context) {
 
     @Override
     public void addPath(final PathExpr path) {
-        if (path.getSubExpressionCount() == 1) {
+        // Only unwrap plain PathExpr containers, not Function/BinaryOp subclasses
+        // which use steps for their own purposes (arguments, operands)
+        if (path.getClass() == PathExpr.class && path.getSubExpressionCount() == 1) {
             add(path.getSubExpression(0));
         } else {
             super.addPath(path);

exist-core/src/main/java/org/exist/xquery/XQuery.java

@@ -195,12 +195,28 @@
      * @throws XPathException if an error occurs during compilation
      * @throws PermissionDeniedException if the caller is not permitted to compile the XQuery
      */
+    /**
+     * System property to select the XQuery parser implementation.
+     * Set to "rd" to use the hand-written recursive descent parser.
+     * Default is "antlr2" (the ANTLR 2 generated parser).
+     */
+    public static final String PROPERTY_PARSER = "exist.parser";
+
+    public static boolean useRdParser() {
+        return "rd".equalsIgnoreCase(System.getProperty(PROPERTY_PARSER, "antlr2"));
+    }
+
     private CompiledXQuery compile(final XQueryContext context, final Reader reader, final boolean xpointer) throws XPathException, PermissionDeniedException {
 
         //check read permission
         if (context.getSource() instanceof DBSource) {
             ((DBSource) context.getSource()).validate(Permission.READ);
         }
+
+        // Feature flag: use hand-written recursive descent parser if enabled
+        if (useRdParser() && !xpointer) {
+            return compileWithRdParser(context, reader);
+        }
 
 
     	//TODO: move XQueryContext.getUserFromHttpSession() here, have to check if servlet.jar is in the classpath
@@ -316,6 +332,60 @@
      *
      * @return true if this is a library module, false otherwise
      */
+    private CompiledXQuery compileWithRdParser(final XQueryContext context, final Reader reader)
+            throws XPathException {
+        final long start = System.currentTimeMillis();
+        try {
+            final String source = readFully(reader);
+            final org.exist.xquery.parser.next.XQueryParser rdParser =
+                    new org.exist.xquery.parser.next.XQueryParser(context, source);
+
+            final Expression rootExpr = rdParser.parse();
+
+            // Set root expression on context — required for resetState() during concurrent execution
+            context.setRootExpression(rootExpr);
+            context.getRootContext().resolveForwardReferences();
+
+            // For library modules, return LibraryModuleRoot so execute() can
+            // dispatch function calls by name (triggers, fn:load-xquery-module)
+            final PathExpr result;
+            if (rdParser.isLibraryModule()) {
+                result = new LibraryModuleRoot(context);
+                if (rootExpr instanceof PathExpr) {
+                    for (int i = 0; i < ((PathExpr) rootExpr).getLength(); i++) {
+                        result.add(((PathExpr) rootExpr).getExpression(i));
+                    }
+                }
+            } else if (rootExpr instanceof PathExpr) {
+                result = (PathExpr) rootExpr;
+            } else {
+                result = new PathExpr(context);
+                result.add(rootExpr);
+            }
+
+            context.analyzeAndOptimizeIfModulesChanged(result);
+
+            if (LOG.isDebugEnabled()) {
+                final NumberFormat nf = NumberFormat.getNumberInstance();
+                LOG.debug("Recursive descent parser compilation took {} ms", nf.format(System.currentTimeMillis() - start));
+            }
+
+            return result;
+        } catch (final IOException e) {
+            throw new XPathException(context.getRootExpression(), "Error reading query source: " + e.getMessage(), e);
+        }
+    }
+
+    private static String readFully(final Reader reader) throws IOException {
+        final StringBuilder sb = new StringBuilder(4096);
+        final char[] buf = new char[4096];
+        int n;
+        while ((n = reader.read(buf)) != -1) {
+            sb.append(buf, 0, n);
+        }
+        return sb.toString();
+    }
+
     static boolean isLibraryModule(AST ast) {
         while (ast != null) {
             if (ast.getType() == XQueryTreeParser.MODULE_DECL) {

exist-core/src/main/java/org/exist/xquery/XQueryContext.java

@@ -2778,6 +2778,82 @@
 
             final XQueryContext modContext = new ModuleContext(this, namespaceURI, prefix, location);
             modExternal.setContext(modContext);
+            // rd parser compileModule routing: GeneralComparison PathExpr unwrapping
+            // bug is fixed. Remaining blocker: rd parser fails on inline functions
+            // inside parenthesized sequences — e.g., (function ($a) {1}, ...) in
+            // bang.xql line 258. The parser doesn't recognize `function` as starting
+            // an inline function in this context. This is a general rd parser bug,
+            // not compileModule-specific. Re-enable once inline function parsing is fixed.
+            if (false && XQuery.useRdParser()) {
+                try {
+                    final StringBuilder sb = new StringBuilder(4096);
+                    final char[] buf = new char[4096];
+                    int n;
+                    while ((n = reader.read(buf)) != -1) sb.append(buf, 0, n);
+                    final String sourceText = sb.toString();
+                    if (LOG.isTraceEnabled()) {
+                        LOG.trace("compileModule rd-parser: source length={}, namespace={}, first200={}",
+                                sourceText.length(), namespaceURI,
+                                sourceText.substring(0, Math.min(200, sourceText.length())).replace("\n", "\\n"));
+                    }
+                    final org.exist.xquery.parser.next.XQueryParser rdParser =
+                            new org.exist.xquery.parser.next.XQueryParser(modContext, sourceText);
+                    final Expression parsedExpr = rdParser.parse();
+                    // Wrap in LibraryModuleRoot for function dispatch
+                    final Expression rootExpr;
+                    if (rdParser.isLibraryModule()) {
+                        final LibraryModuleRoot libRoot = new LibraryModuleRoot(modContext);
+                        if (parsedExpr instanceof PathExpr) {
+                            for (int ii = 0; ii < ((PathExpr) parsedExpr).getLength(); ii++) {
+                                libRoot.add(((PathExpr) parsedExpr).getExpression(ii));
+                            }
+                        }
+                        rootExpr = libRoot;
+                    } else {
+                        rootExpr = parsedExpr;
+                    }
+                    modContext.setRootExpression(rootExpr);
+                    modContext.resolveForwardReferences();
+
+                    for (final java.util.Iterator<UserDefinedFunction> it = modContext.localFunctions(); it.hasNext(); ) {
+                        modExternal.declareFunction(it.next());
+                    }
+                    // Register module-level variables from the parsed expression tree.
+                    // The rd parser adds VariableDeclaration expressions to rootExpr,
+                    // which need to be registered on the module (like ANTLR 2's
+                    // myModule.declareVariable(qn, decl) during tree walking).
+                    if (parsedExpr instanceof PathExpr) {
+                        final PathExpr rootPath = (PathExpr) parsedExpr;
+                        for (int vi = 0; vi < rootPath.getLength(); vi++) {
+                            final Expression step = rootPath.getExpression(vi);
+                            if (step instanceof VariableDeclaration) {
+                                final VariableDeclaration decl = (VariableDeclaration) step;
+                                modExternal.declareVariable(decl.getName(), decl);
+                            }
+                        }
+                    }
+                    // Also register any variables already in the context
+                    for (final Variable var : modContext.getVariables().values()) {
+                        if (var.getQName().getNamespaceURI().equals(namespaceURI)) {
+                            modExternal.declareVariable(var);
+                        }
+                    }
+                    modExternal.setRootExpression(rootExpr);
+
+                    if (namespaceURI != null && !modExternal.getNamespaceURI().equals(namespaceURI)) {
+                        throw new XPathException(rootExpression, ErrorCodes.XQST0059,
+                                "namespace URI declared by module (" + modExternal.getNamespaceURI() +
+                                ") does not match namespace URI in import statement, which was: " + namespaceURI);
+                    }
+                    modExternal.setSource(source);
+                    modContext.setSource(source);
+                    modExternal.setIsReady(true);
+                    return modExternal;
+                } catch (final XPathException e) {
+                    e.prependMessage("Error while loading module " + location + ": ");
+                    throw e;
+                }
+            }
             final XQueryLexer lexer = new XQueryLexer(modContext, reader);
             final XQueryParser parser = new XQueryParser(lexer);
             final XQueryTreeParser astParser = new XQueryTreeParser(modContext, modExternal);
@@ -2807,12 +2883,6 @@
                     throw new XPathException(rootExpression, ErrorCodes.XQST0059, "namespace URI declared by module (" + modExternal.getNamespaceURI() + ") does not match namespace URI in import statement, which was: " + namespaceURI);
                 }
 
-                // Set source information on module context
-//            String sourceClassName = source.getClass().getName();
-//            modContext.setSourceKey(source.getKey().toString());
-                // Extract the source type from the classname by removing the package prefix and the "Source" suffix
-//            modContext.setSourceType( sourceClassName.substring( 17, sourceClassName.length() - 6 ) );
-
                 modExternal.setSource(source);
                 modContext.setSource(source);
                 modExternal.setIsReady(true);

exist-core/src/main/java/org/exist/xquery/functions/util/Compile.java

@@ -28,6 +28,7 @@
 import org.exist.dom.QName;
 import org.exist.dom.memtree.MemTreeBuilder;
 import org.exist.xquery.AnalyzeContextInfo;
+import org.exist.xquery.Expression;
 import org.exist.xquery.BasicFunction;
 import org.exist.xquery.Cardinality;
 import org.exist.xquery.ErrorCodes;
@@ -124,6 +125,21 @@ public Sequence eval(Sequence[] args, Sequence contextSequence)
 			if (getArgumentCount() == 2 && args[1].hasOne()) {
 				pContext.setModuleLoadPath(args[1].getStringValue());
 			}
+			// Route through rd parser if enabled
+			if (org.exist.xquery.XQuery.useRdParser()) {
+				try {
+					final org.exist.xquery.parser.next.XQueryParser rdParser =
+							new org.exist.xquery.parser.next.XQueryParser(pContext, expr);
+					final Expression rootExpr = rdParser.parse();
+					if (rootExpr instanceof PathExpr) {
+						((PathExpr) rootExpr).analyze(new AnalyzeContextInfo());
+					}
+				} catch (final XPathException e) {
+					line = e.getLine();
+					column = e.getColumn();
+					error = e.getDetailMessage();
+				}
+			} else {
 			final XQueryLexer lexer = new XQueryLexer(pContext, new StringReader(expr));
 			final XQueryParser parser = new XQueryParser(lexer);
 			// shares the context of the outer expression
@@ -155,8 +171,8 @@ public Sequence eval(Sequence[] args, Sequence contextSequence)
 			} finally {
 				context.popNamespaceContext();
 				pContext.reset(false);
-
 			}
+			} // end else (ANTLR 2 path)
 
 			if (isCalledAs("compile")) {
 				return error == null ? Sequence.EMPTY_SEQUENCE : new StringValue(this, error);