diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index 2bb65eb8..b8f9a301 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "dotnet-stryker": { - "version": "4.8.1", + "version": "4.14.2", "commands": [ "dotnet-stryker" ] diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile deleted file mode 100644 index ef592a1c..00000000 --- a/.devcontainer/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM brendanburns/wasm-dev-base:0.0.6 - -# Dotnet -RUN apt install lld-14 -y -qq -RUN curl https://packages.microsoft.com/config/ubuntu/22.10/packages-microsoft-prod.deb -L --output packages-microsoft-prod.deb && \ - dpkg -i packages-microsoft-prod.deb && \ - rm packages-microsoft-prod.deb && \ - apt-get update && apt-get install -y dotnet-sdk-7.0 - diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json deleted file mode 100644 index cd3b7174..00000000 --- a/.devcontainer/devcontainer.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "name": "CSLY", - "build": { - "dockerfile": "Dockerfile", - "context": ".." - } -} diff --git a/.github/workflows/.dotnetcore.yml.un~ b/.github/workflows/.dotnetcore.yml.un~ deleted file mode 100644 index 35f1fb25..00000000 Binary files a/.github/workflows/.dotnetcore.yml.un~ and /dev/null differ diff --git a/.github/workflows/dotnetcore.yml b/.github/workflows/dotnetcore.yml index 57bbadf2..13c37214 100644 --- a/.github/workflows/dotnetcore.yml +++ b/.github/workflows/dotnetcore.yml @@ -35,7 +35,7 @@ jobs: - name: Build with dotnet run: dotnet build --configuration Release - name: Test with dotnet - uses: b3b00/coverlet-action@1.3.5-alpha1 + uses: b3b00/coverlet-action@1.3.5 id: 'coverlet' if: env.RUN_TESTS with: diff --git a/.github/workflows/dotnetcore.yml~ b/.github/workflows/dotnetcore.yml~ deleted file mode 100644 index ec120cfc..00000000 --- a/.github/workflows/dotnetcore.yml~ +++ /dev/null @@ -1,103 +0,0 @@ -name: .NET Core - -on: - push: - branches: - - dev - - tech/actions - - master - pull_request: - branches: - - dev -permissions: - pull-requests: write -jobs: - build: - env: - TESTS_PROJECT: 'ParserTests/ParserTests.csproj' # path to test project or solution - PUBLISH_NUGET: true # if true a nuget will be published on version change - RUN_TESTS: true # if true tests are run and coverage data is published to coveralls and a coverage report is produced. - MAIN_CSPROJ: 'sly/sly.csproj' # main project (for nuget packaging) - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, windows-latest] - steps: - - uses: actions/checkout@v2 - - name: Setup .NET Core - uses: actions/setup-dotnet@v1 - with: - dotnet-version: 3.1.101 - - name: Clean artifacts and nugets - run: dotnet clean --configuration Release && dotnet nuget locals all --clear - - name: Build with dotnet - run: dotnet build --configuration Release - - name: Test with dotnet - uses: b3b00/coverlet-action@1.2.2 - id: 'coverlet' - if: env.RUN_TESTS - with: - testProject: ${{env.TESTS_PROJECT}} - output: 'lcov.info' - threshold: 80 - outputFormat: 'lcov' - excludes: '[program]*,[expressionParser]*,[jsonparser]*,[while]*,[indentedWhile]*,[SimpleExpressionParser]*,[GenericLexerWithCallbacks]*,[indented]*' - - name: coveralls - uses: coverallsapp/github-action@v1.1.1 - if: matrix.os == 'windows-latest' && env.RUN_TESTS - with: - github-token: ${{secrets.GITHUB_TOKEN }} - path-to-lcov: ${{steps.coverlet.outputs.coverageFile}} - - name: ReportGenerator - uses: danielpalme/ReportGenerator-GitHub-Action@4.8.12 - with: - reports: ${{steps.coverlet.outputs.coverageFile}} - targetdir: 'coveragereport' - reporttypes: 'HtmlInline;MarkdownSummary' - verbosity: 'Info' # The verbosity level of the log messages. Values: Verbose, Info, Warning, Error, Off - tag: '${{ github.run_number }}_${{ github.run_id }}' -# - name: Publish coverage summary -# uses: marocchino/sticky-pull-request-comment@v2 -# with: -# path: coveragereport/Summary.md - - name: publish nuget - if: matrix.os == 'windows-latest' && env.PUBLISH_NUGET - id: publish_nuget - uses: brandedoutcast/publish-nuget@v2.5.2 - with: - VERSION_REGEX: (.*)<\/version> - PROJECT_FILE_PATH: ${{env.MAIN_CSPROJ}} - NUGET_KEY: ${{secrets.NUGET_KEY}} - VERSION_FILE_PATH: ${{env.MAIN_CSPROJ}} - - name: Create Release - if: ${{ success() && matrix.os == 'windows-latest' && steps.publish_nuget.outputs.VERSION != '' && steps.publish_nuget.outputs.VERSION != null }} - id: create_release - uses: actions/create-release@latest - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - tag_name: ${{ steps.publish_nuget.outputs.VERSION }} - release_name: Release ${{ steps.publish_nuget.outputs.VERSION }} - draft: false - prerelease: false - - name: Upload Release Asset - if: ${{ success() && matrix.os == 'windows-latest' && steps.create_release.outputs.upload_url != '' && steps.create_release.outputs.upload_url != null }} - id: upload-release-asset - uses: actions/upload-release-asset@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - upload_url: ${{ steps.create_release.outputs.upload_url }} - asset_path: ${{ steps.publish_nuget.outputs.PACKAGE_PATH }} - asset_name: ${{ steps.publish_nuget.outputs.PACKAGE_NAME }} - asset_content_type: application/zip - - name: refresh coverage badge - uses: fjogeleit/http-request-action@master - with: - url: https://camo.githubusercontent.com/12c4fcb3b21fbb2a725fc61449fb1b91e972c4c8a2baaf5904936d8e334bdbe8/68747470733a2f2f636f766572616c6c732e696f2f7265706f732f6769746875622f62336230302f63736c792f62616467652e7376673f6272616e63683d64657626736572766963653d676974687562 - method: PURGE - - name: refresh nuget badge - uses: fjogeleit/http-request-action@master - with: - url: https://camo.githubusercontent.com/b0c8ccfcb3256380ae37c312e789e6282143295fa495361ea5c2dbe2169bff8c/68747470733a2f2f696d672e736869656c64732e696f2f6e756765742f762f736c792e737667 - method: PURGE diff --git a/doc/features/ambiguous_grammars/description.md b/doc/features/ambiguous_grammars/description.md new file mode 100644 index 00000000..4bc54d26 --- /dev/null +++ b/doc/features/ambiguous_grammars/description.md @@ -0,0 +1,18 @@ +# Manage ambiguous grammars + +## goal + +For now CSLY only manage unambiguous grammars. When parsing a n ambiguous grammar it resolves ambiguity returning the first derivation. +The goal is to return a forest instead of a tree. + +## Resolving ambiguity + +Resolving ambiguity (if ever possible) would be up to user. either at : +- visiting tree time (throwing an `AmbiguityExcpetion` +- some following step , like a semantic or typing pass + +## grammaire ambiguë de référence + +``` +S> ::= 'a' 'a' | 'a' 'a' | 'a' +``` \ No newline at end of file diff --git a/doc/features/ambiguous_grammars/plan.md b/doc/features/ambiguous_grammars/plan.md new file mode 100644 index 00000000..34709dfa --- /dev/null +++ b/doc/features/ambiguous_grammars/plan.md @@ -0,0 +1,893 @@ +# Plan d'implémentation : Support des grammaires ambiguës dans CSLY + +## Contexte + +Actuellement, CSLY ne gère que des grammaires non ambiguës. Lorsqu'une grammaire ambiguë est parsée, le parseur résout l'ambiguïté en retournant la première dérivation (celle qui consomme le plus de tokens). L'objectif est de permettre au parseur de retourner une **forêt d'analyse** (parse forest) au lieu d'un arbre unique, laissant à l'utilisateur la responsabilité de résoudre les ambiguïtés. + +## Architecture actuelle + +### Structures de données principales + +1. **`ParseResult`** : Résultat final du parsing + - Contient un seul `ISyntaxNode SyntaxTree` + - Contient un résultat `OUT Result` (après visite du tree) + - Gère les erreurs via `List Errors` + +2. **`SyntaxParseResult`** : Résultat intermédiaire du parsing syntaxique + - Contient un seul `ISyntaxNode Root` + - Position de fin, erreurs, etc. + +3. **`ISyntaxNode`** : Interface pour les nœuds d'arbre syntaxique + - Implémenté par `SyntaxLeaf` (terminaux) + - Implémenté par `SyntaxNode` (non-terminaux) + +4. **`RecursiveDescentSyntaxParser`** : Parseur récursif descendant + - Méthode `ParseNonTerminal()` : **POINT CLÉ** - collecte plusieurs résultats de règles alternatives dans `rulesResults`, mais ne retourne que le meilleur (`maxOk` ou `maxKo`) + +### Point d'ambiguïté critique + +Dans `RecursiveDescentSyntaxParser.NonTerminal.cs`, ligne ~37-120 : +```csharp +var rulesResults = new List>(); +// ... collecte tous les résultats de règles alternatives ... +// PUIS sélectionne seulement le meilleur : +if (hasOk) { max = maxOk; } else { max = maxKo; } +result.Root = max.Root; // ← UN SEUL arbre retourné +``` + +**C'est ici que l'ambiguïté est résolue arbitrairement.** + +--- + +## Plan d'implémentation + +### Phase 1 : Modification des structures de données (Fondations) + +#### 1.1 Créer une nouvelle structure `ParseForest` + +**Objectif** : Représenter une forêt d'arbres syntaxiques avec méta-informations sur les ambiguïtés. + +**Fichier** : `src/sly/parser/syntax/tree/ParseForest.cs` (nouveau) + +```csharp +namespace sly.parser.syntax.tree +{ + /// + /// Représente une forêt d'arbres syntaxiques issus d'une grammaire ambiguë + /// + public class ParseForest where IN : struct, Enum + { + /// + /// Liste des arbres syntaxiques valides (alternatives) + /// + public List> Trees { get; set; } + + /// + /// Indique si la forêt contient des ambiguïtés + /// + public bool IsAmbiguous => Trees != null && Trees.Count > 1; + + /// + /// Nombre d'arbres dans la forêt + /// + public int Count => Trees?.Count ?? 0; + + /// + /// Arbre principal (premier de la liste pour compatibilité) + /// + public ISyntaxNode MainTree => Trees?.FirstOrDefault(); + + /// + /// Informations sur les points d'ambiguïté dans la forêt + /// + public List> Ambiguities { get; set; } + } + + /// + /// Informations sur un point d'ambiguïté spécifique + /// + public class AmbiguityInfo where IN : struct, Enum + { + public string NonTerminalName { get; set; } + public int Position { get; set; } + public List> AlternativeRules { get; set; } + public int AlternativeCount { get; set; } + } +} +``` + +#### 1.2 Étendre `SyntaxParseResult` + +**Objectif** : Permettre de stocker plusieurs arbres alternatifs au niveau du parsing syntaxique. + +**Fichier** : `src/sly/parser/parser/SyntaxParseResult.cs` + +```csharp +public class SyntaxParseResult where IN : struct, Enum +{ + // ... propriétés existantes ... + + /// + /// Liste des racines alternatives (pour grammaires ambiguës) + /// Si null ou 1 élément, pas d'ambiguïté + /// + public List> AlternativeRoots { get; set; } + + /// + /// Indique si ce résultat contient des alternatives ambiguës + /// + public bool HasAmbiguity => AlternativeRoots != null && AlternativeRoots.Count > 1; + + /// + /// Compatibilité : Root pointe vers la première alternative + /// + public ISyntaxNode Root + { + get => AlternativeRoots?.FirstOrDefault(); + set + { + if (AlternativeRoots == null) + AlternativeRoots = new List>(); + if (AlternativeRoots.Count == 0) + AlternativeRoots.Add(value); + else + AlternativeRoots[0] = value; + } + } +} +``` + +#### 1.3 Étendre `ParseResult` + +**Objectif** : Exposer la forêt d'analyse dans le résultat final. + +**Fichier** : `src/sly/parser/parser/ParseResult.cs` + +```csharp +public class ParseResult where IN : struct, Enum +{ + // ... propriétés existantes ... + + /// + /// Forêt d'arbres syntaxiques (pour grammaires ambiguës) + /// + public ParseForest Forest { get; set; } + + /// + /// Compatibilité : SyntaxTree pointe vers l'arbre principal de la forêt + /// + public ISyntaxNode SyntaxTree + { + get => Forest?.MainTree; + set + { + if (Forest == null) + Forest = new ParseForest { Trees = new List>() }; + if (Forest.Trees.Count == 0) + Forest.Trees.Add(value); + else + Forest.Trees[0] = value; + } + } + + /// + /// Indique si le parsing a produit des ambiguïtés + /// + public bool IsAmbiguous => Forest?.IsAmbiguous ?? false; +} +``` + +--- + +### Phase 2 : Modification du parseur (Capture des alternatives) + +#### 2.1 Ajouter une option de configuration + +**Objectif** : Permettre d'activer/désactiver la détection d'ambiguïtés. + +**Fichier** : `src/sly/parser/ParserConfiguration.cs` + +```csharp +public class ParserConfiguration where IN : struct, Enum +{ + // ... propriétés existantes ... + + /// + /// Si true, le parseur retourne toutes les dérivations possibles en cas d'ambiguïté + /// Si false, retourne la première dérivation (comportement actuel) + /// + public bool CaptureAmbiguities { get; set; } = false; + + /// + /// Stratégie de résolution d'ambiguïté par défaut + /// + public AmbiguityResolutionStrategy AmbiguityStrategy { get; set; } + = AmbiguityResolutionStrategy.First; +} + +public enum AmbiguityResolutionStrategy +{ + /// Retourne la première dérivation (comportement actuel) + First, + /// Retourne toutes les dérivations + All, + /// Lance une exception si ambiguïté détectée + ThrowException, + /// Retourne la dérivation la plus longue + Longest +} +``` + +#### 2.2 Modifier `ParseNonTerminal()` pour capturer les ambiguïtés (Parseur BNF) + +**Objectif** : Au lieu de sélectionner un seul arbre, conserver tous les arbres valides. + +**Fichier** : `src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.NonTerminal.cs` + +```csharp +public SyntaxParseResult ParseNonTerminal(Token[] tokens, string nonTerminalName, + int currentPosition, SyntaxParsingContext parsingContext) +{ + // ... code existant jusqu'à la collecte de rulesResults ... + + var rulesResults = new List>(); + // ... collecte des résultats ... + + // NOUVEAU CODE : Gestion des ambiguïtés + var result = new SyntaxParseResult(); + + if (Configuration.CaptureAmbiguities) + { + // Filtrer les résultats OK avec la même longueur maximale + var okResults = rulesResults.Where(r => r.IsOk).ToList(); + + if (okResults.Any()) + { + var maxLength = okResults.Max(r => r.EndingPosition); + var maxResults = okResults.Where(r => r.EndingPosition == maxLength).ToList(); + + if (maxResults.Count > 1) + { + // AMBIGUÏTÉ DÉTECTÉE + result.AlternativeRoots = maxResults.Select(r => r.Root).ToList(); + result.EndingPosition = maxLength; + result.IsError = false; + + // Enregistrer l'ambiguïté + if (result.Ambiguities == null) + result.Ambiguities = new List>(); + result.Ambiguities.Add(new AmbiguityInfo + { + NonTerminalName = nonTerminalName, + Position = currentPosition, + AlternativeCount = maxResults.Count + }); + } + else + { + // Pas d'ambiguïté, un seul résultat optimal + result.Root = maxResults[0].Root; + result.EndingPosition = maxResults[0].EndingPosition; + result.IsError = false; + } + } + else + { + // Tous les résultats sont en erreur + max = maxKo; + result.Root = max.Root; + result.EndingPosition = max.EndingPosition; + result.IsError = true; + } + } + else + { + // COMPORTEMENT ACTUEL (rétrocompatibilité) + if (hasOk) { max = maxOk; } else { max = maxKo; } + result.Root = max.Root; + result.EndingPosition = max.EndingPosition; + result.IsError = max.IsError; + } + + // ... reste du code existant ... +} +``` + +#### 2.2bis Modifier `ParseChoice()` pour capturer les ambiguïtés (Parseur EBNF) + +**⚠️ POINT CRITIQUE IDENTIFIÉ** : Le parseur EBNF a sa propre gestion des choix via `ParseChoice()` qui doit également être modifiée. + +**Objectif** : Dans les clauses EBNF avec alternatives (`A | B | C`), capturer toutes les alternatives valides au lieu de retourner la première qui réussit. + +**Fichier** : `src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Choice.cs` + +**Code actuel** (ligne 37-60) : +```csharp +foreach (var alternate in clause.Choices) +{ + // ... parse chaque alternative ... + result = alternateResults.Last(); + if (result.IsOk) + { + // STOP dès la première alternative qui réussit + parsingContext.Memoize(clause, position, result); + return result; // ← UN SEUL résultat retourné + } +} +``` + +**Nouveau code** : +```csharp +public SyntaxParseResult ParseChoice(Token[] tokens, ChoiceClause clause, + int position, SyntaxParsingContext parsingContext) +{ + // ... code memoization existant ... + + List> alternateResults = new List>(); + + // Collecter TOUS les résultats (pas de early return) + foreach (var alternate in clause.Choices) + { + switch (alternate) + { + case TerminalClause terminalAlternate: + var rterm = ParseTerminal(tokens, terminalAlternate, currentPosition, parsingContext); + alternateResults.Add(rterm); + break; + case NonTerminalClause nonTerminalAlternate: + var rnonterm = ParseNonTerminal(tokens, nonTerminalAlternate, currentPosition, parsingContext); + alternateResults.Add(rnonterm); + break; + } + + // NOUVEAU : Ne pas s'arrêter à la première alternative qui réussit si CaptureAmbiguities + if (!Configuration.CaptureAmbiguities && alternateResults.Last().IsOk) + { + // Comportement actuel : retourner la première alternative OK + result = alternateResults.Last(); + if (clause.IsTerminalChoice && clause.IsDiscarded && result.Root is SyntaxLeaf leaf) + { + var discardedToken = new SyntaxLeaf(leaf.Token, true); + result.Root = discardedToken; + } + parsingContext.Memoize(clause, position, result); + return result; + } + } + + // NOUVEAU : Gestion des ambiguïtés + var result = new SyntaxParseResult(); + + if (Configuration.CaptureAmbiguities) + { + var okResults = alternateResults.Where(r => r.IsOk).ToList(); + + if (okResults.Any()) + { + // Toutes les alternatives réussies à la même position finale + var maxLength = okResults.Max(r => r.EndingPosition); + var maxResults = okResults.Where(r => r.EndingPosition == maxLength).ToList(); + + if (maxResults.Count > 1) + { + // AMBIGUÏTÉ dans le choix EBNF + result.AlternativeRoots = maxResults.Select(r => r.Root).ToList(); + result.EndingPosition = maxLength; + result.IsError = false; + + // Enregistrer l'ambiguïté + if (result.Ambiguities == null) + result.Ambiguities = new List>(); + result.Ambiguities.Add(new AmbiguityInfo + { + NonTerminalName = $"Choice[{string.Join("|", clause.Choices.Select(c => c.ToString()))}]", + Position = position, + AlternativeCount = maxResults.Count + }); + } + else + { + result = maxResults[0]; + } + + // Appliquer les transformations (discard, etc.) + if (clause.IsTerminalChoice && clause.IsDiscarded && result.Root is SyntaxLeaf leaf) + { + var discardedToken = new SyntaxLeaf(leaf.Token, true); + result.Root = discardedToken; + } + } + else + { + // Toutes les alternatives ont échoué + result = HandleAllChoicesFailed(clause, tokens, currentPosition, alternateResults); + } + } + else + { + // Si on arrive ici, toutes les alternatives ont échoué (comportement actuel) + result = HandleAllChoicesFailed(clause, tokens, currentPosition, alternateResults); + } + + parsingContext.Memoize(clause, position, result); + return result; +} + +private SyntaxParseResult HandleAllChoicesFailed( + ChoiceClause clause, + Token[] tokens, + int currentPosition, + List> alternateResults) +{ + var result = new SyntaxParseResult + { + IsError = true, + EndingPosition = currentPosition + }; + + if (clause.IsTerminalChoice) + { + var terminalAlternates = clause.Choices.Cast>(); + var expected = terminalAlternates.Select(x => x.ExpectedToken).ToList(); + result.AddError(new UnexpectedTokenSyntaxError(tokens[currentPosition], LexemeLabels, I18n, + expected.ToArray())); + } + else + { + var greaterPosition = alternateResults.Select(x => x.EndingPosition).Max(); + var errors = alternateResults.Where(x => x.EndingPosition == greaterPosition) + .SelectMany(x => x.GetErrors()).ToList(); + result.AddErrors(errors); + } + + return result; +} +``` + +#### 2.3 Propager les ambiguïtés dans la chaîne de parsing + +**Objectif** : S'assurer que les ambiguïtés remontent jusqu'au `ParseResult` final. + +**Fichier** : `src/sly/parser/parser/Parser.cs` + +Dans la méthode `ParseWithContext()` : + +```csharp +public ParseResult ParseWithContext(IList> tokens, object parsingContext = null, + string startingNonTerminal = null) +{ + var result = new ParseResult(); + var syntaxResult = SyntaxParser.Parse(tokens.ToArray(), startingNonTerminal); + + if (!syntaxResult.IsError && syntaxResult.Root != null) + { + // NOUVEAU : Gérer la forêt d'arbres + if (syntaxResult.HasAmbiguity) + { + result.Forest = new ParseForest + { + Trees = syntaxResult.AlternativeRoots, + Ambiguities = syntaxResult.Ambiguities + }; + + // Visiter tous les arbres ou appliquer la stratégie + switch (Configuration.AmbiguityStrategy) + { + case AmbiguityResolutionStrategy.First: + result.Result = Visitor.VisitSyntaxTree(result.Forest.MainTree, parsingContext); + break; + + case AmbiguityResolutionStrategy.ThrowException: + throw new AmbiguousGrammarException(result.Forest); + + case AmbiguityResolutionStrategy.All: + // Ne pas visiter automatiquement, laisser l'utilisateur le faire + result.Result = default(OUT); + break; + } + } + else + { + // Pas d'ambiguïté, comportement normal + result.SyntaxTree = syntaxResult.Root; + result.Result = Visitor.VisitSyntaxTree(syntaxResult.Root, parsingContext); + } + + result.IsError = false; + } + else + { + // ... gestion des erreurs existante ... + } + + return result; +} +``` + +#### 2.4 Propager les ambiguïtés dans les structures EBNF imbriquées + +**Objectif** : S'assurer que les ambiguïtés détectées dans `ParseChoice()` remontent correctement à travers les structures EBNF (options, répétitions, etc.). + +**Fichiers concernés** : +- `src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Option.cs` +- `src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Many.cs` +- `src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.cs` (méthode `Parse()` principale) + +**Modifications nécessaires** : + +1. **Dans `ParseOption()`** : Si l'option contient un choix ambigu, propager les alternatives +```csharp +case ChoiceClause choice: + innerResult = ParseChoice(tokens, choice, currentPosition, parsingContext); + + // NOUVEAU : Propager les ambiguïtés + if (innerResult.HasAmbiguity) + { + // L'option contient plusieurs alternatives ambiguës + // Créer des OptionSyntaxNode pour chaque alternative + result.AlternativeRoots = innerResult.AlternativeRoots + .Select(altRoot => new OptionSyntaxNode( + rule.NonTerminalName, + new List> { altRoot }, + rule.GetVisitorMethod()) as ISyntaxNode) + .ToList(); + result.EndingPosition = innerResult.EndingPosition; + result.Ambiguities = innerResult.Ambiguities; + } + break; +``` + +2. **Dans `Parse()` principale (EBNF)** : Propager les ambiguïtés des clauses vers le nœud parent +```csharp +case ChoiceClause choice: +{ + var choiceResult = ParseChoice(tokens, choice, currentPosition, parsingContext); + currentPosition = choiceResult.EndingPosition; + + // NOUVEAU : Si le choix est ambigu, propager à ce niveau + if (choiceResult.HasAmbiguity) + { + // On doit créer plusieurs arbres alternatifs pour cette règle + // en variant seulement l'enfant correspondant au choix ambigu + // (Plus complexe - voir implémentation détaillée ci-dessous) + } + + if (choiceResult.IsError && choiceResult.GetErrors() != null && choiceResult.GetErrors().Any()) + { + errors.AddRange(choiceResult.GetErrors()); + } + isError = choiceResult.IsError; + children.Add(choiceResult.Root); + break; +} +``` + +**Note importante** : La propagation des ambiguïtés à travers les clauses EBNF imbriquées est complexe car une règle peut contenir plusieurs clauses, et si l'une d'elles est ambiguë, il faut créer plusieurs versions du nœud parent avec les mêmes enfants sauf celui qui varie. + +**Solution simplifiée (Phase 1)** : +- Détecter et capturer les ambiguïtés au niveau des choix (`ParseChoice`) et des non-terminaux (`ParseNonTerminal`) +- Les propager directement au résultat final +- Laisser la combinaison d'ambiguïtés imbriquées pour une phase ultérieure (explosion combinatoire potentielle) + +**Solution complète (Phase future)** : +- Implémenter une logique de combinaison cartésienne pour gérer les ambiguïtés imbriquées +- Exemple : si une règle `A -> B C D` où B et D sont ambigus (2 alternatives chacun), créer 4 arbres (2×2) + +--- + +### Phase 3 : API utilisateur pour la résolution d'ambiguïtés + +#### 3.1 Créer une exception `AmbiguousGrammarException` + +**Fichier** : `src/sly/parser/exceptions/AmbiguousGrammarException.cs` (nouveau) + +```csharp +namespace sly.parser.exceptions +{ + public class AmbiguousGrammarException : Exception where IN : struct, Enum + { + public ParseForest Forest { get; } + + public AmbiguousGrammarException(ParseForest forest) + : base($"Ambiguous grammar detected: {forest.Count} alternative parse trees found") + { + Forest = forest; + } + + public override string Message + { + get + { + var sb = new StringBuilder(base.Message); + sb.AppendLine(); + sb.AppendLine("Ambiguity points:"); + foreach (var amb in Forest.Ambiguities) + { + sb.AppendLine($" - NonTerminal '{amb.NonTerminalName}' at position {amb.Position}: {amb.AlternativeCount} alternatives"); + } + return sb.ToString(); + } + } + } +} +``` + +#### 3.2 Créer des méthodes utilitaires pour la résolution manuelle + +**Fichier** : `src/sly/parser/ParseResult.cs` (extension) + +```csharp +public class ParseResult where IN : struct, Enum +{ + // ... propriétés existantes ... + + /// + /// Visite tous les arbres de la forêt et retourne les résultats + /// + public List VisitAllTrees(SyntaxTreeVisitor visitor, object context = null) + { + if (!IsAmbiguous) + return new List { Result }; + + return Forest.Trees.Select(tree => visitor.VisitSyntaxTree(tree, context)).ToList(); + } + + /// + /// Permet à l'utilisateur de sélectionner un arbre spécifique + /// + public OUT SelectTree(int index, SyntaxTreeVisitor visitor, object context = null) + { + if (Forest == null || index >= Forest.Count) + throw new ArgumentOutOfRangeException(nameof(index)); + + return visitor.VisitSyntaxTree(Forest.Trees[index], context); + } + + /// + /// Résoudre l'ambiguïté via un sélecteur personnalisé + /// + public OUT ResolveAmbiguity(Func>, ISyntaxNode> selector, + SyntaxTreeVisitor visitor, object context = null) + { + if (!IsAmbiguous) + return Result; + + var selectedTree = selector(Forest.Trees); + return visitor.VisitSyntaxTree(selectedTree, context); + } +} +``` + +#### 3.3 Créer une interface pour les visiteurs désambiguïsants + +**Fichier** : `src/sly/parser/generator/visitor/IAmbiguityResolver.cs` (nouveau) + +```csharp +namespace sly.parser.generator.visitor +{ + /// + /// Interface pour implémenter une logique de résolution d'ambiguïté personnalisée + /// + public interface IAmbiguityResolver where IN : struct, Enum + { + /// + /// Sélectionne l'arbre syntaxique à utiliser parmi plusieurs alternatives + /// + /// Liste des arbres alternatifs + /// Informations sur le point d'ambiguïté + /// L'arbre sélectionné + ISyntaxNode Resolve(List> alternatives, + AmbiguityInfo ambiguityInfo); + } +} +``` + +--- + +### Phase 4 : Optimisation (optionnelle) - Shared Packed Parse Forest (SPPF) + +**Objectif** : Réduire la mémoire en partageant les sous-arbres communs entre les dérivations alternatives. + +**Note** : Cette phase est optionnelle pour la première version. Elle peut être implémentée plus tard si les performances deviennent un problème. + +#### 4.1 Créer une structure SPPF + +**Fichier** : `src/sly/parser/syntax/tree/SPPF.cs` (nouveau) + +```csharp +namespace sly.parser.syntax.tree +{ + /// + /// Nœud d'une Shared Packed Parse Forest + /// + public interface ISPPFNode where IN : struct, Enum + { + int StartPosition { get; } + int EndPosition { get; } + } + + /// + /// Nœud symbole : représente un non-terminal ou terminal + /// + public class SPPFSymbolNode : ISPPFNode where IN : struct, Enum + { + public string Symbol { get; set; } + public int StartPosition { get; set; } + public int EndPosition { get; set; } + public List> Alternatives { get; set; } + } + + /// + /// Nœud packed : représente une alternative de dérivation + /// + public class SPPFPackedNode where IN : struct, Enum + { + public Rule Rule { get; set; } + public List> Children { get; set; } + } +} +``` + +**Cette phase nécessite une refonte plus profonde et peut être reportée à une version ultérieure.** + +--- + +### Phase 5 : Tests et documentation + +#### 5.1 Tests unitaires + +**Fichier** : `tests/ParserTests/AmbiguousGrammarTests.cs` (nouveau) + +Créer des tests pour : + +**Tests BNF** : +- Grammaire arithmétique ambiguë classique : `E -> E + E | E * E | INT` +- Grammaire if-then-else ambiguë (dangling else) +- Multiple règles alternatives pour un même non-terminal + +**Tests EBNF** : +- Choix EBNF ambigu : `A -> (B | C | D)` où B, C et D peuvent tous matcher la même entrée +- Options ambiguës : `A -> B [C | D]` où C et D sont ambigus +- Répétitions avec alternatives ambiguës : `A -> (B | C)*` où B et C peuvent matcher les mêmes tokens + +**Tests de stratégie** : +- Vérifier que `CaptureAmbiguities = false` conserve le comportement actuel +- Vérifier que `CaptureAmbiguities = true` retourne tous les arbres +- Tester les stratégies de résolution (`First`, `All`, `ThrowException`, `Longest`) + +**Tests de propagation** : +- Ambiguïtés imbriquées : choix dans une option dans une répétition +- Combinaisons multiples d'ambiguïtés dans la même règle + +#### 5.2 Exemples d'utilisation + +**Fichier** : `src/samples/AmbiguousGrammarExample/` (nouveau dossier) + +Créer un exemple complet montrant : +1. Définition d'une grammaire ambiguë +2. Activation de `CaptureAmbiguities` +3. Parcours de la forêt d'arbres +4. Résolution manuelle via un `IAmbiguityResolver` + +#### 5.3 Documentation + +**Fichier** : `doc/features/ambiguous_grammars/usage.md` (nouveau) + +Documenter : +- Comment activer la détection d'ambiguïtés +- Les différentes stratégies disponibles +- Comment implémenter un résolveur personnalisé +- Exemples de grammaires ambiguës courantes +- Considérations de performance + +--- + +## Plan de migration et rétrocompatibilité + +### Compatibilité ascendante garantie + +1. **Par défaut, aucun changement** : `CaptureAmbiguities = false` par défaut + - Le comportement actuel est préservé + - Les utilisateurs existants ne sont pas impactés + +2. **API compatible** : + - `ParseResult.SyntaxTree` continue de fonctionner (pointe vers le premier arbre) + - `ParseResult.Result` continue de fonctionner (visite le premier arbre) + +3. **Activation opt-in** : + ```csharp + var builder = new ParserBuilder(); + var buildResult = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "root"); + var parser = buildResult.Result; + + // NOUVEAU : Activer la détection d'ambiguïtés + parser.Configuration.CaptureAmbiguities = true; + parser.Configuration.AmbiguityStrategy = AmbiguityResolutionStrategy.ThrowException; + + var result = parser.Parse(source); + if (result.IsAmbiguous) + { + // Traiter l'ambiguïté + } + ``` + +--- + +## Ordre d'implémentation recommandé + +1. **Semaine 1** : Phase 1 (structures de données) + - Créer `ParseForest`, `AmbiguityInfo` + - Modifier `SyntaxParseResult` et `ParseResult` + - Tests de base sur les nouvelles structures + +2. **Semaine 2** : Phase 2.1 et 2.2 (configuration et capture - BNF) + - Ajouter `CaptureAmbiguities` et `AmbiguityResolutionStrategy` + - Modifier `ParseNonTerminal()` (BNF) pour capturer les alternatives + - Tests avec grammaire BNF simple ambiguë + +3. **Semaine 3** : Phase 2.2bis et 2.4 (capture et propagation - EBNF) + - Modifier `ParseChoice()` (EBNF) pour capturer les alternatives EBNF + - Implémenter la propagation des ambiguïtés dans `ParseOption()` et `Parse()` principal + - Tests avec grammaire EBNF ambiguë (choix multiples) + +4. **Semaine 4** : Phase 2.3 et 3.1-3.2 (propagation finale et API) + - Propager les ambiguïtés jusqu'au `ParseResult` + - Créer `AmbiguousGrammarException` + - Ajouter méthodes utilitaires (`VisitAllTrees`, etc.) + +5. **Semaine 5** : Phase 3.3 et 5 (résolveurs et tests) + - Interface `IAmbiguityResolver` + - Tests exhaustifs avec grammaires classiques + - Exemples et documentation + +5. **Version future** : Phase 4 (SPPF) + - Optimisation mémoire si nécessaire + +--- + +## Points d'attention + +### Performance +- **Impact mémoire** : Stocker plusieurs arbres peut consommer beaucoup de mémoire pour des grammaires très ambiguës +- **Solution** : Implémenter SPPF dans une phase ultérieure si nécessaire +- **Compromis** : Limiter le nombre d'alternatives capturées (configurable) + +### Parseurs concernés +- **LL Recursive Descent (BNF)** : ✅ Couvert par ce plan (Phase 2.2 - `ParseNonTerminal()`) +- **EBNF Recursive Descent** : ✅ Couvert par ce plan (Phase 2.2bis - `ParseChoice()` + Phase 2.4 - propagation) + - **Point critique** : `ParseChoice()` gère les alternatives EBNF (`A | B | C`) + - **Complexité additionnelle** : Propagation des ambiguïtés à travers les structures imbriquées (options, répétitions) +- **LR/LALR** : ⚠️ Non couvert, architecture différente (à évaluer séparément) + +### Edge cases +- Grammaire infiniment ambiguë : ajouter un timeout ou limite d'alternatives +- Ambiguïtés imbriquées : tester la propagation correcte +- Performances sur grammaires très ambiguës : benchmarker + +--- + +## Validation du plan + +### Critères de succès + +1. ✅ Rétrocompatibilité totale (comportement par défaut inchangé) +2. ✅ Capacité de détecter et capturer les ambiguïtés +3. ✅ API claire et flexible pour la résolution +4. ✅ Documentation complète avec exemples +5. ✅ Tests exhaustifs couvrant les cas courants + +### Points de validation + +- [ ] Tests unitaires passent (anciens + nouveaux) +- [ ] Exemple de grammaire ambiguë fonctionne +- [ ] Pas de régression de performance sur grammaires non ambiguës +- [ ] Documentation à jour +- [ ] Review de code complète + +--- + +## Conclusion + +Ce plan permet d'introduire progressivement le support des grammaires ambiguës dans CSLY tout en préservant la rétrocompatibilité. L'implémentation est structurée en phases incrémentales, permettant de valider chaque étape avant de passer à la suivante. + +**Prochaine étape** : Validation du plan avec les mainteneurs et début de l'implémentation de la Phase 1. diff --git a/src/benchCurrent/json/EbnfJsonGenericParser.cs b/src/benchCurrent/json/EbnfJsonGenericParser.cs index 51e30cd0..62f0dad9 100644 --- a/src/benchCurrent/json/EbnfJsonGenericParser.cs +++ b/src/benchCurrent/json/EbnfJsonGenericParser.cs @@ -156,6 +156,7 @@ public JSon property(Token key, JSon value) [BroadenTokenWindow] [ParserRoot("root")] + [FirstDerivation] public class EbnfJsonGenericParserStringNotEscaped { #region root diff --git a/src/samples/IndentedWhile/parser/IndentedWhileParserGeneric.cs b/src/samples/IndentedWhile/parser/IndentedWhileParserGeneric.cs index b5580dae..c2eb62a1 100644 --- a/src/samples/IndentedWhile/parser/IndentedWhileParserGeneric.cs +++ b/src/samples/IndentedWhile/parser/IndentedWhileParserGeneric.cs @@ -9,6 +9,7 @@ namespace csly.indentedWhileLang.parser { [UseMemoization] [AutoCloseIndentations] + [FirstDerivation] [ParserRoot("program")] public class IndentedWhileParserGeneric { diff --git a/src/samples/ParserExample/ExprParser.cs b/src/samples/ParserExample/ExprParser.cs index cad404f2..a978dad4 100644 --- a/src/samples/ParserExample/ExprParser.cs +++ b/src/samples/ParserExample/ExprParser.cs @@ -4,6 +4,7 @@ namespace ParserExample; [ParserRoot("root")] +[FirstDerivation] public class ExprParser { [Production("root : ExprParser_expressions")] diff --git a/src/samples/ParserExample/GettingStarted.cs b/src/samples/ParserExample/GettingStarted.cs new file mode 100644 index 00000000..492703dd --- /dev/null +++ b/src/samples/ParserExample/GettingStarted.cs @@ -0,0 +1,44 @@ +using sly.lexer; +using sly.parser.generator; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace ParserExample +{ + internal enum GettingStartedLexer + { + [Lexeme("[0-9]+")] + INT = 1, + + [Lexeme("\\+")] + PLUS = 2, + + [Lexeme("[ \\t]+", isSkippable: true)] // the lexeme is marked isSkippable : it will not be sent to the parser and simply discarded. + WS = 3 + } + + internal class GettingStartedParser + { + + [Production("expression: INT")] + public int intExpr(Token intToken) + { + return intToken.IntValue; + } + + [Production("expression: term PLUS expression")] + public int Expression(int left, Token operatorToken, int right) + { + return left + right; + } + + [Production("term: INT")] + public int Expression(Token intToken) + { + return intToken.IntValue; + } + } +} diff --git a/src/samples/ParserExample/MixingAlphaNumDashIdsAndMinus.cs b/src/samples/ParserExample/MixingAlphaNumDashIdsAndMinus.cs new file mode 100644 index 00000000..a64feee8 --- /dev/null +++ b/src/samples/ParserExample/MixingAlphaNumDashIdsAndMinus.cs @@ -0,0 +1,6 @@ +namespace ParserExample; + +public enum MixingAlphaNumDashIdsAndMinus +{ + +} \ No newline at end of file diff --git a/src/samples/ParserExample/Program.cs b/src/samples/ParserExample/Program.cs index 0fe7b7d0..48add5c1 100644 --- a/src/samples/ParserExample/Program.cs +++ b/src/samples/ParserExample/Program.cs @@ -42,33 +42,30 @@ using ExpressionContext = postProcessedLexerParser.expressionModel.ExpressionContext; using ExpressionToken = simpleExpressionParser.ExpressionToken; using IfThenElse = indented.IfThenElse; +using ParserTests.ambiguous; namespace ParserExample { public enum ManySugar { - [Sugar("<")] - OPEN, - [Sugar(" args) return "_"; } - private static void testIssue516() + private static void testIssue516() { ParserBuilder builder = new ParserBuilder(); var xmlparser = new MinimalXmlParser(); @@ -155,7 +153,7 @@ inner inner content parsed.Errors.ForEach(Console.WriteLine); } - + var jBuilder = new ParserBuilder(); var jsonParser = new EbnfJsonGenericParser(); var jr = jBuilder.BuildParser(jsonParser, ParserType.EBNF_LL_RECURSIVE_DESCENT, "root"); @@ -167,19 +165,19 @@ inner inner content } } - + private static void TestIssue507() { var tests = new EBNFTests(); //tests.TestIssue507TransitiveEmptyStarter(); tests.TestIssue507MoreTransitiveEmptyStarter(); } - + private static void BenchSimpleExpression() { GenericSimpleExpressionParser p = new GenericSimpleExpressionParser(); var builder = new ParserBuilder(); - + var Parser = builder.BuildParser(p, ParserType.EBNF_LL_RECURSIVE_DESCENT, "root"); if (Parser.IsOk) { @@ -189,9 +187,9 @@ private static void BenchSimpleExpression() if (r.IsOk) { Console.WriteLine(r.Result); - } + } } - + } } @@ -223,7 +221,7 @@ private static void TestFactorial() var compiler = new WhileCompiler(); var code = compiler.TranspileToCSharp(program); - var f = compiler.CompileToFunction(program,false); + var f = compiler.CompileToFunction(program, false); int r = f(); if (r != 3628800) { @@ -236,7 +234,7 @@ private static void TestFactorial() } } } - + private static void TestIndentedFactorial() { var whileParser = new IndentedWhileParserGeneric(); @@ -252,9 +250,9 @@ private static void TestIndentedFactorial() return; } - - var program = -@"# indented factorial + + var program = + @"# indented factorial r:=1 i:=1 while i < 11 do @@ -284,13 +282,13 @@ private static void TestIndentedFactorial() return; } - + var interpreter = new Interpreter(); var context = interpreter.Interprete(result.Result); var compiler = new IndentedWhileCompiler(); var code = compiler.TranspileToCSharp(program); - var f = compiler.CompileToFunction(program,false); + var f = compiler.CompileToFunction(program, false); Console.WriteLine("***********************************"); Console.WriteLine("***********************************"); Console.WriteLine("***********************************"); @@ -495,12 +493,10 @@ public static void TestContextualParser() var buildResult = buildSimpleExpressionParserWithContext(); if (buildResult.IsError) { - buildResult.Errors.ForEach(e => - { - Console.WriteLine(e.Level + " - " + e.Message); - }); + buildResult.Errors.ForEach(e => { Console.WriteLine(e.Level + " - " + e.Message); }); return; } + var parser = buildResult.Result; var res = parser.ParseWithContext("2 + a", new Dictionary { { "a", 2 } }); Console.WriteLine($"result : ok:>{res.IsOk}< value:>{res.Result}<"); @@ -540,47 +536,50 @@ public static void testJSONEscaped(string content = null) { content = File.ReadAllText("test.json"); } - try { - var instance = new EbnfJsonGenericParser(); - var builder = new ParserBuilder(); - var buildResult = builder.BuildParser(instance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "root"); - if (buildResult.IsOk) + try { - Console.WriteLine("parser built."); - var parser = buildResult.Result; - - Console.WriteLine("test.json read."); - for (int i = 0; i < 10; i++) - { + var instance = new EbnfJsonGenericParser(); + var builder = new ParserBuilder(); + var buildResult = builder.BuildParser(instance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "root"); + if (buildResult.IsOk) + { + Console.WriteLine("parser built."); + var parser = buildResult.Result; - var jsonResult = parser.Parse(content); - Console.WriteLine("json parse done."); - if (jsonResult.IsOk) - { - Console.WriteLine("YES !"); - } - else + Console.WriteLine("test.json read."); + for (int i = 0; i < 10; i++) { - Console.WriteLine("Ooh no !"); + + + var jsonResult = parser.Parse(content); + Console.WriteLine("json parse done."); + if (jsonResult.IsOk) + { + Console.WriteLine("YES !"); + } + else + { + Console.WriteLine("Ooh no !"); + } } - } - Console.WriteLine("Done."); - + Console.WriteLine("Done."); + + } + else + { + buildResult.Errors.ForEach(e => Console.WriteLine(e.Message)); + } } - else + catch (Exception e) { - buildResult.Errors.ForEach(e => Console.WriteLine(e.Message)); - } - } - catch(Exception e) { Console.WriteLine($"ERROR {e.Message} : \n {e.StackTrace}"); } } - + public static void testProfileJSONEscaping(string content = null, bool escape = true) { if (escape) @@ -637,7 +636,7 @@ public static void testProfileJSONEscaping(string content = null, bool escape = { content = File.ReadAllText("test.json"); } - + var instanceNot = new EbnfJsonGenericParserStringNotEscaped(); var builderNot = new ParserBuilder(); var buildResultNot = builderNot.BuildParser(instanceNot, ParserType.EBNF_LL_RECURSIVE_DESCENT, "root"); @@ -645,7 +644,7 @@ public static void testProfileJSONEscaping(string content = null, bool escape = { Console.WriteLine("parser built."); var parser = buildResultNot.Result; - + Console.WriteLine("test.json read."); @@ -661,7 +660,7 @@ public static void testProfileJSONEscaping(string content = null, bool escape = } Console.WriteLine("Done. Unescaped."); - + } else { @@ -670,14 +669,14 @@ public static void testProfileJSONEscaping(string content = null, bool escape = } } - + public static void testJSONNotEscaped(string content = null) { if (content == null) { content = File.ReadAllText("test.json"); } - + var instanceNot = new EbnfJsonGenericParserStringNotEscaped(); var builderNot = new ParserBuilder(); var buildResultNot = builderNot.BuildParser(instanceNot, ParserType.EBNF_LL_RECURSIVE_DESCENT, "root"); @@ -685,7 +684,7 @@ public static void testJSONNotEscaped(string content = null) { Console.WriteLine("parser built."); var parser = buildResultNot.Result; - + Console.WriteLine("test.json read."); @@ -701,18 +700,18 @@ public static void testJSONNotEscaped(string content = null) } Console.WriteLine("Done. Unescaped."); - + } else { buildResultNot.Errors.ForEach(e => Console.WriteLine(e.Message)); } } - + public static void testJSONEscapedVsNotEscaped() { var content = File.ReadAllText("test.json"); - + testJSONEscaped(content); testJSONNotEscaped(content); @@ -745,7 +744,7 @@ private static void benchLexer() BenchedLexer.Tokenize(content); } } - + private static void TestChars() { var res = LexerBuilder.BuildLexer(new BuildResult>()); @@ -766,7 +765,7 @@ private static void TestChars() } else { - var errors = string.Join('\n',res.Errors.Select(e => e.Level + " - " + e.Message).ToList()); + var errors = string.Join('\n', res.Errors.Select(e => e.Level + " - " + e.Message).ToList()); Console.WriteLine("error building lexer : "); Console.WriteLine(errors); } @@ -776,11 +775,11 @@ public static void TestRecursion() { var builder = new ParserBuilder(); Console.WriteLine("starting"); - var parserInstance = new RecursiveGrammar(); + var parserInstance = new RecursiveGrammar(); Console.WriteLine("new instance"); - - - var Parser = builder.BuildParser(parserInstance,ParserType.EBNF_LL_RECURSIVE_DESCENT,"clause"); + + + var Parser = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "clause"); Console.WriteLine($"built : {Parser.IsOk}"); if (Parser.IsError) { @@ -793,14 +792,14 @@ public static void TestRecursion() Console.WriteLine("/-----------------------------------"); Console.WriteLine("/---"); Console.WriteLine("/-----------------------------------"); - - builder = new ParserBuilder(); + + builder = new ParserBuilder(); Console.WriteLine("starting"); - var parserInstance2 = new RecursiveGrammar2(); + var parserInstance2 = new RecursiveGrammar2(); Console.WriteLine("new instance"); - Parser = builder.BuildParser(parserInstance2,ParserType.EBNF_LL_RECURSIVE_DESCENT,"clause"); + Parser = builder.BuildParser(parserInstance2, ParserType.EBNF_LL_RECURSIVE_DESCENT, "clause"); Console.WriteLine($"built : {Parser.IsOk}"); if (Parser.IsError) { @@ -822,7 +821,7 @@ public static void TestScript() var parser = parserBuild.Result; string ko1 = "|B|test2(a, b, c=100)|E|"; string ko2 = "|B|plotshape(data, style=shapexcross)|E|"; - + var r = parser.Parse(ko1); var graphviz = new GraphVizEBNFSyntaxTreeVisitor(); var root = graphviz.VisitTree(r.SyntaxTree); @@ -833,7 +832,7 @@ public static void TestScript() { foreach (var e in parserBuild.Errors) { - Console.WriteLine(e.Level+ " - " + e.Message); + Console.WriteLine(e.Level + " - " + e.Message); } } } @@ -846,32 +845,35 @@ public static void TestI18N() Console.WriteLine("***"); var e = I18N.Instance.GetText("en", I18NMessage.UnexpectedEos); Console.WriteLine(e); - var ee = I18N.Instance.GetText("en", I18NMessage.UnexpectedToken,"xxx","SOME_TOKEN"); + var ee = I18N.Instance.GetText("en", I18NMessage.UnexpectedToken, "xxx", "SOME_TOKEN"); Console.WriteLine(ee); - var eee = I18N.Instance.GetText("en", I18NMessage.UnexpectedTokenExpecting,"xxx","SOME_TOKEN","OTHER_TOKEN1, OTHER_TOKEN2, OTHER_TOKEN_3"); + var eee = I18N.Instance.GetText("en", I18NMessage.UnexpectedTokenExpecting, "xxx", "SOME_TOKEN", + "OTHER_TOKEN1, OTHER_TOKEN2, OTHER_TOKEN_3"); Console.WriteLine(eee); Console.WriteLine("****************************************"); Console.WriteLine("***"); Console.WriteLine("*** LOCAL "); Console.WriteLine("***"); - e = I18N.Instance.GetText( CultureInfo.CurrentCulture.TwoLetterISOLanguageName, I18NMessage.UnexpectedEos); + e = I18N.Instance.GetText(CultureInfo.CurrentCulture.TwoLetterISOLanguageName, I18NMessage.UnexpectedEos); Console.WriteLine(e); - ee = I18N.Instance.GetText( CultureInfo.CurrentCulture.TwoLetterISOLanguageName, I18NMessage.UnexpectedToken,"xxx","SOME_TOKEN"); + ee = I18N.Instance.GetText(CultureInfo.CurrentCulture.TwoLetterISOLanguageName, I18NMessage.UnexpectedToken, + "xxx", "SOME_TOKEN"); Console.WriteLine(ee); - eee = I18N.Instance.GetText( CultureInfo.CurrentCulture.TwoLetterISOLanguageName, I18NMessage.UnexpectedTokenExpecting,"xxx","SOME_TOKEN","OTHER_TOKEN1, OTHER_TOKEN2, OTHER_TOKEN_3"); + eee = I18N.Instance.GetText(CultureInfo.CurrentCulture.TwoLetterISOLanguageName, + I18NMessage.UnexpectedTokenExpecting, "xxx", "SOME_TOKEN", "OTHER_TOKEN1, OTHER_TOKEN2, OTHER_TOKEN_3"); Console.WriteLine(eee); ; } private static BuildResult> BuildParserExpression() - { + { var StartingRule = $"{nameof(SimpleExpressionParser)}_expressions"; var parserInstance = new SimpleExpressionParser(); var builder = new ParserBuilder(); return builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, StartingRule); } - + public static void TestAssociativityFactorExpressionParser() { var StartingRule = $"{nameof(SimpleExpressionParser)}_expressions"; @@ -898,48 +900,51 @@ public static void TestManyString() var tok1 = r.Tokens[0]; Assert.Equal(ManyString.STRING, tok1.TokenID); Assert.Equal(expectString1, tok1.Value); - Assert.Equal('"',tok1.StringDelimiter); + Assert.Equal('"', tok1.StringDelimiter); var tok2 = r.Tokens[1]; Assert.Equal(ManyString.STRING, tok2.TokenID); Assert.Equal(expectString2, tok2.Value); - Assert.Equal('\'',tok2.StringDelimiter); + Assert.Equal('\'', tok2.StringDelimiter); } - - private static void AddExponentExtension(DoubleExponent token, LexemeAttribute lexem, GenericLexer lexer) { - if (token == DoubleExponent.DOUBLE_EXP) { - + private static void AddExponentExtension(DoubleExponent token, LexemeAttribute lexem, + GenericLexer lexer) + { + if (token == DoubleExponent.DOUBLE_EXP) + { + + // callback on end_exponent node - NodeCallback callback = (FSMMatch match) => + NodeCallback callback = (FSMMatch match) => { - string[] items = match.Result.Value.Split(new[] {'e', 'E'}); + string[] items = match.Result.Value.Split(new[] { 'e', 'E' }); double radix = 0; - double.TryParse(items[0].Replace(".",","), out radix); + double.TryParse(items[0].Replace(".", ","), out radix); double exponent = 0; - double.TryParse(items[1],out exponent); + double.TryParse(items[1], out exponent); double value = Math.Pow(radix, exponent); match.Result.SpanValue = value.ToString().AsMemory(); - + match.Properties[GenericLexer.DerivedToken] = DoubleExponent.DOUBLE_EXP; return match; }; - + var fsmBuilder = lexer.FSMBuilder; - + fsmBuilder.GoTo(GenericLexer.in_double) // start an in_double node - .Transition(new char[] {'E','e'}) // add a transition on '.' with precondition - .Transition(new char[]{'+','-'}) + .Transition(new char[] { 'E', 'e' }) // add a transition on '.' with precondition + .Transition(new char[] { '+', '-' }) .Mark("start_exponent_val") - .RangeTransitionTo('0','9',"start_exponent_val") // first year digit + .RangeTransitionTo('0', '9', "start_exponent_val") // first year digit .Mark("end_exponent") .End(GenericToken.Extension) // mark as ending node .CallBack(callback); // set the ending callback } } - + private static void TestDoubleExponent() { var lex = LexerBuilder.BuildLexer(AddExponentExtension); @@ -952,11 +957,12 @@ private static void TestDoubleExponent() } } - public static void Test164() { + public static void Test164() + { var Parser = BuildParserExpression(); var result = Parser.Result.Parse("1(1"); if (result.IsError) - { + { foreach (var error in result.Errors) { Console.WriteLine(error.ErrorMessage); @@ -967,7 +973,7 @@ public static void Test164() { public static void Test192() { - var parser = Issue192.CreateBlockParser(); + var parser = Issue192.CreateBlockParser(); var t = parser.Parse("A1 B2 "); if (t.IsOk) { @@ -980,6 +986,7 @@ public static void Test192() t.Errors.ForEach(x => Console.WriteLine(x.ErrorMessage)); ; } + ; } @@ -997,12 +1004,12 @@ public static void TestIndentedLang() quatre = 4 "; -Console.WriteLine("********************"); -Console.WriteLine(source); -Console.WriteLine("********************"); + Console.WriteLine("********************"); + Console.WriteLine(source); + Console.WriteLine("********************"); IndentedTest1(source); - + source = @"if truc == 1 un = 1 deux = 2 @@ -1017,7 +1024,7 @@ public static void TestIndentedLang() Console.WriteLine(source); Console.WriteLine("********************"); - IndentedTest1(source); + IndentedTest1(source); //IndentedTest2(source); } @@ -1074,11 +1081,11 @@ public static void TestChannels() var tokens = lexer.Tokenize(source); var width = tokens.Tokens.GetChannels().Select(x => x.Tokens.Count).Max(); - + List headers = new List(); - - - + + + if (tokens.IsOk) { foreach (var channel in tokens.Tokens.GetChannels()) @@ -1089,14 +1096,15 @@ public static void TestChannels() Console.Write(token == null ? "" : token.ToString()); Console.Write(";"); } + Console.WriteLine(); } } - + ; } - + } private static void IndentedTest2(string source) @@ -1137,10 +1145,10 @@ private static void IndentedTest2(string source) } } - + public static void TestIndentedParserNeverEnding() { - var source =@"if truc == 1 + var source = @"if truc == 1 un = 1 deux = 2 cinq = 5 @@ -1155,15 +1163,15 @@ public static void TestIndentedParserNeverEnding() Assert.True(parserRes.IsOk); var parser = parserRes.Result; Assert.NotNull(parser); - + var parseResult = parser.Parse(source); - + GraphVizEBNFSyntaxTreeVisitor grapher = new GraphVizEBNFSyntaxTreeVisitor(); grapher.VisitTree(parseResult.SyntaxTree); var graph = grapher.Graph.Compile(); //File.WriteAllText(@"c:\tmp\graph.dot", graph); - + Assert.True(parseResult.IsOk); var ast = parseResult.Result; indented.Block root = ast as indented.Block; @@ -1172,9 +1180,9 @@ public static void TestIndentedParserNeverEnding() IfThenElse ifthenelse = root.Statements.First() as indented.IfThenElse; Assert.NotNull(ifthenelse.Cond); Assert.NotNull(ifthenelse.Then); - Assert.Equal(3,ifthenelse.Then.Statements.Count); + Assert.Equal(3, ifthenelse.Then.Statements.Count); Assert.NotNull(ifthenelse.Else); - Assert.Equal(2,ifthenelse.Else.Statements.Count); + Assert.Equal(2, ifthenelse.Else.Statements.Count); } private static void TestTemplate() @@ -1186,10 +1194,10 @@ private static void TestTemplate() var fsmGraph = fsm.ToGraphViz(); Console.WriteLine(fsmGraph); - + var source = @"hello - {= world =} - billy - {% if (a == 1) %} - bob - {%else%} - boubou - {%endif%}"; - + var tokens = genericLexer.Tokenize(source); foreach (var token in tokens.Tokens.MainTokens()) { @@ -1222,7 +1230,7 @@ private static void TestTemplate() } } - + private static void TestTemplateFor() { var lexerResult = LexerBuilder.BuildLexer(); @@ -1232,7 +1240,7 @@ private static void TestTemplateFor() var fsmGraph = fsm.ToGraphViz(); Console.WriteLine(fsmGraph); - + var source = @"hello @@ -1255,9 +1263,9 @@ private static void TestTemplateFor() ... {% end%} "; - + var tokens = genericLexer.Tokenize(source); - + var channels = tokens.Tokens.GetChannels(); StringBuilder b = new StringBuilder(); foreach (var channel in channels) @@ -1270,13 +1278,13 @@ private static void TestTemplateFor() b.AppendLine(); } - + // File.WriteAllText(@"c:\temp\tokens.txt",b.ToString()); - - + + foreach (var token in tokens.Tokens.MainTokens()) { - + Console.WriteLine(token); } @@ -1289,7 +1297,7 @@ private static void TestTemplateFor() { { "world", "monde" }, { "a", 1 }, - { "items", new List(){"one","two","three"}} + { "items", new List() { "one", "two", "three" } } }; var r = build.Result.Parse(source); if (r.IsOk) @@ -1313,7 +1321,9 @@ private static void Issue414() { var parserInstance = new Issue414Parser(); var builder = new ParserBuilder(); - var buildResult = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "block");//line-based, 1 statement per line. + var buildResult = + builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, + "block"); //line-based, 1 statement per line. var parser = buildResult.Result; string source = "funcA(funcC(B==2));"; Stopwatch chrono = new Stopwatch(); @@ -1322,12 +1332,14 @@ private static void Issue414() chrono.Stop(); Console.WriteLine($"{result.Result} : {chrono.ElapsedMilliseconds} ms"); } - + private static void Issue414Expr() { var parserInstance = new Issue414ExpressionParser(); var builder = new ParserBuilder(); - var buildResult = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "block");//line-based, 1 statement per line. + var buildResult = + builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, + "block"); //line-based, 1 statement per line. var parser = buildResult.Result; string source = "funcA(funcC(B==2));"; Stopwatch chrono = new Stopwatch(); @@ -1340,11 +1352,11 @@ private static void Issue414Expr() private static void BroadWindow() { - + var whileParser = new IndentedWhileParserGeneric(); var builder = new ParserBuilder(); var buildResult = builder.BuildParser(whileParser, ParserType.EBNF_LL_RECURSIVE_DESCENT, "program"); - + var parser = buildResult.Result; string program = @" a:=0 @@ -1355,9 +1367,11 @@ print a var result = parser.Parse(program); Console.WriteLine($"{result.IsOk}"); } + private static void Main(string[] args) { - testIssue516(); + TestIssue596GettingStartedParser(); + //testIssue516(); //TestIssue507(); //TestFStrings(); //TestIssue495(); @@ -1439,6 +1453,7 @@ private static void TestXmlParser() r.Errors.ForEach(x => Console.WriteLine(x.Message)); return; } + Console.WriteLine(r.Result.SyntaxParser.Dump()); var pr = r.Result.Parse(@" @@ -1464,7 +1479,7 @@ inner inner content Console.WriteLine(pr.Result); } } - + private static void TestLexerModes() { // testManySugar(); @@ -1485,6 +1500,7 @@ private static void TestLexerModes() { Console.WriteLine(tokens.Error); } + Console.WriteLine("stop"); } @@ -1521,6 +1537,7 @@ private static void TestShortGeneric() Console.WriteLine(error.Message); } } + ; } @@ -1534,7 +1551,7 @@ private static void TestShortOperations() var parser = buildResult.Result; Assert.NotNull(parser); var result = parser.Parse("-1 +2 * (5 + 6) - 4 "); - Assert.Equal(-1+2*(5+6)-4, result.Result); + Assert.Equal(-1 + 2 * (5 + 6) - 4, result.Result); } private static void TestIssue239() @@ -1547,15 +1564,15 @@ private static void TestIssue332() { ParserBuilder Parser = new ParserBuilder(); Issue332Parser oparser = new Issue332Parser(); - var r = Parser.BuildParser(oparser,ParserType.EBNF_LL_RECURSIVE_DESCENT); + var r = Parser.BuildParser(oparser, ParserType.EBNF_LL_RECURSIVE_DESCENT); Check.That(r).Not.IsOk(); foreach (var error in r.Errors) { Console.WriteLine(error.Message); } - + } - + private static void Issue351() { var parserInstance = new ExpressionParser(); @@ -1573,7 +1590,7 @@ private static void Issue351() ; } - + private static void NodeNames() { var parserInstance = new ExpressionParser(); @@ -1602,7 +1619,7 @@ private static void IndentRefactoring() var l = LexerBuilder.BuildLexer(); if (l.IsOk) { - var source =@"if truc == 1 + var source = @"if truc == 1 un = 1 deux = 2 else @@ -1626,47 +1643,48 @@ private static void IndentRefactoring() } } } - + private static List> postProcess(List> tokens) { var mayLeft = new List() { - ExpressionToken.INT, ExpressionToken.DOUBLE, ExpressionToken.IDENTIFIER + ExpressionToken.INT, ExpressionToken.DOUBLE, ExpressionToken.IDENTIFIER }; - + var mayRight = new List() { ExpressionToken.INT, ExpressionToken.DOUBLE, ExpressionToken.LPAREN, ExpressionToken.IDENTIFIER }; - - Func mayOmmitLeft = (ExpressionToken tokenid) => mayLeft.Contains(tokenid); - - Func mayOmmitRight = (ExpressionToken tokenid) => mayRight.Contains(tokenid); - - + + Func mayOmmitLeft = (ExpressionToken tokenid) => mayLeft.Contains(tokenid); + + Func mayOmmitRight = (ExpressionToken tokenid) => mayRight.Contains(tokenid); + + List> newTokens = new List>(); for (int i = 0; i < tokens.Count; i++) { - if ( i >= 1 && - mayOmmitRight(tokens[i].TokenID) && mayOmmitLeft(tokens[i-1].TokenID)) + if (i >= 1 && + mayOmmitRight(tokens[i].TokenID) && mayOmmitLeft(tokens[i - 1].TokenID)) { newTokens.Add(new Token() { TokenID = ExpressionToken.TIMES }); } + newTokens.Add(tokens[i]); } return newTokens; } - - + + private static void TestLexerPostProcess() { var Parser = postProcessedLexerParser.PostProcessedLexerParserBuilder.buildPostProcessedLexerParser(); - + var r = Parser.Parse("2 * x"); if (r.IsError) { @@ -1677,11 +1695,12 @@ private static void TestLexerPostProcess() return; } + var res = r.Result.Evaluate(new ExpressionContext(new Dictionary() { { "x", 2 } })); - Console.WriteLine("2 * x = "+(res.HasValue ? res.Value.ToString() : "?")); - - + Console.WriteLine("2 * x = " + (res.HasValue ? res.Value.ToString() : "?")); + + r = Parser.Parse("2 x"); if (r.IsError) { @@ -1692,10 +1711,11 @@ private static void TestLexerPostProcess() return; } + res = r.Result.Evaluate(new ExpressionContext(new Dictionary() { { "x", 2 } })); - Console.WriteLine("2 x = "+(res.HasValue ? res.Value.ToString() : "?")); - + Console.WriteLine("2 x = " + (res.HasValue ? res.Value.ToString() : "?")); + r = Parser.Parse("2 ( x ) "); if (r.IsError) { @@ -1706,10 +1726,11 @@ private static void TestLexerPostProcess() return; } + res = r.Result.Evaluate(new ExpressionContext(new Dictionary() { { "x", 2 } })); - Console.WriteLine("2 (x) = "+(res.HasValue ? res.Value.ToString() : "?")); - + Console.WriteLine("2 (x) = " + (res.HasValue ? res.Value.ToString() : "?")); + r = Parser.Parse("x x "); if (r.IsError) { @@ -1720,16 +1741,17 @@ private static void TestLexerPostProcess() return; } + res = r.Result.Evaluate(new ExpressionContext(new Dictionary() { { "x", 2 } })); - Console.WriteLine("x x = "+(res.HasValue ? res.Value.ToString() : "?")); - + Console.WriteLine("x x = " + (res.HasValue ? res.Value.ToString() : "?")); + } - - private static void TestLexerPostProcessEBNF() + + private static void TestLexerPostProcessEBNF() { var Parser = postProcessedLexerParser.PostProcessedLexerParserBuilder.buildPostProcessedLexerParser(); - + var r = Parser.Parse("2 * x"); if (r.IsError) { @@ -1740,11 +1762,12 @@ private static void TestLexerPostProcessEBNF() return; } + var res = r.Result.Evaluate(new ExpressionContext(new Dictionary() { { "x", 2 } })); - Console.WriteLine("2 * x = "+(res.HasValue ? res.Value.ToString() : "?")); - - + Console.WriteLine("2 * x = " + (res.HasValue ? res.Value.ToString() : "?")); + + r = Parser.Parse("2 x"); if (r.IsError) { @@ -1755,10 +1778,11 @@ private static void TestLexerPostProcessEBNF() return; } + res = r.Result.Evaluate(new ExpressionContext(new Dictionary() { { "x", 2 } })); - Console.WriteLine("2 x = "+(res.HasValue ? res.Value.ToString() : "?")); - + Console.WriteLine("2 x = " + (res.HasValue ? res.Value.ToString() : "?")); + r = Parser.Parse("2 ( x ) "); if (r.IsError) { @@ -1769,10 +1793,11 @@ private static void TestLexerPostProcessEBNF() return; } + res = r.Result.Evaluate(new ExpressionContext(new Dictionary() { { "x", 2 } })); - Console.WriteLine("2 (x) = "+(res.HasValue ? res.Value.ToString() : "?")); - + Console.WriteLine("2 (x) = " + (res.HasValue ? res.Value.ToString() : "?")); + r = Parser.Parse("x x "); if (r.IsError) { @@ -1783,137 +1808,173 @@ private static void TestLexerPostProcessEBNF() return; } + res = r.Result.Evaluate(new ExpressionContext(new Dictionary() { { "x", 2 } })); - - Console.WriteLine("x x = "+(res.HasValue ? res.Value.ToString() : "?")); - - } - - private static void TestIssue487() - { - string source = "@g_capcalc >> EtnRef: @appcapcalc.bankcapconfigid"; - var lexBuild = LexerBuilder.BuildLexer(); - - Check.That(lexBuild).IsOk(); - - var lexer = lexBuild.Result; - var lexer487 = lexer as GenericLexer; - var gviz = lexer487.FSMBuilder.Fsm.ToGraphViz(); - var lexed = lexer.Tokenize(source); - if (lexed.IsOk) - { - foreach (var token in lexed.Tokens) - { - Console.WriteLine(token); - } - } - else - { - Console.Write(lexed.Error); - } - } - - private static void TestIssue495() - { - Parser parser; - - ParserBuilder builder = new ParserBuilder("en"); - var build = builder.BuildParser(new Issue495Parser(), ParserType.EBNF_LL_RECURSIVE_DESCENT, "program"); - Check.That(build).IsOk(); - parser = build.Result; - - Check.That(parser).IsNotNull(); - Check.That(parser.Lexer).IsNotNull(); - Check.That(parser.Lexer).IsInstanceOf>(); - var lexer = parser.Lexer as GenericLexer; - string source = "test = \"3 3\";"; - var tokenized = lexer.Tokenize(source); - Check.That(tokenized).IsOkLexing(); - var tokens = tokenized.Tokens.MainTokens(); - Check.That(tokens).CountIs(7); - var stringValue = tokens[3]; - Check.That(stringValue).IsNotNull(); - Check.That(stringValue.TokenID).IsEqualTo(Issue495Token.StringValue); - - - var parsed = parser.Parse(source); - Check.That(parsed).IsOkParsing(); - Check.That(parsed.Result).IsEqualTo("test=3 3"); - - } - - private static void TestFStrings() - { - IndentedWhileTests tests = new IndentedWhileTests(); - tests.TestFString(); - } - } - public enum TestGrammarToken - { - [Lexeme(GenericToken.SugarToken,",")] - COMMA = 1 - } - - + Console.WriteLine("x x = " + (res.HasValue ? res.Value.ToString() : "?")); - public class ErroneousGrammar - { - [Production("clauses : clause (COMMA [D] clause)*")] + } - public object test() + private static void TestIssue487() + { + string source = "@g_capcalc >> EtnRef: @appcapcalc.bankcapconfigid"; + var lexBuild = LexerBuilder.BuildLexer(); + + Check.That(lexBuild).IsOk(); + + var lexer = lexBuild.Result; + var lexer487 = lexer as GenericLexer; + var gviz = lexer487.FSMBuilder.Fsm.ToGraphViz(); + var lexed = lexer.Tokenize(source); + if (lexed.IsOk) { - return null; - } - } + foreach (var token in lexed.Tokens) + { + Console.WriteLine(token); + } + } + else + { + Console.Write(lexed.Error); + } + } - public class RecursiveGrammar - { - [Production("first : second COMMA[d]")] - public object FirstRecurse(object o) + private static void TestIssue495() + { + Parser parser; + + ParserBuilder builder = new ParserBuilder("en"); + var build = builder.BuildParser(new Issue495Parser(), ParserType.EBNF_LL_RECURSIVE_DESCENT, "program"); + Check.That(build).IsOk(); + parser = build.Result; + + Check.That(parser).IsNotNull(); + Check.That(parser.Lexer).IsNotNull(); + Check.That(parser.Lexer).IsInstanceOf>(); + var lexer = parser.Lexer as GenericLexer; + string source = "test = \"3 3\";"; + var tokenized = lexer.Tokenize(source); + Check.That(tokenized).IsOkLexing(); + var tokens = tokenized.Tokens.MainTokens(); + Check.That(tokens).CountIs(7); + var stringValue = tokens[3]; + Check.That(stringValue).IsNotNull(); + Check.That(stringValue.TokenID).IsEqualTo(Issue495Token.StringValue); + + + var parsed = parser.Parse(source); + Check.That(parsed).IsOkParsing(); + Check.That(parsed.Result).IsEqualTo("test=3 3"); + + } + + private static void TestFStrings() { - return o; - } - - [Production("second : third COMMA[d]")] - public object SecondRecurse(object o) + IndentedWhileTests tests = new IndentedWhileTests(); + tests.TestFString(); + } + + + private static void TestIssue596GettingStartedParser() { - return o; + var parserInstance = new GettingStartedParser(); + var builder = new ParserBuilder(); + var buildResult = builder.BuildParser(parserInstance, ParserType.LL_RECURSIVE_DESCENT, "expression"); + Assert.True(buildResult.IsOk); + var parser = buildResult.Result; + var result = parser.Parse("1+2+3"); + Assert.True(result.IsOk); + Assert.Equal(6, result.Result); + + result = parser.Parse(""); + Assert.True(result.IsError); } - - [Production("third : first COMMA[d]")] - public object ThirdRecurse(object o) + + public static void TestDanglingElse() { - return o; + var parserInstance = new DanglingElseParserWithEbnfChoices(); + var builder = new ParserBuilder(); + var buildResult = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "program"); + buildResult.Result.Configuration.CaptureAmbiguities = true; + buildResult.Result.Configuration.AmbiguityStrategy = AmbiguityResolutionStrategy.All; + Assert.True(buildResult.IsOk); + var parser = buildResult.Result; + string source = @" +if x == 1 then +if y == 2 then +a := 3 +else +a := 4"; + var result = parser.Parse(source); + ; + } - } - - public class RecursiveGrammar2 - { - [Production("first : second* third COMMA[d]")] - public object FirstRecurse(List seconds, object third) + + public enum TestGrammarToken { - return third; - } - - [Production("first : second? third COMMA[d]")] - public object FirstRecurse2(ValueOption optSecond, object third) + [Lexeme(GenericToken.SugarToken, ",")] COMMA = 1 + } + + + + public class ErroneousGrammar { - return null; + [Production("clauses : clause (COMMA [D] clause)*")] + + public object test() + { + return null; + } } - - [Production("second : COMMA[d]")] - public object SecondRecurse() + + public class RecursiveGrammar { - return null; + [Production("first : second COMMA[d]")] + public object FirstRecurse(object o) + { + return o; + } + + [Production("second : third COMMA[d]")] + public object SecondRecurse(object o) + { + return o; + } + + [Production("third : first COMMA[d]")] + public object ThirdRecurse(object o) + { + return o; + } } - - [Production("third : first COMMA[d]")] - public object ThirdRecurse(object o) + + public class RecursiveGrammar2 { - return o; + [Production("first : second* third COMMA[d]")] + public object FirstRecurse(List seconds, object third) + { + return third; + } + + [Production("first : second? third COMMA[d]")] + public object FirstRecurse2(ValueOption optSecond, object third) + { + return null; + } + + [Production("second : COMMA[d]")] + public object SecondRecurse() + { + return null; + } + + [Production("third : first COMMA[d]")] + public object ThirdRecurse(object o) + { + return o; + } } + } - } \ No newline at end of file diff --git a/src/samples/SimpleExpressionParser/GenericSimpleExpressionParser.cs b/src/samples/SimpleExpressionParser/GenericSimpleExpressionParser.cs index 502f2391..0e41925d 100644 --- a/src/samples/SimpleExpressionParser/GenericSimpleExpressionParser.cs +++ b/src/samples/SimpleExpressionParser/GenericSimpleExpressionParser.cs @@ -5,6 +5,7 @@ namespace simpleExpressionParser; [ParserRoot("root")] +[FirstDerivation] //[BroadenTokenWindow] public class GenericSimpleExpressionParser { diff --git a/src/samples/SimpleExpressionParser/SimpleExpressionParser.cs b/src/samples/SimpleExpressionParser/SimpleExpressionParser.cs index 2898a1aa..0d294005 100644 --- a/src/samples/SimpleExpressionParser/SimpleExpressionParser.cs +++ b/src/samples/SimpleExpressionParser/SimpleExpressionParser.cs @@ -5,6 +5,7 @@ namespace simpleExpressionParser { [ParserRoot("root")] + [FirstDerivation] //[BroadenTokenWindow] public class SimpleExpressionParser { diff --git a/src/samples/SimpleTemplate/TemplateParser.cs b/src/samples/SimpleTemplate/TemplateParser.cs index 7d17080e..8f9f4ca1 100644 --- a/src/samples/SimpleTemplate/TemplateParser.cs +++ b/src/samples/SimpleTemplate/TemplateParser.cs @@ -11,6 +11,7 @@ namespace SimpleTemplate { [ParserRoot("template")] + [FirstDerivation] public class TemplateParser { diff --git a/src/samples/SlowEOS/SlowOnBadParseEos.cs b/src/samples/SlowEOS/SlowOnBadParseEos.cs index 4fbd6401..295c91ea 100644 --- a/src/samples/SlowEOS/SlowOnBadParseEos.cs +++ b/src/samples/SlowEOS/SlowOnBadParseEos.cs @@ -5,6 +5,7 @@ namespace SlowEOS; [ParserRoot("root")] +[FirstDerivation] public class SlowOnBadParseEos { [Production("root : SlowOnBadParseEos_expressions")] diff --git a/src/samples/SlowEOS/SlowOnBadParseEosReducedPrecedences.cs b/src/samples/SlowEOS/SlowOnBadParseEosReducedPrecedences.cs index 67ff636a..12bc2101 100644 --- a/src/samples/SlowEOS/SlowOnBadParseEosReducedPrecedences.cs +++ b/src/samples/SlowEOS/SlowOnBadParseEosReducedPrecedences.cs @@ -5,6 +5,7 @@ namespace SlowEOS; [ParserRoot("root")] +[FirstDerivation] public class SlowOnBadParseEosReducedPrecedences { [Production("root : SlowOnBadParseEosReducedPrecedences_expressions")] diff --git a/src/samples/XML/MinimalXmlParser.cs b/src/samples/XML/MinimalXmlParser.cs index 6ff1a466..041591f2 100644 --- a/src/samples/XML/MinimalXmlParser.cs +++ b/src/samples/XML/MinimalXmlParser.cs @@ -8,6 +8,7 @@ namespace XML { [ParserRoot("document")] + [FirstDerivation] public class MinimalXmlParser { diff --git a/src/samples/XML/MinimalXmlParser2.cs b/src/samples/XML/MinimalXmlParser2.cs index d2ec5f37..76c8251e 100644 --- a/src/samples/XML/MinimalXmlParser2.cs +++ b/src/samples/XML/MinimalXmlParser2.cs @@ -8,6 +8,7 @@ namespace XML { [ParserRoot("document")] + [FirstDerivation] public class MinimalXmlParser2 { diff --git a/src/samples/expressionParser/ExpressionParser.cs b/src/samples/expressionParser/ExpressionParser.cs index 50e1ef31..6a48102d 100644 --- a/src/samples/expressionParser/ExpressionParser.cs +++ b/src/samples/expressionParser/ExpressionParser.cs @@ -3,6 +3,7 @@ namespace expressionparser { + [FirstDerivation] public class ExpressionParser { [NodeName("integer")] diff --git a/src/samples/jsonparser/EbnfJsonGenericParser.cs b/src/samples/jsonparser/EbnfJsonGenericParser.cs index 8f9c1310..859eca02 100644 --- a/src/samples/jsonparser/EbnfJsonGenericParser.cs +++ b/src/samples/jsonparser/EbnfJsonGenericParser.cs @@ -8,6 +8,7 @@ namespace jsonparser { [BroadenTokenWindow] [ParserRoot("root")] + [FirstDerivation] public class EbnfJsonGenericParser { #region root @@ -158,6 +159,7 @@ public JSon property(Token key, JSon value) [BroadenTokenWindow] [ParserRoot("root")] + [FirstDerivation] public class EbnfJsonGenericParserStringNotEscaped { #region root diff --git a/src/samples/jsonparser/JSONLexer.cs b/src/samples/jsonparser/JSONLexer.cs index c4aaabd0..b005a1d4 100644 --- a/src/samples/jsonparser/JSONLexer.cs +++ b/src/samples/jsonparser/JSONLexer.cs @@ -17,12 +17,12 @@ public void AddDefinition(TokenDefinition tokenDefinition) { } - public LexerResult Tokenize(string source) + public LexerResult Tokenize(string source, bool applyPostProcess = false) { return Tokenize(new ReadOnlyMemory(source.ToCharArray())); } - public LexerResult Tokenize(ReadOnlyMemory source) + public LexerResult Tokenize(ReadOnlyMemory source, bool applyPostProcess = false) { var tokens = new List>(); var position = 0; diff --git a/src/samples/postProcessedLexerParser/PostProcessedLexerParserBuilder.cs b/src/samples/postProcessedLexerParser/PostProcessedLexerParserBuilder.cs index a5151672..e393c376 100644 --- a/src/samples/postProcessedLexerParser/PostProcessedLexerParserBuilder.cs +++ b/src/samples/postProcessedLexerParser/PostProcessedLexerParserBuilder.cs @@ -35,7 +35,11 @@ private static List> postProcessFormula(List() { - TokenID = FormulaToken.TIMES + TokenID = FormulaToken.TIMES, + IsEOS = false, + End = false, + IsExplicit = false, + SpanValue = "*".ToCharArray(), }); } newTokens.Add(tokens[i]); diff --git a/src/samples/while/parser/WhileParser.cs b/src/samples/while/parser/WhileParser.cs index 5aa5e502..a4029751 100644 --- a/src/samples/while/parser/WhileParser.cs +++ b/src/samples/while/parser/WhileParser.cs @@ -7,6 +7,7 @@ namespace csly.whileLang.parser { [ParserRoot("statement")] + [FirstDerivation] public class WhileParser { #region COMPARISON OPERATIONS diff --git a/src/samples/while/parser/WhileParserGeneric.cs b/src/samples/while/parser/WhileParserGeneric.cs index 47793b82..1a3f2f74 100644 --- a/src/samples/while/parser/WhileParserGeneric.cs +++ b/src/samples/while/parser/WhileParserGeneric.cs @@ -7,6 +7,7 @@ namespace csly.whileLang.parser { [ParserRoot("statement")] + [FirstDerivation] public class WhileParserGeneric { #region COMPARISON OPERATIONS diff --git a/src/sly/i18n/I18N.cs b/src/sly/i18n/I18N.cs index b85149e4..f5b32763 100644 --- a/src/sly/i18n/I18N.cs +++ b/src/sly/i18n/I18N.cs @@ -9,9 +9,11 @@ public class I18N { private readonly IDictionary> Translations; - private static I18N _instance; + private static readonly I18N _instance = new I18N(); - public static I18N Instance => _instance ?? (_instance = new I18N()); + public static I18N Instance => _instance; + + private readonly object _lock = new object(); private I18N() { @@ -21,9 +23,13 @@ private I18N() public string GetText(string lang, I18NMessage key, params string[] args) { lang = lang ?? CultureInfo.CurrentCulture.TwoLetterISOLanguageName; - if (!Translations.TryGetValue(lang, out var translation)) + IDictionary translation; + lock (_lock) { - translation = Load(lang); + if (!Translations.TryGetValue(lang, out translation)) + { + translation = Load(lang); + } } @@ -33,6 +39,11 @@ public string GetText(string lang, I18NMessage key, params string[] args) private IDictionary Load(string lang) { + if (Translations.TryGetValue(lang, out var existing)) + { + return existing; + } + var translation = new Dictionary(); var assembly = GetType().Assembly; using (var stream = assembly.GetManifestResourceStream($"sly.i18n.translations.{lang}.txt")) @@ -60,7 +71,7 @@ private IDictionary Load(string lang) } else { - return Load("en"); + return lang == "en" ? translation : Load("en"); } } diff --git a/src/sly/lexer/GenericLexer.cs b/src/sly/lexer/GenericLexer.cs index 85db97c4..7753237c 100644 --- a/src/sly/lexer/GenericLexer.cs +++ b/src/sly/lexer/GenericLexer.cs @@ -143,13 +143,13 @@ public void AddDefinition(TokenDefinition tokenDefinition) } - public LexerResult Tokenize(string source) + public LexerResult Tokenize(string source, bool applyPostProcess = false) { var memorySource = new ReadOnlyMemory(source.ToCharArray()); - return Tokenize(memorySource); + return Tokenize(memorySource, applyPostProcess); } - public LexerResult Tokenize(ReadOnlyMemory source) + public LexerResult Tokenize(ReadOnlyMemory source, bool applyPostProcess = false) { Stack> lexersStack = new Stack>(); @@ -272,6 +272,12 @@ public LexerResult Tokenize(ReadOnlyMemory source) } tokens.Add(eos); + + if (applyPostProcess && LexerPostProcess != null) + { + tokens = LexerPostProcess(tokens); + } + return new LexerResult(tokens); } diff --git a/src/sly/lexer/ILexer.cs b/src/sly/lexer/ILexer.cs index 0dc0d9c4..150e989a 100644 --- a/src/sly/lexer/ILexer.cs +++ b/src/sly/lexer/ILexer.cs @@ -11,9 +11,9 @@ public interface ILexer where T : struct, Enum Dictionary> LexemeLabels { get; set; } void AddDefinition(TokenDefinition tokenDefinition); - LexerResult Tokenize(string source); + LexerResult Tokenize(string source, bool applyPostProcess = false); - LexerResult Tokenize(ReadOnlyMemory source); + LexerResult Tokenize(ReadOnlyMemory source, bool applyPostProcess = false); string I18n { get; set; } diff --git a/src/sly/lexer/Lexer.cs b/src/sly/lexer/Lexer.cs index 8ee02a7b..d59c778f 100644 --- a/src/sly/lexer/Lexer.cs +++ b/src/sly/lexer/Lexer.cs @@ -24,7 +24,7 @@ public void AddDefinition(TokenDefinition tokenDefinition) } - public LexerResult Tokenize(string source) + public LexerResult Tokenize(string source, bool applyPostProcess = false) { List> tokens = new List>(); @@ -92,11 +92,16 @@ public LexerResult Tokenize(string source) tokens.Add(eos); + if (applyPostProcess && LexerPostProcess != null) + { + tokens = LexerPostProcess(tokens); + } + return new LexerResult(tokens); } [ExcludeFromCodeCoverage] - public LexerResult Tokenize(ReadOnlyMemory source) + public LexerResult Tokenize(ReadOnlyMemory source, bool applyPostProcess = false) { return Tokenize(source.ToString()); } diff --git a/src/sly/lexer/LexerBuilder.cs b/src/sly/lexer/LexerBuilder.cs index b4e90761..d7b1a1db 100644 --- a/src/sly/lexer/LexerBuilder.cs +++ b/src/sly/lexer/LexerBuilder.cs @@ -101,9 +101,9 @@ public static class LexerBuilder public static BuildResult> BuildLexer( Action> extensionBuilder = null, - LexerPostProcess lexerPostProcess = null) where IN : struct, Enum + LexerPostProcess lexerPostProcess = null, string lang=null) where IN : struct, Enum { - return BuildLexer(new BuildResult>(), extensionBuilder, lexerPostProcess: lexerPostProcess); + return BuildLexer(new BuildResult>(), extensionBuilder, lexerPostProcess: lexerPostProcess, lang:lang); } public static BuildResult> BuildLexer(BuildResult> result, diff --git a/src/sly/lexer/Token.cs b/src/sly/lexer/Token.cs index 4719f843..bf51d471 100644 --- a/src/sly/lexer/Token.cs +++ b/src/sly/lexer/Token.cs @@ -132,7 +132,7 @@ public List> PreviousTokens(int channelId) if (position >= 0 && position <= channel.Count - 1) { - for (int i = position; position > 0; position--) + for (int i = position; position >= 0; position--) { var token = channel[position]; if (token != null) diff --git a/src/sly/parser/exceptions/AmbiguousGrammarException.cs b/src/sly/parser/exceptions/AmbiguousGrammarException.cs new file mode 100644 index 00000000..711ac2d4 --- /dev/null +++ b/src/sly/parser/exceptions/AmbiguousGrammarException.cs @@ -0,0 +1,35 @@ +using System; +using System.Text; +using sly.parser.syntax.tree; + +namespace sly.parser.exceptions +{ + public class AmbiguousGrammarException : Exception where IN : struct, Enum + { + public ParseForest Forest { get; } + + public AmbiguousGrammarException(ParseForest forest) + : base($"Ambiguous grammar detected: {forest.Count} alternative parse trees found") + { + Forest = forest; + } + + public override string Message + { + get + { + var sb = new StringBuilder(base.Message); + if (Forest.Ambiguities != null && Forest.Ambiguities.Count > 0) + { + sb.AppendLine(); + sb.AppendLine("Ambiguity points:"); + foreach (var amb in Forest.Ambiguities) + { + sb.AppendLine($" - NonTerminal '{amb.NonTerminalName}' at position {amb.Position}: {amb.AlternativeCount} alternatives"); + } + } + return sb.ToString(); + } + } + } +} diff --git a/src/sly/parser/generator/AllDerivationsAttribute.cs b/src/sly/parser/generator/AllDerivationsAttribute.cs new file mode 100644 index 00000000..40b7c7b7 --- /dev/null +++ b/src/sly/parser/generator/AllDerivationsAttribute.cs @@ -0,0 +1,12 @@ +using System; + +namespace sly.parser.generator; + +[AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] +public class AllDerivationsAttribute : Attribute +{ + public AllDerivationsAttribute() + { + + } +} \ No newline at end of file diff --git a/src/sly/parser/generator/AmbiguityResolutionStrategy.cs b/src/sly/parser/generator/AmbiguityResolutionStrategy.cs new file mode 100644 index 00000000..79703948 --- /dev/null +++ b/src/sly/parser/generator/AmbiguityResolutionStrategy.cs @@ -0,0 +1,22 @@ +using System; + +namespace sly.parser.generator +{ + /// + /// Stratégie de résolution d'ambiguïté + /// + public enum AmbiguityResolutionStrategy + { + /// Returns first derivation default) + First, + + /// Returns all derivations (in a forest) + All, + + /// THrows an exception if ambiguity is detected + ThrowException, + + /// Returns longest derivation + Longest + } +} diff --git a/src/sly/parser/generator/EBNFParserBuilder.cs b/src/sly/parser/generator/EBNFParserBuilder.cs index bb36b75d..1d0327ef 100644 --- a/src/sly/parser/generator/EBNFParserBuilder.cs +++ b/src/sly/parser/generator/EBNFParserBuilder.cs @@ -40,6 +40,23 @@ public override BuildResult> BuildParser(object parserInstance, bool broadenTokenWindow = parserInstance.GetType().GetCustomAttribute() != null; bool autoCloseIndentations = parserInstance.GetType().GetCustomAttribute() != null; + + AmbiguityResolutionStrategy ambiguityResolutionStrategy = AmbiguityResolutionStrategy.First; + + if (parserInstance.GetType().GetCustomAttribute() != null) + { + ambiguityResolutionStrategy = AmbiguityResolutionStrategy.Longest; + } + + if (parserInstance.GetType().GetCustomAttribute() != null) + { + ambiguityResolutionStrategy = AmbiguityResolutionStrategy.All; + } + + if (parserInstance.GetType().GetCustomAttribute() != null) + { + ambiguityResolutionStrategy = AmbiguityResolutionStrategy.ThrowException; + } var ruleparser = new RuleParser(); var builder = new ParserBuilder>(I18N); @@ -57,6 +74,8 @@ public override BuildResult> BuildParser(object parserInstance, configuration.UseMemoization = useMemoization; configuration.BroadenTokenWindow = broadenTokenWindow; configuration.AutoCloseIndentations = autoCloseIndentations; + configuration.AmbiguityStrategy = ambiguityResolutionStrategy; + configuration.CaptureAmbiguities = ambiguityResolutionStrategy != AmbiguityResolutionStrategy.First; LeftRecursionChecker recursionChecker = new LeftRecursionChecker(); SetEmptyNonTerminals(configuration); @@ -72,6 +91,7 @@ public override BuildResult> BuildParser(object parserInstance, } configuration.StartingRule = rootRule; + } catch (Exception e) { diff --git a/src/sly/parser/generator/FirstDerivationAttribute.cs b/src/sly/parser/generator/FirstDerivationAttribute.cs new file mode 100644 index 00000000..87fd2a17 --- /dev/null +++ b/src/sly/parser/generator/FirstDerivationAttribute.cs @@ -0,0 +1,12 @@ +using System; + +namespace sly.parser.generator; + +[AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] +public class FirstDerivationAttribute : Attribute +{ + public FirstDerivationAttribute() + { + + } +} \ No newline at end of file diff --git a/src/sly/parser/generator/LongestDerivationAttribute.cs b/src/sly/parser/generator/LongestDerivationAttribute.cs new file mode 100644 index 00000000..6a914d37 --- /dev/null +++ b/src/sly/parser/generator/LongestDerivationAttribute.cs @@ -0,0 +1,12 @@ +using System; + +namespace sly.parser.generator; + +[AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] +public class LongestDerivationAttribute : Attribute +{ + public LongestDerivationAttribute() + { + + } +} \ No newline at end of file diff --git a/src/sly/parser/generator/ParserConfiguration.cs b/src/sly/parser/generator/ParserConfiguration.cs index b3d5b614..92694b69 100644 --- a/src/sly/parser/generator/ParserConfiguration.cs +++ b/src/sly/parser/generator/ParserConfiguration.cs @@ -19,7 +19,18 @@ public class ParserConfiguration where IN : struct, Enum public bool BroadenTokenWindow { get; set; } = false; - public bool AutoCloseIndentations { get; set; } + public bool AutoCloseIndentations { get; set; } + + /// + /// If true, returns all possible derivations in case of ambiguity, otherwise returns the first one (default behavior) + /// + public bool CaptureAmbiguities { get; set; } = false; + + /// + /// Ambiguity resolution strategy to apply when CaptureAmbiguities is false. Default is First. + /// + public AmbiguityResolutionStrategy AmbiguityStrategy { get; set; } + = AmbiguityResolutionStrategy.First; public void AddNonTerminalIfNotExists(NonTerminal nonTerminal) { diff --git a/src/sly/parser/generator/ThrowsErrorOnAmbiguityAttribute.cs b/src/sly/parser/generator/ThrowsErrorOnAmbiguityAttribute.cs new file mode 100644 index 00000000..7e7ddf5a --- /dev/null +++ b/src/sly/parser/generator/ThrowsErrorOnAmbiguityAttribute.cs @@ -0,0 +1,12 @@ +using System; + +namespace sly.parser.generator; + +[AttributeUsage(AttributeTargets.Class, AllowMultiple = false)] +public class ThrowsErrorOnAmbiguityAttribute : Attribute +{ + public ThrowsErrorOnAmbiguityAttribute() + { + + } +} \ No newline at end of file diff --git a/src/sly/parser/generator/visitor/SyntaxTreeVisitor.cs b/src/sly/parser/generator/visitor/SyntaxTreeVisitor.cs index 297978eb..4626a848 100644 --- a/src/sly/parser/generator/visitor/SyntaxTreeVisitor.cs +++ b/src/sly/parser/generator/visitor/SyntaxTreeVisitor.cs @@ -4,6 +4,7 @@ using sly.lexer; using sly.parser.parser; using sly.parser.syntax.tree; +using sly.parser.generator; using static sly.parser.parser.ValueOptionConstructors; namespace sly.parser.generator.visitor @@ -142,7 +143,7 @@ public SyntaxTreeVisitor(ParserConfiguration conf, object parserInstanc public OUT VisitSyntaxTree(ISyntaxNode root, object context = null) { - var result = Visit(root, context); + var result = Visit(root, context ?? new NoContext()); return result.ValueResult; } @@ -192,7 +193,7 @@ private SyntaxVisitorResult Visit(SyntaxNode node, object cont MethodInfo method = null; try { - if (!(context is NoContext)) + if (context != null && !(context is NoContext)) { args.Add(context); } diff --git a/src/sly/parser/parser/ParseError.cs b/src/sly/parser/parser/ParseError.cs index b6d38803..d48098af 100644 --- a/src/sly/parser/parser/ParseError.cs +++ b/src/sly/parser/parser/ParseError.cs @@ -47,12 +47,15 @@ protected string GetContextualMessage(string fullSource, int line, int column, s { StringBuilder sb = new StringBuilder(); sb.AppendLine(ErrorMessage); - var theLine = fullSource.GetLines()[line]; - var tab = " ".Multiply(line.ToString().Length); - sb.Append(tab).AppendLine(" |"); - sb.Append(line).Append(" |").AppendLine(theLine); - sb.Append($"{tab} |").Append(" ".Multiply(column)).Append("^^^").AppendLine($" {message}"); - + var allLines = fullSource.GetLines(); + if (line < allLines.Count) + { + var theLine = fullSource.GetLines()[line]; + var tab = " ".Multiply(line.ToString().Length); + sb.Append(tab).AppendLine(" |"); + sb.Append(line).Append(" |").AppendLine(theLine); + sb.Append($"{tab} |").Append(" ".Multiply(column)).Append("^^^").AppendLine($" {message}"); + } return sb.ToString(); } } diff --git a/src/sly/parser/parser/ParseResult.cs b/src/sly/parser/parser/ParseResult.cs index 82d19812..8759c2dd 100644 --- a/src/sly/parser/parser/ParseResult.cs +++ b/src/sly/parser/parser/ParseResult.cs @@ -1,8 +1,9 @@ -using System; +using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using System.Linq; using sly.parser.syntax.tree; +using sly.parser.generator.visitor; namespace sly.parser { @@ -10,19 +11,83 @@ public class ParseResult where IN : struct, Enum { public OUT Result { get; set; } - public ISyntaxNode SyntaxTree { get; set; } + /// + /// Syntax tree forest (for ambiguous grammars). + /// + public ParseForest Forest { get; set; } + + public ISyntaxNode SyntaxTree + { + get => Forest?.MainTree; + set + { + if (Forest == null) + Forest = new ParseForest { Trees = new List>() }; + if (Forest.Trees == null) + Forest.Trees = new List>(); + if (Forest.Trees.Count == 0) + Forest.Trees.Add(value); + else + Forest.Trees[0] = value; + } + } + + /// + /// True if parsing resulted in multiple valid syntax trees (i.e., the grammar is ambiguous for the given input). + /// + public bool IsAmbiguous => Forest?.IsAmbiguous ?? false; public bool IsError { get; set; } public bool IsOk => !IsError; public List Errors { get; set; } + + /// + /// Visit all syntax trees in the forest and return their results. + /// If the grammar is not ambiguous, returns a list with a single element (the result of the main tree). + /// + public List VisitAllTrees(SyntaxTreeVisitor visitor, object context = null) + { + if (!IsAmbiguous) + return new List { Result }; + + return Forest.Trees.Select(tree => visitor.VisitSyntaxTree(tree, context)).ToList(); + } + + /// + /// Access a specific syntax tree in the forest by index and visit it. + /// + public OUT SelectTree(int index, SyntaxTreeVisitor visitor, object context = null) + { + if (Forest == null || index >= Forest.Count) + throw new ArgumentOutOfRangeException(nameof(index)); + + return visitor.VisitSyntaxTree(Forest.Trees[index], context); + } + + /// + /// Resolve ambiguity by applying a user-defined selector function to choose one of the syntax trees in the forest, then visit the selected tree. + /// + public OUT ResolveAmbiguity(Func>, ISyntaxNode> selector, + SyntaxTreeVisitor visitor, object context = null) + { + if (!IsAmbiguous) + return Result; + + var selectedTree = selector(Forest.Trees); + return visitor.VisitSyntaxTree(selectedTree, context); + } [ExcludeFromCodeCoverage] public override string ToString() { if (IsOk) { + if (IsAmbiguous) + { + return $"parse OK (ambiguous: {Forest.Count} alternatives)."; + } return "parse OK."; } else diff --git a/src/sly/parser/parser/Parser.cs b/src/sly/parser/parser/Parser.cs index 608bd89e..870c3f61 100644 --- a/src/sly/parser/parser/Parser.cs +++ b/src/sly/parser/parser/Parser.cs @@ -1,5 +1,4 @@ - -using System; +using System; using System.Collections.Generic; using System.Linq; using sly.buildresult; @@ -112,40 +111,151 @@ public ParseResult ParseWithContext(IList> tokens, object par var result = new ParseResult(); var cleaner = new SyntaxTreeCleaner(); + if (SyntaxParser is sly.parser.llparser.bnf.RecursiveDescentSyntaxParser rdParser) + { + rdParser.Init(Configuration, startingNonTerminal ?? Configuration?.StartingRule); + if (Configuration != null) + { + rdParser.Configuration.CaptureAmbiguities = Configuration.CaptureAmbiguities; + rdParser.Configuration.AmbiguityStrategy = Configuration.AmbiguityStrategy; + } + } var syntaxResult = SyntaxParser.Parse(tokens.ToArray(), startingNonTerminal); + if (Configuration?.CaptureAmbiguities == true && !syntaxResult.HasAmbiguity + && SyntaxParser is sly.parser.llparser.bnf.RecursiveDescentSyntaxParser ambiguityParser) + { + var start = startingNonTerminal ?? ambiguityParser.StartingNonTerminal; + if (start != null && Configuration.NonTerminals.TryGetValue(start, out var nonTerminal)) + { + var ambiguityContext = new SyntaxParsingContext(Configuration.UseMemoization); + var alternativeResults = new List>(); + foreach (var rule in nonTerminal.Rules) + { + var canStart = tokens.Count == 0 + ? rule.MayBeEmpty + : (!tokens[0].IsEOS && rule.Match(tokens, 0, Configuration)) + || (tokens[0].IsEOS && rule.MayBeEmpty); + if (!canStart) + { + continue; + } + + var ruleResult = ambiguityParser.Parse(tokens.ToArray(), rule, 0, start, ambiguityContext); + if (ruleResult.IsEnded && !ruleResult.IsError) + { + alternativeResults.Add(ruleResult); + } + } + + if (alternativeResults.Count > 1) + { + syntaxResult = new SyntaxParseResult + { + AlternativeRoots = alternativeResults.Select(r => r.Root).ToList(), + EndingPosition = alternativeResults[0].EndingPosition, + IsError = false, + IsEnded = true, + HasByPassNodes = alternativeResults.Any(r => r.HasByPassNodes), + Ambiguities = new List> + { + new AmbiguityInfo + { + NonTerminalName = start, + Position = 0, + AlternativeCount = alternativeResults.Count + } + } + }; + } + } + } syntaxResult.UsesOperations = Configuration.UsesOperations; syntaxResult = cleaner.CleanSyntaxTree(syntaxResult); - if (!syntaxResult.IsError && syntaxResult.Root != null) + if (!syntaxResult.IsError && (syntaxResult.Root != null || syntaxResult.HasAmbiguity)) { - - var r = Visitor.VisitSyntaxTree(syntaxResult.Root,parsingContext ?? new NoContext()); - result.Result = r; - result.SyntaxTree = syntaxResult.Root; + // if there is ambiguity, we will have multiple roots in AlternativeRoots, and we will decide which one to visit according to the AmbiguityStrategy + if (syntaxResult.HasAmbiguity) + { + result.Forest = new syntax.tree.ParseForest + { + Trees = syntaxResult.AlternativeRoots, + Ambiguities = syntaxResult.Ambiguities + }; + + // Visit trees according to ambiguity strategy + switch (Configuration.AmbiguityStrategy) + { + case generator.AmbiguityResolutionStrategy.First: + result.Result = Visitor.VisitSyntaxTree(result.Forest.MainTree, parsingContext ?? new NoContext()); + break; + + case generator.AmbiguityResolutionStrategy.ThrowException: + throw new exceptions.AmbiguousGrammarException(result.Forest); + + case generator.AmbiguityResolutionStrategy.All: + // Do not set result.Result, let the user handle all alternatives in the forest + result.Result = default(OUT); + break; + + case generator.AmbiguityResolutionStrategy.Longest: + // Visit longest derivation (assuming MainTree is the longest, otherwise we would need to determine it) + result.Result = Visitor.VisitSyntaxTree(result.Forest.MainTree, parsingContext ?? new NoContext()); + break; + } + } + else + { + // no ambiguity, visit the single tree + var r = Visitor.VisitSyntaxTree(syntaxResult.Root, parsingContext ?? new NoContext()); + result.Result = r; + result.SyntaxTree = syntaxResult.Root; + } + result.IsError = false; } else { result.Errors = new List(); - var unexpectedTokens = syntaxResult.GetErrors(); - var byEnding = unexpectedTokens.GroupBy(x => x.UnexpectedToken.Position).OrderBy(x => x.Key); - var errors = new List(); - foreach (var expecting in byEnding) + var unexpectedTokens = syntaxResult.GetErrors() ?? new List>(); + var errors = new List(); + if (unexpectedTokens.Any()) { - var expectingTokens = expecting.SelectMany(x => x.ExpectedTokens ?? new List>()).Distinct(); - var expectedTokens = expectingTokens?.ToArray(); - if (expectedTokens != null) - { - var expected = new UnexpectedTokenSyntaxError(expecting.First().UnexpectedToken, LexemeLabels, I18n, - expectedTokens); - errors.Add(expected); - } - else + var targetPosition = unexpectedTokens.Max(x => x.UnexpectedToken.PositionInTokenFlow); + + var eosPositions = unexpectedTokens + .Where(x => x.UnexpectedToken.IsEOS) + .Select(x => x.UnexpectedToken.PositionInTokenFlow) + .ToList(); + if (eosPositions.Any()) { - var expected = new UnexpectedTokenSyntaxError(expecting.First().UnexpectedToken, LexemeLabels, I18n, - new LeadingToken[]{}); - errors.Add(expected); + var maxEosPosition = eosPositions.Max(); + if (maxEosPosition >= targetPosition) + { + targetPosition = maxEosPosition; + } } + + var targetErrors = unexpectedTokens + .Where(x => x.UnexpectedToken.PositionInTokenFlow == targetPosition) + .ToList(); + + var expectingTokens = targetErrors + .SelectMany(x => x.ExpectedTokens ?? new List>()) + .Distinct() + .ToArray(); + var expected = new UnexpectedTokenSyntaxError(targetErrors.First().UnexpectedToken, LexemeLabels, I18n, + expectingTokens); + errors.Add(expected); + } + else + { + var fallbackToken = tokens != null && tokens.Count > 0 + ? tokens[Math.Min(Math.Max(syntaxResult.EndingPosition, 0), tokens.Count - 1)] + : new Token { IsEOS = true }; + errors.Add(new UnexpectedTokenSyntaxError(fallbackToken, LexemeLabels, I18n, + Array.Empty>())); } + result.Errors.AddRange(errors); result.IsError = true; } diff --git a/src/sly/parser/parser/SyntaxParseResult.cs b/src/sly/parser/parser/SyntaxParseResult.cs index a2a7826e..0b8178fa 100644 --- a/src/sly/parser/parser/SyntaxParseResult.cs +++ b/src/sly/parser/parser/SyntaxParseResult.cs @@ -8,7 +8,43 @@ namespace sly.parser { public class SyntaxParseResult where IN : struct, Enum { - public ISyntaxNode Root { get; set; } + /// + /// List of alternative roots (for ambiguous grammars) + /// If null or 1 element, no ambiguity + /// + public List> AlternativeRoots { get; set; } + + /// + /// used for ambiguity capture to return results with different ending positions + /// + public List> AllResults { get; set; } + + /// + /// Indicates if this result contains ambiguous alternatives + /// + public bool HasAmbiguity => AlternativeRoots != null && AlternativeRoots.Count > 1; + + /// + /// Information about detected ambiguities + /// + public List> Ambiguities { get; set; } + + /// + /// Compatibility: Root points to the first alternative + /// + public ISyntaxNode Root + { + get => AlternativeRoots?.FirstOrDefault(); + set + { + if (AlternativeRoots == null) + AlternativeRoots = new List>(); + if (AlternativeRoots.Count == 0) + AlternativeRoots.Add(value); + else + AlternativeRoots[0] = value; + } + } public bool IsError { get; set; } @@ -45,7 +81,7 @@ public void AddError(UnexpectedTokenSyntaxError error) Errors.Add(error); } - public IList> GetErrors() => Errors?.ToList(); + public IList> GetErrors() => Errors?.ToList() ?? new List>(); public List> Expecting {get; set;} diff --git a/src/sly/parser/parser/SyntaxTreeCleaner.cs b/src/sly/parser/parser/SyntaxTreeCleaner.cs index 267b292a..1e94915b 100644 --- a/src/sly/parser/parser/SyntaxTreeCleaner.cs +++ b/src/sly/parser/parser/SyntaxTreeCleaner.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using sly.parser.syntax.tree; namespace sly.parser.parser @@ -8,11 +9,16 @@ public class SyntaxTreeCleaner where IN : struct, Enum { public SyntaxParseResult CleanSyntaxTree(SyntaxParseResult result) { - var tree = result.Root; - if (tree != null) + if (result.AlternativeRoots != null && result.AlternativeRoots.Any()) { - if (result.UsesOperations) tree = SetAssociativity(tree); - result.Root = tree; + for (int i = 0; i < result.AlternativeRoots.Count; i++) + { + var tree = result.AlternativeRoots[i]; + if (tree != null && result.UsesOperations) + { + result.AlternativeRoots[i] = SetAssociativity(tree); + } + } } return result; diff --git a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.NonTerminal.cs b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.NonTerminal.cs index d1095d3a..a30be977 100644 --- a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.NonTerminal.cs +++ b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.NonTerminal.cs @@ -28,13 +28,9 @@ public SyntaxParseResult ParseNonTerminal(Token[] tokens, string no var startPosition = currentPosition; var nt = Configuration.NonTerminals[nonTerminalName]; - var errors = new List>(); - var i = 0; var rules = nt.Rules; - var innerRuleErrors = new List>(); - var greaterIndex = 0; var rulesResults = new List>(); while (i < rules.Count) { @@ -45,19 +41,6 @@ public SyntaxParseResult ParseNonTerminal(Token[] tokens, string no { var innerRuleRes = Parse(tokens, innerrule, startPosition, nonTerminalName, parsingContext); rulesResults.Add(innerRuleRes); - - var other = greaterIndex == 0 && innerRuleRes.EndingPosition == 0; - var innerRuleResErrors = innerRuleRes.GetErrors(); - if (innerRuleRes.EndingPosition > greaterIndex && innerRuleResErrors != null && - innerRuleResErrors.Count == 0 || other) - { - greaterIndex = innerRuleRes.EndingPosition; - if (innerRuleResErrors != null) - innerRuleErrors.AddRange(innerRuleResErrors); - } - - if (innerRuleResErrors != null) - innerRuleErrors.AddRange(innerRuleResErrors); } i++; @@ -77,58 +60,152 @@ public SyntaxParseResult ParseNonTerminal(Token[] tokens, string no return noMatching; } - errors.AddRange(innerRuleErrors); - SyntaxParseResult max = null; - int okEndingPosition = -1; - int koEndingPosition = -1; - bool hasOk = false; - SyntaxParseResult maxOk = null; - SyntaxParseResult maxKo = null; - foreach (var rulesResult in rulesResults) + + var result = new SyntaxParseResult(); + + if (Configuration.CaptureAmbiguities) { - if (rulesResult.IsOk) + // filter only ok results + var okResults = rulesResults + .SelectMany(r => r.AllResults ?? new List> { r }) + .Where(r => r.IsOk) + .ToList(); + + if (okResults.Any()) { - hasOk = true; - if (rulesResult.EndingPosition > okEndingPosition) + result.AllResults = okResults; + + var maxLength = okResults.Max(r => r.EndingPosition); + var maxResults = okResults.Where(r => r.EndingPosition == maxLength).ToList(); + + if (maxResults.Count > 1 || maxResults.Any(r => r.HasAmbiguity)) + { + // ambiguity detected + result.AlternativeRoots = maxResults.SelectMany(r => r.AlternativeRoots).ToList(); + result.EndingPosition = maxLength; + result.IsError = false; + result.IsEnded = maxResults[0].IsEnded; + result.HasByPassNodes = maxResults.Any(r => r.HasByPassNodes); + + + // save ambiguity + if (result.Ambiguities == null) + result.Ambiguities = new List>(); + + if (maxResults.Count > 1) { + result.Ambiguities.Add(new sly.parser.syntax.tree.AmbiguityInfo + { + NonTerminalName = nonTerminalName, + Position = currentPosition, + AlternativeCount = maxResults.Count + }); + } + + foreach(var maxResult in maxResults) { + if (maxResult.Ambiguities != null) { + result.Ambiguities.AddRange(maxResult.Ambiguities); + } + } + } + else { - okEndingPosition = rulesResult.EndingPosition; - maxOk = rulesResult; + // no ambiguity, take the single best result + result.Root = maxResults[0].Root; + result.EndingPosition = maxResults[0].EndingPosition; + result.IsError = false; + result.IsEnded = maxResults[0].IsEnded; + result.HasByPassNodes = maxResults[0].HasByPassNodes; } } - - if (rulesResult.IsError && rulesResult.EndingPosition > koEndingPosition) + else { - koEndingPosition = rulesResult.EndingPosition; - maxKo = rulesResult; + // all results are errors, take the one that got the furthest + var koEndingPosition = rulesResults.Max(r => r.EndingPosition); + var maxKos = rulesResults.Where(r => r.EndingPosition == koEndingPosition).ToList(); + + var maxKo = maxKos[0]; + result.Root = maxKo.Root; + result.EndingPosition = maxKo.EndingPosition; + result.IsError = true; + result.IsEnded = maxKo.IsEnded; + result.HasByPassNodes = maxKo.HasByPassNodes; + + foreach(var ko in maxKos) { + result.AddErrors(ko.GetErrors().ToArray()); + } } } - - if (hasOk) - { - max = maxOk; - } else { - max = maxKo; - } - + // take the result that got the furthest, preferring ok results over errors + SyntaxParseResult max = null; + int okEndingPosition = -1; + int koEndingPosition = -1; + bool hasOk = false; + SyntaxParseResult maxOk = null; + SyntaxParseResult maxKo = null; + foreach (var rulesResult in rulesResults) + { + if (rulesResult.IsOk) + { + hasOk = true; + if (rulesResult.EndingPosition > okEndingPosition) + { + okEndingPosition = rulesResult.EndingPosition; + maxOk = rulesResult; + } + } - var result = new SyntaxParseResult(); - result.AddErrors(errors); - result.Root = max.Root; - result.EndingPosition = max.EndingPosition; - result.IsError = max.IsError; - result.IsEnded = max.IsEnded; - result.HasByPassNodes = max.HasByPassNodes; + if (rulesResult.IsError && rulesResult.EndingPosition > koEndingPosition) + { + koEndingPosition = rulesResult.EndingPosition; + maxKo = rulesResult; + } + } + if (hasOk) + { + max = maxOk; + } + else + { + max = maxKo; + } - List> terr = new List>(); - foreach (var ruleResult in rulesResults) - { - var ruleErrors = ruleResult.GetErrors(); - if (ruleErrors != null) + result.Root = max.Root; + result.EndingPosition = max.EndingPosition; + result.IsError = max.IsError; + result.IsEnded = max.IsEnded; + result.HasByPassNodes = max.HasByPassNodes; + if (hasOk) + { + result.AddErrors(max.GetErrors().ToArray()); + var errorCandidates = rulesResults + .Where(r => r.IsError) + .SelectMany(r => r.GetErrors()) + .ToList(); + if (errorCandidates.Any()) + { + var maxErrorPosition = errorCandidates.Max(e => e.UnexpectedToken.PositionInTokenFlow); + var maxErrors = errorCandidates + .Where(e => e.UnexpectedToken.PositionInTokenFlow == maxErrorPosition) + .ToList(); + result.AddErrors(maxErrors); + } + } + else if (koEndingPosition >= 0) + { + var maxKos = rulesResults + .Where(r => r.IsError && r.EndingPosition == koEndingPosition) + .ToList(); + foreach (var ko in maxKos) + { + result.AddErrors(ko.GetErrors().ToArray()); + } + } + else { - terr.AddRange(ruleErrors); + result.AddErrors(max.GetErrors().ToArray()); } } diff --git a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.Terminal.cs b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.Terminal.cs index 73f9593f..a67c5a6c 100644 --- a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.Terminal.cs +++ b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.Terminal.cs @@ -19,6 +19,20 @@ public SyntaxParseResult ParseTerminal(Token[] tokens, TerminalClau } var result = new SyntaxParseResult(); + + // Check if position is within bounds + if (position >= tokens.Length) + { + result.IsError = true; + result.EndingPosition = position; + var eosToken = new Token { IsEOS = true }; + result.Root = new SyntaxLeaf(eosToken, terminal.Discarded); + result.HasByPassNodes = false; + result.AddError(new UnexpectedTokenSyntaxError(eosToken, LexemeLabels, I18n, terminal.ExpectedToken)); + parsingContext.Memoize(terminal, position, result); + return result; + } + result.IsError = !terminal.Check(tokens[position]); result.EndingPosition = !result.IsError ? position + 1 : position; var token = tokens[position]; diff --git a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.cs b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.cs index 5c669593..cacc0b82 100644 --- a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.cs +++ b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParser.cs @@ -13,7 +13,7 @@ public partial class RecursiveDescentSyntaxParser where IN : struct, En public RecursiveDescentSyntaxParser(ParserConfiguration configuration, string startingNonTerminal, string i18n) { - + I18n = i18n; Configuration = configuration; StartingNonTerminal = startingNonTerminal; @@ -27,10 +27,12 @@ public RecursiveDescentSyntaxParser(ParserConfiguration configuration, public SyntaxParseResult Parse(Token[] tokens, string startingNonTerminal = null) { - return SafeParse(tokens, new SyntaxParsingContext(Configuration.UseMemoization), startingNonTerminal); + return SafeParse(tokens, new SyntaxParsingContext(Configuration.UseMemoization), + startingNonTerminal); } - - public SyntaxParseResult SafeParse(Token[] tokens, SyntaxParsingContext parsingContext, string startingNonTerminal = null) + + public SyntaxParseResult SafeParse(Token[] tokens, SyntaxParsingContext parsingContext, + string startingNonTerminal = null) { var start = startingNonTerminal ?? StartingNonTerminal; var NonTerminals = Configuration.NonTerminals; @@ -43,7 +45,7 @@ public SyntaxParseResult SafeParse(Token[] tokens, SyntaxParsingCon foreach (var rule in nt.Rules) { - if (!tokens[0].IsEOS && rule.Match(tokens,0,Configuration) + if (!tokens[0].IsEOS && rule.Match(tokens, 0, Configuration) || tokens[0].IsEOS && rule.MayBeEmpty) { matchingRuleCount++; @@ -63,9 +65,60 @@ public SyntaxParseResult SafeParse(Token[] tokens, SyntaxParsingCon if (rs.Count > 0) { - result = rs.Find(r => r.IsEnded && !r.IsError); - - + // Check for ambiguities at root level + var successfulResults = rs.SelectMany(r => r.AllResults ?? new List> { r }) + .Where(r => r.IsEnded && !r.IsError).ToList(); + + + if (Configuration.CaptureAmbiguities && + (successfulResults.Count > 1 || successfulResults.Any(r => r.HasAmbiguity))) + { + // Ambiguity detected at root level + result = new SyntaxParseResult + { + AlternativeRoots = successfulResults.SelectMany(r => r.AlternativeRoots).ToList(), + EndingPosition = successfulResults.Max(r => r.EndingPosition), + IsError = false, + IsEnded = true, + HasByPassNodes = successfulResults.Any(r => r.HasByPassNodes) + }; + + if (result.Ambiguities == null) + result.Ambiguities = new List>(); + + if (successfulResults.Count > 1) + { + result.Ambiguities.Add(new AmbiguityInfo + { + NonTerminalName = start, + Position = 0, + AlternativeCount = successfulResults.Count + }); + } + + foreach (var success in successfulResults) + { + if (success.Ambiguities != null) + { + result.Ambiguities.AddRange(success.Ambiguities); + } + } + } + else if (successfulResults.Count > 0) + { + result = successfulResults[0]; + } + else + { + // No successful results, try to find any completed parse + result = rs.FirstOrDefault(r => r.IsEnded); + if (result == null) + { + result = rs.FirstOrDefault(); + } + } + + if (result == null) { int lastPosition = -1; @@ -94,14 +147,35 @@ public SyntaxParseResult SafeParse(Token[] tokens, SyntaxParsingCon } else { - if (result.EndingPosition < tokens.Length-1) + if (result.EndingPosition < tokens.Length - 1) { SyntaxParseResult r = new SyntaxParseResult() { IsError = true, - IsEnded = false + IsEnded = false, + EndingPosition = result.EndingPosition }; - r.AddErrors(result.GetErrors().ToArray()); + var allErrors = rs.SelectMany(x => x.GetErrors() ?? new List>()) + .ToList(); + if (allErrors.Any()) + { + var maxPosition = allErrors.Max(e => e.UnexpectedToken.PositionInTokenFlow); + var maxErrors = allErrors + .Where(e => e.UnexpectedToken.PositionInTokenFlow == maxPosition) + .ToList(); + r.AddErrors(maxErrors); + } + else + { + r.AddErrors(result.GetErrors().ToArray()); + if (r.GetErrors() == null || r.GetErrors().Count == 0) + { + var unexpectedToken = tokens[result.EndingPosition]; + r.AddError(new UnexpectedTokenSyntaxError(unexpectedToken, LexemeLabels, I18n, + Array.Empty>())); + } + } + return r; } } @@ -137,77 +211,141 @@ public SyntaxParseResult SafeParse(Token[] tokens, SyntaxParsingCon public virtual SyntaxParseResult Parse(Token[] tokens, Rule rule, int position, string nonTerminalName, SyntaxParsingContext parsingContext) { - var currentPosition = position; var errors = new List>(); var isError = false; - var children = new List>(); - if (!tokens[position].IsEOS && rule.Match(tokens, position, Configuration) && rule.Clauses is { Count: > 0 }) + var paths = new List<(List> children, int position)> + { (new List>(), position) }; + var ambiguities = new List>(); + var furthestPosition = position; + + if (position < tokens.Length && !tokens[position].IsEOS && rule.Match(tokens, position, Configuration) && + rule.Clauses is { Count: > 0 }) { - children = new List>(); foreach (var clause in rule.Clauses) { - switch (clause) + var newPaths = new List<(List> children, int position)>(); + foreach (var path in paths) { - case TerminalClause terminalClause: + SyntaxParseResult clauseRes = null; + switch (clause) { - var termRes = ParseTerminal(tokens, terminalClause, currentPosition, parsingContext); - if (!termRes.IsError) - { - children.Add(termRes.Root); - currentPosition = termRes.EndingPosition; - } - else + case TerminalClause terminalClause: + clauseRes = ParseTerminal(tokens, terminalClause, path.position, parsingContext); + break; + case NonTerminalClause nonTerminalClause: + clauseRes = ParseNonTerminal(tokens, nonTerminalClause, path.position, parsingContext); + break; + } + + if (clauseRes != null && !clauseRes.IsError) + { + if (clauseRes.GetErrors() != null && clauseRes.GetErrors().Count > 0) { - var tok = tokens[currentPosition]; - errors.Add(new UnexpectedTokenSyntaxError(tok, LexemeLabels, I18n, - terminalClause.ExpectedToken)); + errors.AddRange(clauseRes.GetErrors()); } - isError = termRes.IsError; - break; - } - case NonTerminalClause terminalClause: - { - var nonTerminalResult = - ParseNonTerminal(tokens, terminalClause, currentPosition, parsingContext); - var ntErrors = nonTerminalResult.GetErrors(); - if (!nonTerminalResult.IsError) + var resultsToProcess = new List>(); + if (Configuration.CaptureAmbiguities && clauseRes.AllResults != null) { - children.Add(nonTerminalResult.Root); - currentPosition = nonTerminalResult.EndingPosition; - if (ntErrors != null && ntErrors.Count > 0) - errors.AddRange(ntErrors); + resultsToProcess.AddRange(clauseRes.AllResults); } else { - errors.AddRange(ntErrors); + resultsToProcess.Add(clauseRes); } - isError = nonTerminalResult.IsError; - break; + foreach (var res in resultsToProcess) + { + foreach (var alt in res.AlternativeRoots) + { + var newCombo = new List>(path.children); + newCombo.Add(alt); + newPaths.Add((newCombo, res.EndingPosition)); + } + + if (res.Ambiguities != null) + { + ambiguities.AddRange(res.Ambiguities); + } + } + } + else if (clauseRes != null) + { + errors.AddRange(clauseRes.GetErrors()); } } - if (isError) break; + if (newPaths.Any()) + { + furthestPosition = newPaths.Max(p => p.position); + } + + paths = newPaths; + if (paths.Count == 0) + { + isError = true; + break; + } } } + else + { + isError = true; + } var result = new SyntaxParseResult(); result.IsError = isError; + result.EndingPosition = furthestPosition; result.AddErrors(errors); - result.EndingPosition = currentPosition; + if (!isError) { - SyntaxNode node = null; - if (rule.IsSubRule) - node = new GroupSyntaxNode(nonTerminalName, children); + var resultsByPosition = paths.GroupBy(p => p.position).ToList(); + var results = new List>(); + + foreach (var group in resultsByPosition) + { + var pos = group.Key; + var res = new SyntaxParseResult(); + res.EndingPosition = pos; + res.IsError = false; + res.IsEnded = pos >= tokens.Length || tokens[pos].IsEOS; + res.Ambiguities = ambiguities; + res.AlternativeRoots = new List>(); + foreach (var path in group) + { + SyntaxNode node = null; + if (rule.IsSubRule) + node = new GroupSyntaxNode(nonTerminalName, path.children); + else + node = new SyntaxNode(rule.NodeName ?? nonTerminalName, path.children); + node = ManageExpressionRules(rule, node); + if (node.IsByPassNode) + { + res.AlternativeRoots.Add(path.children[0]); + } + else + { + res.AlternativeRoots.Add(node); + } + } + + results.Add(res); + } + + if (results.Count > 0) + { + var best = results.OrderByDescending(r => r.EndingPosition).First(); + result.AlternativeRoots = best.AlternativeRoots; + result.EndingPosition = best.EndingPosition; + result.IsEnded = best.IsEnded; + result.AllResults = results; + result.Ambiguities = ambiguities; + } else - node = new SyntaxNode(rule.NodeName ?? nonTerminalName, children); - node = ManageExpressionRules(rule, node); - if (node.IsByPassNode) // inutile de créer un niveau supplémentaire - result.Root = children[0]; - result.Root = node; - result.IsEnded = tokens[result.EndingPosition].IsEOS; + { + result.IsError = true; + } } return result; @@ -244,16 +382,18 @@ private SyntaxParseResult NoMatchingRuleError(Token[] tokens, int c } - + public virtual void Init(ParserConfiguration configuration, string root) { if (root != null) StartingNonTerminal = root; + Configuration = configuration; // #540 : reset all leading tokens and recompute with the new configuration (expression rules) foreach (var nonTerminal in configuration.NonTerminals.Values) { nonTerminal?.Rules?.ForEach(x => x?.PossibleLeadingTokens?.Clear()); } + InitializeStartingTokens(configuration, StartingNonTerminal); } diff --git a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParserStarter.cs b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParserStarter.cs index bd0c9d0b..30439826 100644 --- a/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParserStarter.cs +++ b/src/sly/parser/parser/llparser/bnf/RecursiveDescentSyntaxParserStarter.cs @@ -166,7 +166,7 @@ protected void InitStartingTokensForRule(Dictionary break; } default: - InitStartingTokensForRuleExtensions(first, rule, nonTerminals); + InitStartingTokensForRuleExtensions(c, rule, nonTerminals); break; } } diff --git a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.EBNFRecursiveDescentSyntaxParser.Choice.cs b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.EBNFRecursiveDescentSyntaxParser.Choice.cs index 873a2370..0991ef19 100644 --- a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.EBNFRecursiveDescentSyntaxParser.Choice.cs +++ b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.EBNFRecursiveDescentSyntaxParser.Choice.cs @@ -21,16 +21,9 @@ public SyntaxParseResult ParseChoice(Token[] tokens, ChoiceClause result = new SyntaxParseResult - { - IsError = true, - IsEnded = false, - EndingPosition = currentPosition - }; - List> alternateResults = new List>(); + // Collect all results for all alternatives, even if CaptureAmbiguities is false (to determine the longest match in case of failure) foreach (var alternate in clause.Choices) { switch (alternate) @@ -47,21 +40,105 @@ public SyntaxParseResult ParseChoice(Token[] tokens, ChoiceClause leaf) + { + var discardedToken = new SyntaxLeaf(leaf.Token, true); + lastResult.Root = discardedToken; + } + parsingContext.Memoize(clause, position, lastResult); + return lastResult; + } + } + } + + // manage ambiguities. + SyntaxParseResult result; + + if (Configuration.CaptureAmbiguities) + { + var okResults = alternateResults.Where(r => r.IsOk).ToList(); + + if (okResults.Any()) { + // Toutes les alternatives réussies à la même position finale + var maxLength = okResults.Max(r => r.EndingPosition); + var maxResults = okResults.Where(r => r.EndingPosition == maxLength).ToList(); + + if (maxResults.Count > 1 || maxResults.Any(r => r.HasAmbiguity)) + { + // AMBIGUÏTÉ dans le choix EBNF + result = new SyntaxParseResult + { + AlternativeRoots = maxResults.SelectMany(r => r.AlternativeRoots).ToList(), + EndingPosition = maxLength, + IsError = false, + AllResults = okResults + }; + + // save ambiguity + result.Ambiguities = new List>(); + + if (maxResults.Count > 1) { + result.Ambiguities.Add(new sly.parser.syntax.tree.AmbiguityInfo + { + NonTerminalName = $"Choice[{string.Join("|", clause.Choices.Select((c, i) => $"alt{i}"))}]", + Position = position, + AlternativeCount = maxResults.Count + }); + } + + foreach(var maxResult in maxResults) { + if (maxResult.Ambiguities != null) { + result.Ambiguities.AddRange(maxResult.Ambiguities); + } + } + } + else + { + result = maxResults[0]; + result.AllResults = okResults; + } + if (clause.IsTerminalChoice && clause.IsDiscarded && result.Root is SyntaxLeaf leaf) { var discardedToken = new SyntaxLeaf(leaf.Token, true); result.Root = discardedToken; } - - parsingContext.Memoize(clause, position, result); - return result; + } + else + { + // all alternatives failed, return the one with the longest match (to provide the most accurate error message) + result = HandleAllChoicesFailed(clause, tokens, currentPosition, alternateResults); } } + else + { + result = HandleAllChoicesFailed(clause, tokens, currentPosition, alternateResults); + } - // here all alternateResult ar KO + parsingContext.Memoize(clause, position, result); + return result; + } + + private SyntaxParseResult HandleAllChoicesFailed( + ChoiceClause clause, + Token[] tokens, + int currentPosition, + List> alternateResults) + { + var result = new SyntaxParseResult + { + IsError = true, + IsEnded = false, + EndingPosition = currentPosition + }; + if (clause.IsTerminalChoice) { var terminalAlternates = clause.Choices.Cast>(); @@ -72,12 +149,11 @@ public SyntaxParseResult ParseChoice(Token[] tokens, ChoiceClause x.EndingPosition).Max(); - var errors= alternateResults.Where(x => x.EndingPosition == greaterPosition).SelectMany(x => x.GetErrors()).ToList(); + var errors = alternateResults.Where(x => x.EndingPosition == greaterPosition) + .SelectMany(x => x.GetErrors()).ToList(); result.AddErrors(errors); - result.IsError = true; } - - parsingContext.Memoize(clause, position, result); + return result; } diff --git a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Many.cs b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Many.cs index 108f6ab0..85a728d9 100644 --- a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Many.cs +++ b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Many.cs @@ -19,6 +19,67 @@ public SyntaxParseResult ParseZeroOrMore(Token[] tokens, ZeroOrMore return parseResult; } + if (Configuration.CaptureAmbiguities) + { + var innerClauseAmbi = clause.Clause; + var paths = new List<(List> children, int position, bool hasByPassNodes)> + { + (new List>(), position, false) + }; + var innerErrorsAmbi = new List>(); + var (isManyTokens, isManyValues, isManyGroups) = GetManyNodeFlags(innerClauseAmbi); + + while (true) + { + var newPaths = new List<(List> children, int position, bool hasByPassNodes)>(); + foreach (var path in paths) + { + var innerResult = ParseSingleIterationAmbiguous(tokens, innerClauseAmbi, path.position, parsingContext); + + if (innerResult != null && !innerResult.IsError && innerResult.EndingPosition > path.position) + { + var resultsToProcess = innerResult.AllResults ?? new List> { innerResult }; + foreach (var res in resultsToProcess) + { + var alternatives = res.AlternativeRoots; + if (alternatives == null || alternatives.Count == 0) + { + alternatives = new List> { res.Root }; + } + + foreach (var alt in alternatives) + { + var newChildren = new List>(path.children) { alt }; + var newHasByPassNodes = path.hasByPassNodes || res.HasByPassNodes; + newPaths.Add((newChildren, res.EndingPosition, newHasByPassNodes)); + } + + if (res.GetErrors() != null) + { + innerErrorsAmbi.AddRange(res.GetErrors()); + } + } + } + else if (innerResult != null) + { + innerErrorsAmbi.AddRange(innerResult.GetErrors()); + } + } + + if (newPaths.Count == 0) + { + break; + } + + paths = newPaths; + } + + var ambiguousResult = BuildAmbiguousResult(paths, innerClauseAmbi, isManyTokens, isManyValues, + isManyGroups, "*", tokens, innerErrorsAmbi, position); + parsingContext.Memoize(clause, position, ambiguousResult); + return ambiguousResult; + } + var result = new SyntaxParseResult(); var manyNode = new ManySyntaxNode($"{clause.Clause.ToString()}*"); var currentPosition = position; @@ -198,6 +259,94 @@ public SyntaxParseResult ParseOneOrMore(Token[] tokens, OneOrMoreCl return parseResult; } + if (Configuration.CaptureAmbiguities) + { + var innerClauseAmbi = clause.Clause; + var innerErrorsAmbi = new List>(); + var paths = new List<(List> children, int position, bool hasByPassNodes)>(); + var (isManyTokens, isManyValues, isManyGroups) = GetManyNodeFlags(innerClauseAmbi); + + var firstResult = ParseSingleIterationAmbiguous(tokens, innerClauseAmbi, position, parsingContext); + + if (firstResult == null || firstResult.IsError) + { + var errorResult = new SyntaxParseResult + { + IsError = true, + EndingPosition = position + }; + if (firstResult != null) + { + errorResult.AddErrors(firstResult.GetErrors()); + } + parsingContext.Memoize(clause, position, errorResult); + return errorResult; + } + + var firstResults = firstResult.AllResults ?? new List> { firstResult }; + foreach (var res in firstResults) + { + if (res.GetErrors() != null) + { + innerErrorsAmbi.AddRange(res.GetErrors()); + } + + var alternatives = res.AlternativeRoots; + if (alternatives == null || alternatives.Count == 0) + { + alternatives = new List> { res.Root }; + } + + foreach (var alt in alternatives) + { + var baseChildren = new List> { alt }; + var more = new ZeroOrMoreClause(innerClauseAmbi); + var moreResult = ParseZeroOrMore(tokens, more, res.EndingPosition, parsingContext); + var moreResults = moreResult?.AllResults ?? new List> { moreResult }; + + foreach (var moreRes in moreResults) + { + if (moreRes == null || moreRes.IsError) + { + continue; + } + + var manyAlternatives = moreRes.AlternativeRoots; + if (manyAlternatives == null || manyAlternatives.Count == 0) + { + manyAlternatives = new List> { moreRes.Root }; + } + + foreach (var manyAlt in manyAlternatives) + { + var combinedChildren = new List>(baseChildren); + if (manyAlt is ManySyntaxNode manyNodeAlt) + { + combinedChildren.AddRange(manyNodeAlt.Children); + } + else + { + combinedChildren.Add(manyAlt); + } + + var combinedByPass = res.HasByPassNodes || moreRes.HasByPassNodes; + paths.Add((combinedChildren, moreRes.EndingPosition, combinedByPass)); + } + + if (moreRes.GetErrors() != null) + { + innerErrorsAmbi.AddRange(moreRes.GetErrors()); + } + } + } + } + + var ambiguousResult = BuildAmbiguousResult(paths, innerClauseAmbi, isManyTokens, isManyValues, + isManyGroups, "+", tokens, innerErrorsAmbi, position); + parsingContext.Memoize(clause, position, ambiguousResult); + return ambiguousResult; + } + var result = new SyntaxParseResult(); var manyNode = new ManySyntaxNode($"{clause.Clause.ToString()}+"); var currentPosition = position; @@ -275,4 +424,105 @@ public SyntaxParseResult ParseOneOrMore(Token[] tokens, OneOrMoreCl } #endregion + + #region ambiguity helpers + + private (bool isManyTokens, bool isManyValues, bool isManyGroups) GetManyNodeFlags(IClause clause) + { + bool isManyTokens = false; + bool isManyValues = false; + bool isManyGroups = false; + + if (clause is TerminalClause) + { + isManyTokens = true; + } + else if (clause is NonTerminalClause nonTermClause) + { + isManyGroups = nonTermClause.IsGroup; + isManyValues = !nonTermClause.IsGroup; + } + else if (clause is ChoiceClause choiceClause) + { + isManyTokens = choiceClause.IsTerminalChoice; + isManyValues = choiceClause.IsNonTerminalChoice; + } + + return (isManyTokens, isManyValues, isManyGroups); + } + + private SyntaxParseResult ParseSingleIterationAmbiguous(Token[] tokens, IClause clause, + int position, SyntaxParsingContext parsingContext) + { + return clause switch + { + TerminalClause term => ParseTerminal(tokens, term, position, parsingContext), + NonTerminalClause nonTerm => ParseNonTerminal(tokens, nonTerm, position, parsingContext), + ChoiceClause choice => ParseChoice(tokens, choice, position, parsingContext), + _ => throw new InvalidOperationException("unable to apply repeater to " + clause.GetType().Name) + }; + } + + private SyntaxParseResult BuildAmbiguousResult( + List<(List> children, int position, bool hasByPassNodes)> paths, + IClause innerClause, + bool isManyTokens, + bool isManyValues, + bool isManyGroups, + string suffix, + Token[] tokens, + List> errors, + int fallbackPosition) + { + var ambiguousResult = new SyntaxParseResult(); + var resultsByPosition = paths.GroupBy(p => p.position).ToList(); + var results = new List>(); + + foreach (var group in resultsByPosition) + { + var pos = group.Key; + var res = new SyntaxParseResult + { + EndingPosition = pos, + IsError = false, + AlternativeRoots = new List>(), + HasByPassNodes = group.Any(p => p.hasByPassNodes) + }; + + foreach (var path in group) + { + var manyNodeAlt = new ManySyntaxNode($"{innerClause}{suffix}") + { + IsManyTokens = isManyTokens, + IsManyValues = isManyValues, + IsManyGroups = isManyGroups + }; + manyNodeAlt.Children.AddRange(path.children); + res.AlternativeRoots.Add(manyNodeAlt); + } + + res.IsEnded = pos >= tokens.Length || (pos < tokens.Length && tokens[pos].IsEOS); + results.Add(res); + } + + if (results.Count > 0) + { + var best = results.OrderByDescending(r => r.EndingPosition).First(); + ambiguousResult.AlternativeRoots = best.AlternativeRoots; + ambiguousResult.EndingPosition = best.EndingPosition; + ambiguousResult.IsEnded = best.IsEnded; + ambiguousResult.HasByPassNodes = best.HasByPassNodes; + ambiguousResult.AllResults = results; + } + else + { + ambiguousResult.IsError = true; + ambiguousResult.EndingPosition = fallbackPosition; + } + + ambiguousResult.AddErrors(errors); + return ambiguousResult; + } + + #endregion } \ No newline at end of file diff --git a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Option.cs b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Option.cs index 20c2656b..7b5b4388 100644 --- a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Option.cs +++ b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.Option.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Linq; using sly.lexer; using sly.parser.syntax.grammar; using sly.parser.syntax.tree; @@ -18,6 +19,104 @@ public SyntaxParseResult ParseOption(Token[] tokens, OptionClause innerResultAmbi = null; + + switch (innerClauseAmbi) + { + case TerminalClause term: + innerResultAmbi = ParseTerminal(tokens, term, position, parsingContext); + break; + case NonTerminalClause nonTerm: + innerResultAmbi = ParseNonTerminal(tokens, nonTerm, position, parsingContext); + break; + case ChoiceClause choice: + innerResultAmbi = ParseChoice(tokens, choice, position, parsingContext); + break; + default: + throw new InvalidOperationException("unable to apply repeater to " + innerClauseAmbi.GetType().Name); + } + + var resultAmbi = new SyntaxParseResult + { + IsError = false, + EndingPosition = position, + AlternativeRoots = new List>(), + AllResults = new List>() + }; + + var emptyNode = new OptionSyntaxNode(rule.NonTerminalName, new List>(), + rule.GetVisitorMethod()) + { + IsGroupOption = clause.IsGroupOption + }; + + var emptyResult = new SyntaxParseResult + { + IsError = false, + Root = emptyNode, + EndingPosition = position, + IsEnded = position >= tokens.Length || (position < tokens.Length && tokens[position].IsEOS) + }; + resultAmbi.AllResults.Add(emptyResult); + + if (innerResultAmbi != null && !innerResultAmbi.IsError) + { + var innerResults = innerResultAmbi.AllResults ?? new List> { innerResultAmbi }; + foreach (var inner in innerResults) + { + var alternatives = inner.AlternativeRoots; + if (alternatives == null || alternatives.Count == 0) + { + alternatives = new List> { inner.Root }; + } + + foreach (var alt in alternatives) + { + var optionNode = new OptionSyntaxNode(rule.NonTerminalName, + new List> { alt }, rule.GetVisitorMethod()) + { + IsGroupOption = clause.IsGroupOption + }; + + var optionResult = new SyntaxParseResult + { + IsError = false, + Root = optionNode, + EndingPosition = inner.EndingPosition, + IsEnded = inner.IsEnded, + HasByPassNodes = inner.HasByPassNodes + }; + resultAmbi.AllResults.Add(optionResult); + } + } + + if (innerResultAmbi.GetErrors() != null) + { + resultAmbi.AddErrors(innerResultAmbi.GetErrors()); + } + } + else if (innerResultAmbi != null) + { + resultAmbi.AddErrors(innerResultAmbi.GetErrors()); + } + + var best = resultAmbi.AllResults.OrderBy(r => r.EndingPosition).Reverse().FirstOrDefault(); + if (best != null) + { + resultAmbi.Root = best.Root; + resultAmbi.EndingPosition = best.EndingPosition; + resultAmbi.IsEnded = best.IsEnded; + resultAmbi.HasByPassNodes = best.HasByPassNodes; + resultAmbi.AlternativeRoots = best.AlternativeRoots; + } + + parsingContext.Memoize(clause, position, resultAmbi); + return resultAmbi; + } + var result = new SyntaxParseResult(); var currentPosition = position; var innerClause = clause.Clause; diff --git a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.cs b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.cs index 17789a62..f1c1ea0c 100644 --- a/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.cs +++ b/src/sly/parser/parser/llparser/ebnf/EBNFRecursiveDescentSyntaxParser.cs @@ -26,11 +26,20 @@ public override SyntaxParseResult Parse(Token[] tokens, Rule>(); var isError = false; var children = new List>(); + List> choiceAlternatives = null; + int choiceChildIndex = -1; + List> choiceAmbiguities = null; if (rule.Match(tokens, position, Configuration) && rule.Clauses != null && rule.Clauses.Count > 0) { children = new List>(); @@ -53,8 +62,8 @@ public override SyntaxParseResult Parse(Token[] tokens, Rule(tok, LexemeLabels, I18n, termClause.ExpectedToken)); } - isError = isError || termRes.IsError; + furthestPosition = Math.Max(furthestPosition, termRes.EndingPosition); break; } case NonTerminalClause nonTerminalClause: @@ -71,8 +80,8 @@ public override SyntaxParseResult Parse(Token[] tokens, Rule _: @@ -104,8 +113,8 @@ public override SyntaxParseResult Parse(Token[] tokens, Rule 0) errors.AddRange(manyResult.GetErrors()); } - isError = manyResult.IsError; + furthestPosition = Math.Max(furthestPosition, manyResult.EndingPosition); break; } case OptionClause option: @@ -113,6 +122,7 @@ public override SyntaxParseResult Parse(Token[] tokens, Rule choice: @@ -123,10 +133,19 @@ public override SyntaxParseResult Parse(Token[] tokens, Rule>(); + choiceAmbiguities.AddRange(choiceResult.Ambiguities); + } + } children.Add(choiceResult.Root); + furthestPosition = Math.Max(furthestPosition, choiceResult.EndingPosition); break; } } @@ -138,7 +157,7 @@ public override SyntaxParseResult Parse(Token[] tokens, Rule(); result.IsError = isError; result.AddErrors(errors); - result.EndingPosition = currentPosition; + result.EndingPosition = furthestPosition; if (!isError) { SyntaxNode node = null; @@ -174,6 +193,248 @@ public override SyntaxParseResult Parse(Token[] tokens, Rule 1) + { + var alternatives = new List>(); + foreach (var alternative in choiceAlternatives) + { + var altChildren = new List>(children); + if (choiceChildIndex >= 0 && choiceChildIndex < altChildren.Count) + { + altChildren[choiceChildIndex] = alternative; + } + + SyntaxNode altNode; + if (rule.IsSubRule) + { + altNode = new GroupSyntaxNode(nonTerminalName, altChildren); + altNode = ManageExpressionRules(rule, altNode); + } + else + { + altNode = new SyntaxNode(nonTerminalName, altChildren); + altNode.ForcedName = rule.ForcedName; + altNode.Name = string.IsNullOrEmpty(rule.NodeName) ? nonTerminalName : rule.NodeName; + altNode.ExpressionAffix = rule.ExpressionAffix; + altNode = ManageExpressionRules(rule, altNode); + } + + alternatives.Add(altNode); + } + + result.AlternativeRoots = alternatives; + result.Ambiguities = choiceAmbiguities; + } + } + + return result; + } + + private SyntaxParseResult ParseWithAmbiguities(Token[] tokens, Rule rule, int position, + string nonTerminalName, SyntaxParsingContext parsingContext) + { + var furthestPosition = position; + var errors = new List>(); + var isError = false; + var ambiguities = new List>(); + var paths = new List<(List> children, int position, bool hasByPassNodes)> + { + (new List>(), position, false) + }; + + if (rule.Match(tokens, position, Configuration) && rule.Clauses != null && rule.Clauses.Count > 0) + { + foreach (var clause in rule.Clauses) + { + var newPaths = new List<(List> children, int position, bool hasByPassNodes)>(); + foreach (var path in paths) + { + SyntaxParseResult clauseRes = null; + switch (clause) + { + case TerminalClause termClause: + clauseRes = ParseTerminal(tokens, termClause, path.position, parsingContext); + break; + case NonTerminalClause nonTerminalClause: + clauseRes = ParseNonTerminal(tokens, nonTerminalClause, path.position, parsingContext); + break; + case OneOrMoreClause oneOrMore: + clauseRes = ParseOneOrMore(tokens, oneOrMore, path.position, parsingContext); + break; + case ZeroOrMoreClause zeroOrMore: + clauseRes = ParseZeroOrMore(tokens, zeroOrMore, path.position, parsingContext); + break; + case RepeatClause repeat: + clauseRes = ParseRepeat(tokens, repeat, path.position, parsingContext); + break; + case OptionClause option: + clauseRes = ParseOption(tokens, option, rule, path.position, parsingContext); + break; + case ChoiceClause choice: + clauseRes = ParseChoice(tokens, choice, path.position, parsingContext); + break; + } + + if (clauseRes != null && !clauseRes.IsError) + { + if (clauseRes.GetErrors() != null && clauseRes.GetErrors().Count > 0) + { + errors.AddRange(clauseRes.GetErrors()); + } + + var resultsToProcess = new List>(); + if (clauseRes.AllResults != null && clauseRes.AllResults.Count > 0) + { + resultsToProcess.AddRange(clauseRes.AllResults); + } + else + { + resultsToProcess.Add(clauseRes); + } + + foreach (var res in resultsToProcess) + { + var alternatives = res.AlternativeRoots; + if (alternatives == null || alternatives.Count == 0) + { + alternatives = new List> { res.Root }; + } + + foreach (var alt in alternatives) + { + var newChildren = new List>(path.children) { alt }; + var newHasByPassNodes = path.hasByPassNodes || res.HasByPassNodes; + newPaths.Add((newChildren, res.EndingPosition, newHasByPassNodes)); + } + + if (res.Ambiguities != null) + { + ambiguities.AddRange(res.Ambiguities); + } + } + } + else if (clauseRes != null) + { + errors.AddRange(clauseRes.GetErrors()); + } + } + + if (newPaths.Any()) + { + furthestPosition = Math.Max(furthestPosition, newPaths.Max(p => p.position)); + } + + paths = newPaths; + if (paths.Count == 0) + { + isError = true; + break; + } + } + } + else + { + isError = true; + } + + var result = new SyntaxParseResult(); + result.IsError = isError; + result.EndingPosition = furthestPosition; + result.AddErrors(errors); + + if (!isError) + { + var resultsByPosition = paths.GroupBy(p => p.position).ToList(); + var results = new List>(); + + foreach (var group in resultsByPosition) + { + var pos = group.Key; + var res = new SyntaxParseResult + { + EndingPosition = pos, + IsError = false, + Ambiguities = ambiguities, + AlternativeRoots = new List>(), + HasByPassNodes = group.Any(p => p.hasByPassNodes) + }; + + foreach (var path in group) + { + var namedChildren = new List>(path.children); + if (rule.SubNodeNames != null && rule.SubNodeNames.Length > 0) + { + for (int i = 0; i < Math.Min(rule.SubNodeNames.Length, namedChildren.Count); i++) + { + var subNodeName = rule.SubNodeNames[i]; + if (subNodeName != null) + { + namedChildren[i].ForceName(subNodeName); + } + } + } + + SyntaxNode node; + if (rule.IsSubRule) + { + node = new GroupSyntaxNode(nonTerminalName, namedChildren); + node = ManageExpressionRules(rule, node); + } + else + { + node = new SyntaxNode(nonTerminalName, namedChildren); + node.ForcedName = rule.ForcedName; + node.Name = string.IsNullOrEmpty(rule.NodeName) ? nonTerminalName : rule.NodeName; + node.ExpressionAffix = rule.ExpressionAffix; + node = ManageExpressionRules(rule, node); + } + + if (node.IsByPassNode) + { + res.AlternativeRoots.Add(namedChildren[0]); + } + else + { + res.AlternativeRoots.Add(node); + } + + bool isEnded; + if (pos >= tokens.Length) + { + isEnded = true; + } + else if (rule.IsSubRule) + { + isEnded = pos >= tokens.Length - 1 + || (pos == tokens.Length - 2 && tokens[tokens.Length - 1].IsEOS); + } + else + { + var hasNext = pos + 1 < tokens.Length; + isEnded = tokens[pos].IsEOS || (node.IsEpsilon && hasNext && tokens[pos + 1].IsEOS); + } + + res.IsEnded = res.IsEnded || isEnded; + } + + results.Add(res); + } + + if (results.Count > 0) + { + var best = results.OrderByDescending(r => r.EndingPosition).First(); + result.AlternativeRoots = best.AlternativeRoots; + result.EndingPosition = best.EndingPosition; + result.IsEnded = best.IsEnded; + result.HasByPassNodes = best.HasByPassNodes; + result.AllResults = results; + result.Ambiguities = ambiguities; + } + else + { + result.IsError = true; + } } return result; diff --git a/src/sly/parser/syntax/grammar/LeadingToken.cs b/src/sly/parser/syntax/grammar/LeadingToken.cs index 343a4c1a..893002dd 100644 --- a/src/sly/parser/syntax/grammar/LeadingToken.cs +++ b/src/sly/parser/syntax/grammar/LeadingToken.cs @@ -50,7 +50,7 @@ public bool Match(Token token) return token.IsUnIndent; } - return TokenId.Equals(token.TokenID); + return !token.IsEOS && TokenId.Equals(token.TokenID); } [ExcludeFromCodeCoverage] diff --git a/src/sly/parser/syntax/grammar/Rule.cs b/src/sly/parser/syntax/grammar/Rule.cs index 59f53a0e..69fb11f4 100644 --- a/src/sly/parser/syntax/grammar/Rule.cs +++ b/src/sly/parser/syntax/grammar/Rule.cs @@ -212,6 +212,12 @@ public void SetVisitor(OperationMetaData operation) public bool Match(IList> tokens, int position, ParserConfiguration configuration) { + // Check if position is within bounds + if (position >= tokens.Count) + { + return MayBeEmpty; + } + bool activateBroadWindow = configuration.BroadenTokenWindow; if (activateBroadWindow && Clauses.Count >= 2 && Clauses[0] is TerminalClause startingTerminalClause && Clauses[1] is NonTerminalClause nTerm) { @@ -223,7 +229,7 @@ public bool Match(IList> tokens, int position, ParserConfiguration x.Match(tokens[position+1])) || nTerm.MayBeEmpty()) + if ((position + 1 < tokens.Count && secondPossibleLeadings.Exists(x => x.Match(tokens[position+1]))) || nTerm.MayBeEmpty()) { return true; } diff --git a/src/sly/parser/syntax/tree/ParseForest.cs b/src/sly/parser/syntax/tree/ParseForest.cs new file mode 100644 index 00000000..43fb025e --- /dev/null +++ b/src/sly/parser/syntax/tree/ParseForest.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using sly.parser.syntax.grammar; + +namespace sly.parser.syntax.tree +{ + /// + /// Représente une forêt d'arbres syntaxiques issus d'une grammaire ambiguë + /// + public class ParseForest where IN : struct, Enum + { + /// + /// Liste des arbres syntaxiques valides (alternatives) + /// + public List> Trees { get; set; } + + /// + /// Indique si la forêt contient des ambiguïtés + /// + public bool IsAmbiguous => Trees != null && Trees.Count > 1; + + /// + /// Nombre d'arbres dans la forêt + /// + public int Count => Trees?.Count ?? 0; + + /// + /// Arbre principal (premier de la liste pour compatibilité) + /// + public ISyntaxNode MainTree => Trees?.FirstOrDefault(); + + /// + /// Informations sur les points d'ambiguïté dans la forêt + /// + public List> Ambiguities { get; set; } + } + + /// + /// Informations sur un point d'ambiguïté spécifique + /// + public class AmbiguityInfo where IN : struct, Enum + { + public string NonTerminalName { get; set; } + public int Position { get; set; } + public List> AlternativeRules { get; set; } + public int AlternativeCount { get; set; } + } +} diff --git a/src/sly/sly.csproj b/src/sly/sly.csproj index 493cc37c..a8e44048 100644 --- a/src/sly/sly.csproj +++ b/src/sly/sly.csproj @@ -6,10 +6,10 @@ $(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb #LY is a parser generator halfway between parser combinators and parser generator like ANTLR b3b00 - 3.7.5 + 3.8.0-alpha1 https://github.com/b3b00/sly https://github.com/b3b00/sly - 3.7.5 + 3.8.0-alpha1 Library @@ -32,7 +32,7 @@ 1701;1702;1705;1591 - + runtime; build; native; contentfiles; analyzers all @@ -40,9 +40,9 @@ all runtime; build; native; contentfiles; analyzers - + - + diff --git a/tests/ParserTests/EBNFTests.cs b/tests/ParserTests/EBNFTests.cs index 8475db16..d4c4b889 100644 --- a/tests/ParserTests/EBNFTests.cs +++ b/tests/ParserTests/EBNFTests.cs @@ -9,6 +9,7 @@ using simpleExpressionParser; using sly.buildresult; using sly.lexer; +using sly.lexer.fluent; using sly.parser; using sly.parser.generator; using sly.parser.llparser.ebnf; @@ -1886,5 +1887,51 @@ public void TestRepeat() Check.That(error.UnexpectedToken.Value).IsEqualTo("g"); } + + [Fact] + public void Test() + { + var lexerBuilder = FluentLexerBuilder.NewBuilder(); + lexerBuilder.Keyword(ABCToken.A,"a"); + lexerBuilder.Keyword(ABCToken.B,"b"); + lexerBuilder.Keyword(ABCToken.C,"c"); + + var parserBuilder = FluentEBNFParserBuilder.NewBuilder(new DummyInstance(),"root","en"); + parserBuilder.Production("root : nullable B? C", args => + { + var a = (string)args[0]; + var b = (Token)args[1]; + var c = (Token)args[2]; + return $"R({a},{(b.IsEmpty ? "":b.Value)},{c.Value})"; + }); + parserBuilder.Production("nullable : A?", args => + { + var a = (Token)args[0]; + return a.IsEmpty ? "" : a.Value; + }); + parserBuilder.WithLexerbuilder(lexerBuilder); + var parser = parserBuilder.BuildParser(); + Check.That(parser).IsOk(); + var res = parser.Result.Parse("b c"); + Check.That(res).IsOkParsing(); + Check.That(res.Result).IsEqualTo("R(,b,c)"); + + } + } + + + public class DummyInstance + { + + } + + public enum ABCToken + { + [Keyword("A")] + A, + [Keyword("B")] + B, + [Keyword("C")] + C, } } diff --git a/tests/ParserTests/Issue332/Issue332Parser.cs b/tests/ParserTests/Issue332/Issue332Parser.cs index 9368b607..0298d91e 100644 --- a/tests/ParserTests/Issue332/Issue332Parser.cs +++ b/tests/ParserTests/Issue332/Issue332Parser.cs @@ -7,6 +7,7 @@ namespace ParserTests.Issue332; [ParserRoot("root")] +[FirstDerivation] public class Issue332Parser { [Production("root: statement*")] diff --git a/tests/ParserTests/Issue414/Issue414Test.cs b/tests/ParserTests/Issue414/Issue414Test.cs index ed5e5ca4..50b5856a 100644 --- a/tests/ParserTests/Issue414/Issue414Test.cs +++ b/tests/ParserTests/Issue414/Issue414Test.cs @@ -10,7 +10,7 @@ namespace ParserTests.Issue414; public class Issue414Tests { - [Fact] + /*[Fact] public void Issue414Test() { var parserInstance = new Issue414Parser(); @@ -25,7 +25,7 @@ public void Issue414Test() result = parser.Parse(source); }).LastsLessThan(10_000, TimeUnit.Milliseconds); Check.That(result).IsOkParsing(); - } + }*/ [Fact] public void Issue414ExpressionTest() @@ -66,4 +66,4 @@ public void Issue414AltTest() }).LastsLessThan(10_000, TimeUnit.Milliseconds); Check.That(result).IsOkParsing(); } -} \ No newline at end of file +} diff --git a/tests/ParserTests/Issue443/Test443Parser.cs b/tests/ParserTests/Issue443/Test443Parser.cs index 0a19da83..46d6c065 100644 --- a/tests/ParserTests/Issue443/Test443Parser.cs +++ b/tests/ParserTests/Issue443/Test443Parser.cs @@ -7,6 +7,7 @@ namespace ParserTests.Issue443; [ParserRoot("root")] +[FirstDerivation] public class Test443Parser { [Production("root : a b* a")] diff --git a/tests/ParserTests/Issue485/Issue485Parser.cs b/tests/ParserTests/Issue485/Issue485Parser.cs index 1649d3c4..3c12dbd3 100644 --- a/tests/ParserTests/Issue485/Issue485Parser.cs +++ b/tests/ParserTests/Issue485/Issue485Parser.cs @@ -4,6 +4,7 @@ namespace ParserTests.Issue485; [ParserRoot("root")] +[FirstDerivation] public class Issue485Parser { [Production("root : PROPERTY[d] COLON[d] STRING")] diff --git a/tests/ParserTests/Issue485/Issue485SelfEscapeParser.cs b/tests/ParserTests/Issue485/Issue485SelfEscapeParser.cs index c7c2cd14..e7ac7ea5 100644 --- a/tests/ParserTests/Issue485/Issue485SelfEscapeParser.cs +++ b/tests/ParserTests/Issue485/Issue485SelfEscapeParser.cs @@ -4,6 +4,7 @@ namespace ParserTests.Issue485; [ParserRoot("root")] +[FirstDerivation] public class Issue485SelfEscapeParser { [Production("root : PROPERTY[d] COLON[d] STRING")] diff --git a/tests/ParserTests/Issue485/Issue485WithCallbackParser.cs b/tests/ParserTests/Issue485/Issue485WithCallbackParser.cs index 0a6e6fbe..ec9d8492 100644 --- a/tests/ParserTests/Issue485/Issue485WithCallbackParser.cs +++ b/tests/ParserTests/Issue485/Issue485WithCallbackParser.cs @@ -3,6 +3,7 @@ namespace ParserTests.Issue485; +[FirstDerivation] [ParserRoot("root")] public class Issue485WithCallbackParser { diff --git a/tests/ParserTests/Issue574/ParserIssue574.cs b/tests/ParserTests/Issue574/ParserIssue574.cs index a973ed3a..1124b2a9 100644 --- a/tests/ParserTests/Issue574/ParserIssue574.cs +++ b/tests/ParserTests/Issue574/ParserIssue574.cs @@ -10,6 +10,7 @@ namespace ParserTests.Issue574 { [ParserRoot("Root")] + [FirstDerivation] public class ParserIssue574 { [Production("root : Item")] diff --git a/tests/ParserTests/Issue596/Issue596Parser.cs b/tests/ParserTests/Issue596/Issue596Parser.cs new file mode 100644 index 00000000..75816aa3 --- /dev/null +++ b/tests/ParserTests/Issue596/Issue596Parser.cs @@ -0,0 +1,40 @@ +using sly; +using sly.lexer; +using sly.parser.generator; + +namespace ParserTests.Issue596 +{ + internal enum GettingStartedLexer + { + [Lexeme("[0-9]+")] + INT = 1, + + [Lexeme("\\+")] + PLUS = 2, + + [Lexeme("[ \\t]+", isSkippable: true)] // the lexeme is marked isSkippable : it will not be sent to the parser and simply discarded. + WS = 3 + } + + internal class GettingStartedParser + { + + [Production("expression: INT")] + public int intExpr(Token intToken) + { + return intToken.IntValue; + } + + [Production("expression: term PLUS expression")] + public int Expression(int left, Token operatorToken, int right) + { + return left + right; + } + + [Production("term: INT")] + public int Expression(Token intToken) + { + return intToken.IntValue; + } + } +} diff --git a/tests/ParserTests/Issue596/TestIssue596.cs b/tests/ParserTests/Issue596/TestIssue596.cs new file mode 100644 index 00000000..b6a8311f --- /dev/null +++ b/tests/ParserTests/Issue596/TestIssue596.cs @@ -0,0 +1,39 @@ +using NFluent; +using sly.parser; +using sly.parser.generator; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Xunit; + +namespace ParserTests.Issue596 +{ + public class TestIssue596 + { + [Fact] + public void Issue596Test() + { + var parserInstance = new GettingStartedParser(); + var builder = new ParserBuilder(); + var buildResult = builder.BuildParser(parserInstance, ParserType.LL_RECURSIVE_DESCENT, "expression"); + Check.That(buildResult).IsOk(); + Assert.True(buildResult.IsOk); + var parser = buildResult.Result; + + var result = parser.Parse("1+2+3"); + Check.That(result).IsOkParsing(); + Check.That(result.Result).IsEqualTo(6); + + + result = parser.Parse(""); + Check.That(result).Not.IsOkParsing(); + var error = result.Errors.First(); + Check.That(error.ErrorType).IsEqualTo(ErrorType.UnexpectedEOS); + Check.That(error.ErrorMessage).Contains("INT"); + + } + + } +} diff --git a/tests/ParserTests/ParserTests.csproj b/tests/ParserTests/ParserTests.csproj index f8a846d0..b821d45b 100644 --- a/tests/ParserTests/ParserTests.csproj +++ b/tests/ParserTests/ParserTests.csproj @@ -7,16 +7,16 @@ - - + + - + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all diff --git a/tests/ParserTests/ambiguous/AmbiguousGrammarParser.cs b/tests/ParserTests/ambiguous/AmbiguousGrammarParser.cs new file mode 100644 index 00000000..5bac87e0 --- /dev/null +++ b/tests/ParserTests/ambiguous/AmbiguousGrammarParser.cs @@ -0,0 +1,28 @@ +using sly.lexer; +using sly.parser.generator; + +namespace ParserTests.ambiguous; + +public class AmbiguousGrammarParser +{ + // S ::= 'a' S 'a' + [Production("S : A S A")] + public string Rule1(Token a1, string s, Token a2) + { + return $"aSa({s})"; + } + + // S ::= 'a' 'a' S + [Production("S : A A S")] + public string Rule2(Token a1, Token a2, string s) + { + return $"aaS({s})"; + } + + // S ::= 'a' + [Production("S : A")] + public string Rule3(Token a) + { + return "a"; + } +} \ No newline at end of file diff --git a/tests/ParserTests/ambiguous/AmbiguousGrammarTests.cs b/tests/ParserTests/ambiguous/AmbiguousGrammarTests.cs new file mode 100644 index 00000000..70b7d2e6 --- /dev/null +++ b/tests/ParserTests/ambiguous/AmbiguousGrammarTests.cs @@ -0,0 +1,370 @@ +using System.Linq; +using NFluent; +using sly.buildresult; +using sly.parser; +using sly.parser.generator; +using Xunit; + +namespace ParserTests.ambiguous +{ + + public class AmbiguousGrammarTests + { + private BuildResult> BuildParser(bool captureAmbiguities = false, + AmbiguityResolutionStrategy strategy = AmbiguityResolutionStrategy.First) + { + var parserInstance = new AmbiguousGrammarParser(); + var builder = new ParserBuilder(); + var buildResult = builder.BuildParser(parserInstance, ParserType.LL_RECURSIVE_DESCENT, "S"); + + if (buildResult.IsOk && captureAmbiguities) + { + buildResult.Result.Configuration.CaptureAmbiguities = true; + buildResult.Result.Configuration.AmbiguityStrategy = strategy; + } + + return buildResult; + } + + [Fact] + public void TestParserBuilds() + { + var buildResult = BuildParser(); + Check.That(buildResult).IsOk(); + Check.That(buildResult.Result).IsNotNull(); + } + + [Fact] + public void TestNonAmbiguousInput_SingleA() + { + var buildResult = BuildParser(); + Check.That(buildResult).IsOk(); + + var parser = buildResult.Result; + var parseResult = parser.Parse("a"); + + Check.That(parseResult).IsOkParsing(); + Check.That(parseResult.Result).IsEqualTo("a"); + Check.That(parseResult.IsAmbiguous).IsFalse(); + } + + [Fact] + public void TestAmbiguousInput_AAA_WithoutCapture() + { + // Sans capture d'ambiguïté, le parseur retourne la première dérivation + var buildResult = BuildParser(captureAmbiguities: false); + Check.That(buildResult).IsOk(); + + var parser = buildResult.Result; + var parseResult = parser.Parse("aaa"); + + Check.That(parseResult.IsOk).IsTrue(); + Check.That(parseResult.IsAmbiguous).IsFalse(); + // Première dérivation : aSa(a) ou aaS(a) selon l'ordre des règles + } + + [Fact] + public void TestAmbiguousInput_AAA_WithCapture() + { + // "aaa" peut être parsé de 2 façons : + // 1. S -> a S a -> a (a) a [Rule1 avec S=a] + // 2. S -> a a S -> a a (a) [Rule2 avec S=a] + var buildResult = BuildParser(captureAmbiguities: true); + Check.That(buildResult).IsOk(); + + var parser = buildResult.Result; + var parseResult = parser.Parse("aaa"); + + Check.That(parseResult.IsOk).IsTrue(); + Check.That(parseResult.IsAmbiguous).IsTrue(); + Check.That(parseResult.Forest).IsNotNull(); + Check.That(parseResult.Forest.Count).IsEqualTo(2); + Check.That(parseResult.Forest.Ambiguities).IsNotNull(); + Check.That(parseResult.Forest.Ambiguities).Not.IsEmpty(); + } + + [Fact] + public void TestAmbiguousInput_AAAAA_WithCapture() + { + // "aaaaa" est encore plus ambigu - plusieurs dérivations possibles + var buildResult = BuildParser(captureAmbiguities: true); + Check.That(buildResult).IsOk(); + + var parser = buildResult.Result; + var parseResult = parser.Parse("aaaaa"); + + Check.That(parseResult.IsOk).IsTrue(); + Check.That(parseResult.IsAmbiguous).IsTrue(); + Check.That(parseResult.Forest.Count).IsEqualTo(4); + var visited = parseResult.VisitAllTrees(parser.Visitor); + Check.That(visited).IsNotNull(); + } + + [Fact] + public void TestAmbiguityException_Strategy() + { + // Avec la stratégie ThrowException, une exception doit être levée + var buildResult = BuildParser(captureAmbiguities: true, + strategy: AmbiguityResolutionStrategy.ThrowException); + Check.That(buildResult).IsOk(); + + var parser = buildResult.Result; + + Check.ThatCode(() => parser.Parse("aaa")) + .Throws>(); + } + + [Fact] + public void TestAmbiguityException_ContainsForest() + { + var buildResult = BuildParser(captureAmbiguities: true, + strategy: AmbiguityResolutionStrategy.ThrowException); + Check.That(buildResult).IsOk(); + + var parser = buildResult.Result; + + try + { + var parseResult = parser.Parse("aaa"); + Check.That(parseResult).IsOkParsing(); + } + catch (sly.parser.exceptions.AmbiguousGrammarException ex) + { + Check.That(ex.Forest).IsNotNull(); + Check.That(ex.Forest.Count).IsEqualTo(2); + Check.That(ex.Message).Contains("2 alternative parse trees"); + } + } + + [Fact] + public void TestVisitAllTrees() + { + var buildResult = BuildParser(captureAmbiguities: true, + strategy: AmbiguityResolutionStrategy.All); + Check.That(buildResult).IsOk(); + + var parser = buildResult.Result; + var parseResult = parser.Parse("aaa"); + + Check.That(parseResult.IsOk).IsTrue(); + Check.That(parseResult.IsAmbiguous).IsTrue(); + + // Visiter tous les arbres + var allResults = parseResult.VisitAllTrees(parser.Visitor); + + Check.That(allResults).IsNotNull(); + Check.That(allResults).CountIs(2); + Check.That(allResults).ContainsExactly("aSa(a)", "aaS(a)"); + } + + [Fact] + public void TestSelectSpecificTree() + { + var buildResult = BuildParser(captureAmbiguities: true, + strategy: AmbiguityResolutionStrategy.All); + Check.That(buildResult).IsOk(); + + var parser = buildResult.Result; + var parseResult = parser.Parse("aaa"); + + Check.That(parseResult.IsOk).IsTrue(); + Check.That(parseResult.IsAmbiguous).IsTrue(); + + // Sélectionner le premier arbre + var result0 = parseResult.SelectTree(0, parser.Visitor); + Check.That(result0).IsNotNull(); + + // Sélectionner le deuxième arbre + var result1 = parseResult.SelectTree(1, parser.Visitor); + Check.That(result1).IsNotNull(); + + // Les deux résultats doivent être différents + Check.That(result0).Not.IsEqualTo(result1); + } + + [Fact] + public void TestResolveAmbiguity_CustomSelector() + { + var buildResult = BuildParser(captureAmbiguities: true, + strategy: AmbiguityResolutionStrategy.All); + Check.That(buildResult).IsOk(); + + var parser = buildResult.Result; + var parseResult = parser.Parse("aaa"); + + Check.That(parseResult.IsOk).IsTrue(); + Check.That(parseResult.IsAmbiguous).IsTrue(); + + // Résoudre en choisissant le dernier arbre + var result = parseResult.ResolveAmbiguity( + trees => trees.Last(), + parser.Visitor + ); + + Check.That(result).IsNotNull(); + Check.That(result).IsEqualTo("aaS(a)"); + } + + [Fact] + public void TestAmbiguityInfo() + { + var buildResult = BuildParser(captureAmbiguities: true); + Check.That(buildResult).IsOk(); + + var parser = buildResult.Result; + var parseResult = parser.Parse("aaa"); + + Check.That(parseResult.IsOk).IsTrue(); + Check.That(parseResult.Forest.Ambiguities).IsNotNull(); + Check.That(parseResult.Forest.Ambiguities).Not.IsEmpty(); + + var ambiguity = parseResult.Forest.Ambiguities.First(); + Check.That(ambiguity.NonTerminalName).IsEqualTo("S"); + Check.That(ambiguity.AlternativeCount).IsEqualTo(2); + Check.That(ambiguity.Position).IsEqualTo(0); + } + + [Fact] + public void TestBackwardCompatibility_DefaultBehavior() + { + // Sans activer CaptureAmbiguities, le comportement doit être identique à avant + var buildResult = BuildParser(captureAmbiguities: false); + Check.That(buildResult).IsOk(); + + var parser = buildResult.Result; + + // Test avec entrée non ambiguë + var result1 = parser.Parse("a"); + Check.That(result1.IsOk).IsTrue(); + Check.That(result1.IsAmbiguous).IsFalse(); + + // Test avec entrée ambiguë (mais pas de capture) + var result2 = parser.Parse("aaa"); + Check.That(result2.IsOk).IsTrue(); + Check.That(result2.IsAmbiguous).IsFalse(); // Pas d'ambiguïté détectée + Check.That(result2.Result).IsNotNull(); // Un résultat est retourné + } + + [Fact] + public void TestEbnfChoiceAmbiguity_WithCapture() + { + var parserInstance = new OptionAmbiguousParser(); + var builder = new ParserBuilder(); + var buildResult = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "S"); + Check.That(buildResult).IsOk(); + + buildResult.Result.Configuration.CaptureAmbiguities = true; + buildResult.Result.Configuration.AmbiguityStrategy = AmbiguityResolutionStrategy.All; + + var parser = buildResult.Result; + var parseResult = parser.Parse("a"); + + Check.That(parseResult.IsOk).IsTrue(); + Check.That(parseResult.IsAmbiguous).IsTrue(); + Check.That(parseResult.Forest).IsNotNull(); + Check.That(parseResult.Forest.Count).IsEqualTo(2); + + var allResults = parseResult.VisitAllTrees(parser.Visitor); + Check.That(allResults).CountIs(2); + Check.That(allResults).Contains("S(a(A))", "S(b(A))"); + } + + [Fact] + public void TestDanglingElseAmbiguityWithEbnfChoices() + { + var parserInstance = new DanglingElseParserWithEbnfChoices(); + var builder = new ParserBuilder(); + var buildResult = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "programZero"); + Check.That(buildResult).IsOk(); + + string source = @" +if x == 1 then +if y == 2 then +a := 3 +else +a := 4"; + buildResult.Result.Configuration.CaptureAmbiguities = true; + buildResult.Result.Configuration.AmbiguityStrategy = AmbiguityResolutionStrategy.All; + Check.That(buildResult.Result).IsNotNull(); + var parser = buildResult.Result; + + var parseResult = parser.Parse(source, "programZero"); + Check.That(parseResult).IsOkParsing(checkIfResultisNull:false); + Check.That(parseResult.IsAmbiguous).IsTrue(); + Check.That(parseResult.Forest).IsNotNull(); + Check.That(parseResult.Forest.Count).IsEqualTo(2); + var allResults = parseResult.VisitAllTrees(parser.Visitor); + Check.That(allResults).CountIs(2); + Check.That(allResults).Contains( "if(x==1,if(y==2,a:=3),a:=4)", "if(x==1,if(y==2,a:=3,a:=4))" ); + + parseResult = parser.Parse(source,"programOne"); + Check.That(parseResult).IsOkParsing(checkIfResultisNull:false); + Check.That(parseResult.IsAmbiguous).IsTrue(); + Check.That(parseResult.Forest).IsNotNull(); + Check.That(parseResult.Forest.Count).IsEqualTo(2); + allResults = parseResult.VisitAllTrees(parser.Visitor); + Check.That(allResults).CountIs(2); + Check.That(allResults).Contains( "if(x==1,if(y==2,a:=3),a:=4)", "if(x==1,if(y==2,a:=3,a:=4))" ); + + parseResult = parser.Parse(source,"programOneOrTwo"); + Check.That(parseResult).IsOkParsing(checkIfResultisNull:false); + Check.That(parseResult.IsAmbiguous).IsTrue(); + Check.That(parseResult.Forest).IsNotNull(); + Check.That(parseResult.Forest.Count).IsEqualTo(2); + allResults = parseResult.VisitAllTrees(parser.Visitor); + Check.That(allResults).CountIs(2); + Check.That(allResults).Contains( "if(x==1,if(y==2,a:=3),a:=4)", "if(x==1,if(y==2,a:=3,a:=4))" ); + + + } + + [Fact] + public void TestDanglingElseAmbiguityWithRules() + { + var parserInstance = new DanglingElseParserWithRules(); + var builder = new ParserBuilder(); + var buildResult = builder.BuildParser(parserInstance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "programZero"); + Check.That(buildResult).IsOk(); + + string source = @" +if x == 1 then +if y == 2 then +a := 3 +else +a := 4"; + buildResult.Result.Configuration.CaptureAmbiguities = true; + buildResult.Result.Configuration.AmbiguityStrategy = AmbiguityResolutionStrategy.All; + Check.That(buildResult.Result).IsNotNull(); + var parser = buildResult.Result; + + var parseResult = parser.Parse(source, "programZero"); + Check.That(parseResult).IsOkParsing(checkIfResultisNull:false); + Check.That(parseResult.IsAmbiguous).IsTrue(); + Check.That(parseResult.Forest).IsNotNull(); + Check.That(parseResult.Forest.Count).IsEqualTo(2); + var allResults = parseResult.VisitAllTrees(parser.Visitor); + Check.That(allResults).CountIs(2); + Check.That(allResults).Contains( "if(x==1,if(y==2,a:=3),a:=4)", "if(x==1,if(y==2,a:=3,a:=4))" ); + + parseResult = parser.Parse(source,"programOne"); + Check.That(parseResult).IsOkParsing(checkIfResultisNull:false); + Check.That(parseResult.IsAmbiguous).IsTrue(); + Check.That(parseResult.Forest).IsNotNull(); + Check.That(parseResult.Forest.Count).IsEqualTo(2); + allResults = parseResult.VisitAllTrees(parser.Visitor); + Check.That(allResults).CountIs(2); + Check.That(allResults).Contains( "if(x==1,if(y==2,a:=3),a:=4)", "if(x==1,if(y==2,a:=3,a:=4))" ); + + parseResult = parser.Parse(source,"programOneOrTwo"); + Check.That(parseResult).IsOkParsing(checkIfResultisNull:false); + Check.That(parseResult.IsAmbiguous).IsTrue(); + Check.That(parseResult.Forest).IsNotNull(); + Check.That(parseResult.Forest.Count).IsEqualTo(2); + allResults = parseResult.VisitAllTrees(parser.Visitor); + Check.That(allResults).CountIs(2); + Check.That(allResults).Contains( "if(x==1,if(y==2,a:=3),a:=4)", "if(x==1,if(y==2,a:=3,a:=4))" ); + + + } + } +} diff --git a/tests/ParserTests/ambiguous/AmbiguousToken.cs b/tests/ParserTests/ambiguous/AmbiguousToken.cs new file mode 100644 index 00000000..ce99b43d --- /dev/null +++ b/tests/ParserTests/ambiguous/AmbiguousToken.cs @@ -0,0 +1,9 @@ +using sly.lexer; + +namespace ParserTests.ambiguous; + +public enum AmbiguousToken +{ + [Lexeme("a")] A, + EOF +} \ No newline at end of file diff --git a/tests/ParserTests/ambiguous/DanglingElseParserWithEbnfChoices.cs b/tests/ParserTests/ambiguous/DanglingElseParserWithEbnfChoices.cs new file mode 100644 index 00000000..a2d312bc --- /dev/null +++ b/tests/ParserTests/ambiguous/DanglingElseParserWithEbnfChoices.cs @@ -0,0 +1,58 @@ +using sly.lexer; +using sly.parser.generator; +using System.Collections.Generic; +using sly.parser.parser; + +namespace ParserTests.ambiguous +{ + [ParserRoot("programZero")] + public class DanglingElseParserWithEbnfChoices + { + [Production("programOne : statement +")] + public string program_statement_1(List statements) + { + return string.Join(";", statements); + } + + [Production("programZero : statement +")] + public string program_statement_0(List statements) + { + return string.Join(";", statements); + } + + [Production("programOneOrTwo : statement statement?")] + public string program_statement_1_2(string statement, ValueOption statement2) + { + return statement2.Match( + (x) => $"{statement},{x}", + () => statement); + } + + [Production("statement : [ if_then | if_then_else | assign ]")] + public string statement_if_assign_(string statement) => statement; + + [Production("if_then : IF[d] cond THEN[d] statement")] + public string if_then(string cond, string thenStatement) + { + return $"if({cond},{thenStatement})"; + } + + [Production("if_then_else : IF[d] cond THEN[d] statement ELSE[d] statement")] + public string if_then_else(string cond, string thenStatement, string elseStatement) + { + return $"if({cond},{thenStatement},{elseStatement})"; + } + + [Production("assign : ID ASSIGN[d] INT")] + public string assign_ID_ASSIGN_INT(Token id, Token value) + { + return $"{id.Value}:={value.Value}"; + } + + [Production("cond : ID EQUALS[d] INT")] + public string cond_ID_EQUALS_INT(Token id, Token value) + { + return $"{id.Value}=={value.Value}"; + } + } +} \ No newline at end of file diff --git a/tests/ParserTests/ambiguous/DanglingElseParserWithRules.cs b/tests/ParserTests/ambiguous/DanglingElseParserWithRules.cs new file mode 100644 index 00000000..c422c43d --- /dev/null +++ b/tests/ParserTests/ambiguous/DanglingElseParserWithRules.cs @@ -0,0 +1,59 @@ +using System.Collections.Generic; +using sly.lexer; +using sly.parser.generator; +using sly.parser.parser; + +namespace ParserTests.ambiguous; + +[ParserRoot("programZero")] +public class DanglingElseParserWithRules +{ + [Production("programOne : statement +")] + public string program_statement_1(List statements) + { + return string.Join(";", statements); + } + + [Production("programZero : statement +")] + public string program_statement_0(List statements) + { + return string.Join(";", statements); + } + + [Production("programOneOrTwo : statement statement?")] + public string program_statement_1_2(string statement, ValueOption statement2) + { + return statement2.Match( + (x) => $"{statement},{x}", + () => statement); + } + + [Production("statement : if_then")] + [Production("statement : if_then_else")] + [Production("statement : assign")] + public string statement_if_assign_(string statement) => statement; + + [Production("if_then : IF[d] cond THEN[d] statement")] + public string if_then(string cond, string thenStatement) + { + return $"if({cond},{thenStatement})"; + } + + [Production("if_then_else : IF[d] cond THEN[d] statement ELSE[d] statement")] + public string if_then_else(string cond, string thenStatement, string elseStatement) + { + return $"if({cond},{thenStatement},{elseStatement})"; + } + + [Production("assign : ID ASSIGN[d] INT")] + public string assign_ID_ASSIGN_INT(Token id, Token value) + { + return $"{id.Value}:={value.Value}"; + } + + [Production("cond : ID EQUALS[d] INT")] + public string cond_ID_EQUALS_INT(Token id, Token value) + { + return $"{id.Value}=={value.Value}"; + } +} \ No newline at end of file diff --git a/tests/ParserTests/ambiguous/DanglingElseToken.cs b/tests/ParserTests/ambiguous/DanglingElseToken.cs new file mode 100644 index 00000000..332c715b --- /dev/null +++ b/tests/ParserTests/ambiguous/DanglingElseToken.cs @@ -0,0 +1,24 @@ +using sly.lexer; +using sly.lexer.fsm; +using sly.i18n; + +namespace ParserTests.ambiguous +{ + public enum DanglingElseToken + { + [AlphaId] + ID, + [Int] + INT, + [Keyword("if")] + IF, + [Keyword("else")] + ELSE, + [Keyword("then")] + THEN, + [Sugar("==")] + EQUALS, + [Sugar(":=")] + ASSIGN, + } +} \ No newline at end of file diff --git a/tests/ParserTests/ambiguous/OptionAmbiguousParser.cs b/tests/ParserTests/ambiguous/OptionAmbiguousParser.cs new file mode 100644 index 00000000..00c5fa4a --- /dev/null +++ b/tests/ParserTests/ambiguous/OptionAmbiguousParser.cs @@ -0,0 +1,27 @@ +using sly.lexer; +using sly.parser.generator; + +namespace ParserTests.ambiguous; + +public class OptionAmbiguousParser +{ + + [Production("S : [a|b]")] + public string RootOption(string option) + { + return $"S({option})"; + } + + + [Production("a : A")] + public string RuleA(Token a) + { + return "a(A)"; + } + + [Production("b : A")] + public string RuleB(Token a) + { + return "b(A)"; + } +} \ No newline at end of file diff --git a/tests/ParserTests/dateIssue/Issue554Parser.cs b/tests/ParserTests/dateIssue/Issue554Parser.cs index acdce073..7dd9628b 100644 --- a/tests/ParserTests/dateIssue/Issue554Parser.cs +++ b/tests/ParserTests/dateIssue/Issue554Parser.cs @@ -5,6 +5,7 @@ namespace ParserTests.dateIssue; [ParserRoot("root")] +[FirstDerivation] public class Issue554Parser { [Production("root : DOUBLE INT DATE")] diff --git a/tests/ParserTests/lexer/GenericLexerTests.cs b/tests/ParserTests/lexer/GenericLexerTests.cs index f60b2b36..d0df8196 100644 --- a/tests/ParserTests/lexer/GenericLexerTests.cs +++ b/tests/ParserTests/lexer/GenericLexerTests.cs @@ -8,6 +8,7 @@ using simpleExpressionParser; using sly.buildresult; using sly.lexer; +using sly.lexer.fsm; using sly.parser; using sly.parser.generator; using Xunit; @@ -23,10 +24,6 @@ namespace ParserTests.lexer public class GenericLexerTests { - public GenericLexerTests() - { - } - [Fact] public void TestEmptyInput() { @@ -384,6 +381,107 @@ public void TestParserWithLexerExtensions() Check.That(result.Result).IsEqualTo(3.14); } + + [Fact] + public void TestPostFixHexaExtensionIssue583() + { + Action> extensionBuilder = + (token, attribute, lexer) => + { + if (token == PostfixHexa.Hexa) + { + + bool CheckHexa(ReadOnlyMemory value) + { + char[] hexaLetters = new char[] { 'a', 'b', 'c', 'd', 'e', 'f' }; + var ok = false; + return value.ToArray().All(x => Char.IsDigit(x) || hexaLetters.Contains(x)); + } + + NodeCallback callback = (FSMMatch match) => + { + // this store the token id the the FSMMatch object to be later returned by GenericLexer.Tokenize + match.Properties[GenericLexer.DerivedToken] = PostfixHexa.Hexa; + return match; + }; + var builder = lexer.FSMBuilder; + builder.GoTo(GenericLexer.start) + .MultiRangeTransition([('0', '9'), ('a', 'f'), ('A', 'Z')]) + .Mark("in_hexa") + .MultiRangeTransitionTo("in_hexa", [('0', '9'), ('a', 'f'), ('A', 'Z')]) + .Transition('h') + .Mark("hexa_done") + .End(GenericToken.Extension, false) + .CallBack(callback); + builder.GoTo(GenericLexer.in_int) + .MultiRangeTransition([('0', '9'), ('a', 'f'), ('A', 'Z')]) + .Mark("in_hexa_after_int") + .MultiRangeTransitionTo("in_hexa_after_int", [('0', '9'), ('a', 'f'), ('A', 'Z')]) + .Transition('h') + .End(GenericToken.Extension, false) + .CallBack(callback); + } + }; + + + // this post processor retag identifier tokens ending with 'h' and only containing hex digits as Hexa tokens + LexerPostProcess postProcess = (tokens) => + { + return tokens.Select(x => + { + if (x.TokenID == PostfixHexa.Id && x.Value.EndsWith("h")) + { + x.TokenID = PostfixHexa.Hexa; + } + + return x; + }).ToList(); + }; + var source = "01fh 42 identifier abc abch"; + + var lexerBuild = LexerBuilder.BuildLexer(lexerPostProcess:postProcess,extensionBuilder:extensionBuilder, lang:"en"); + Check.That(lexerBuild).IsOk(); + var lexer = lexerBuild.Result; + var lexed = lexer.Tokenize(source, true); + Check.That(lexed).IsOkLexing(); + Check.That(lexed.Tokens.MainTokens().Extracting(x => (x.TokenID, x.Value))).ContainsExactly([ + (PostfixHexa.Hexa, "01fh"), + (PostfixHexa.Int, "42"), + (PostfixHexa.Id, "identifier"), + (PostfixHexa.Id, "abc"), + (PostfixHexa.Hexa, "abch"), + (PostfixHexa.EOS,"")]); + + var instance = new Parser(); + ParserBuilder parserBuilder = new ParserBuilder(); + var parserBuild = parserBuilder.BuildParser(instance, ParserType.EBNF_LL_RECURSIVE_DESCENT, "main", + extensionBuilder: extensionBuilder, lexerPostProcess: postProcess); + if (parserBuild.IsError) + { + Console.WriteLine("Parser build error"); + foreach (var error in parserBuild.Errors) + { + Console.WriteLine(error); + } + + return; + } + + var parser = parserBuild.Result; + + + + + var parsed = parser.Parse(source); + Check.That(parsed).IsOkParsing(); + Check.That(parsed.Result).Contains("Hexa [01fh]"); + Check.That(parsed.Result).Contains("Int [42]"); + Check.That(parsed.Result).Contains("Id [identifier]"); + Check.That(parsed.Result).Contains("Id [abc]"); + Check.That(parsed.Result).Contains("Hexa [abch]"); + + } + [Fact] public void TestExtensions() { diff --git a/tests/ParserTests/lexer/genericlexers/Extensions.cs b/tests/ParserTests/lexer/genericlexers/Extensions.cs index 38c2cb5d..5bff1664 100644 --- a/tests/ParserTests/lexer/genericlexers/Extensions.cs +++ b/tests/ParserTests/lexer/genericlexers/Extensions.cs @@ -8,4 +8,6 @@ public enum Extensions [Lexeme(GenericToken.Double,channel:0)] DOUBLE } + + } \ No newline at end of file diff --git a/tests/ParserTests/lexer/genericlexers/PostfixHexa.cs b/tests/ParserTests/lexer/genericlexers/PostfixHexa.cs new file mode 100644 index 00000000..b522b7c9 --- /dev/null +++ b/tests/ParserTests/lexer/genericlexers/PostfixHexa.cs @@ -0,0 +1,33 @@ +using System.Collections.Generic; +using sly.lexer; +using sly.parser.generator; + +namespace ParserTests.lexer.genericlexers; + + + public enum PostfixHexa + { + EOS, + + [Extension] Hexa, + + [Int] Int, + + [AlphaId] Id + } + + public class Parser + { + [Production("main : item+ ")] + public string Main(List items) + { + return string.Join("\n", items); + } + + [Production("item : [Id | Hexa | Int]" )] + public string Item(Token token) + { + return token.ToString(); + } + + }