diff --git a/YamlDotNet.Test/Core/ScannerTests.cs b/YamlDotNet.Test/Core/ScannerTests.cs index 408a524b..967097f3 100644 --- a/YamlDotNet.Test/Core/ScannerTests.cs +++ b/YamlDotNet.Test/Core/ScannerTests.cs @@ -389,6 +389,73 @@ public void CommentsAreOmittedUnlessRequested() StreamEnd); } + [Fact] + public void JsonCommentsAreReturnedWhenRequested() + { + var sut = new Scanner(Yaml.ReaderForText(@" + // Top comment + - first // Comment on first item + - second + // Bottom comment + "), skipComments: false); + sut.JsonComments = true; + AssertSequenceOfTokensFrom(sut, + StreamStart, + StandaloneComment("Top comment"), + BlockSequenceStart, + BlockEntry, + PlainScalar("first"), + InlineComment("Comment on first item"), + BlockEntry, + PlainScalar("second"), + StandaloneComment("Bottom comment"), + BlockEnd, + StreamEnd); + } + + [Fact] + public void JsonCommentsAreCorrectlyMarked() + { + var sut = new Scanner(Yaml.ReaderForText(@" + - first // Comment on first item + "), skipComments: false); + sut.JsonComments = true; + + while (sut.MoveNext()) + { + if (sut.Current is Comment comment) + { + Assert.Equal(8, comment.Start.Index); + Assert.Equal(32, comment.End.Index); + + return; + } + } + + Assert.Fail("Did not find a comment"); + } + + [Fact] + public void JsonCommentsAreOmittedUnlessRequested() + { + var sut = Yaml.ScannerForText(@" + // Top comment + - first // Comment on first item + - second + // Bottom comment + "); + sut.JsonComments = true; + AssertSequenceOfTokensFrom(sut, + StreamStart, + BlockSequenceStart, + BlockEntry, + PlainScalar("first"), + BlockEntry, + PlainScalar("second"), + BlockEnd, + StreamEnd); + } + [Fact] public void MarksOnDoubleQuotedScalarsAreCorrect() { diff --git a/YamlDotNet/Core/Scanner.cs b/YamlDotNet/Core/Scanner.cs index 4cabd49a..79ba3749 100644 --- a/YamlDotNet/Core/Scanner.cs +++ b/YamlDotNet/Core/Scanner.cs @@ -102,6 +102,11 @@ public bool SkipComments get; private set; } + public bool JsonComments + { + get; set; // Only way to set right now + } + /// /// Gets the current token. /// @@ -497,6 +502,7 @@ private void FetchNextToken() // '-', '?', ':', ',', '[', ']', '{', '}', // '#', '&', '*', '!', '|', '>', '\'', '\"', // '%', '@', '`'. + // Need to also block "//" when JSON Comments are enabled // In the block context (and, for the '-' indicator, in the flow context // too), it may also start with the characters @@ -508,7 +514,7 @@ private void FetchNextToken() // The last rule is more restrictive than the specification requires. - var isInvalidPlainScalarCharacter = analyzer.IsWhiteBreakOrZero() || analyzer.Check("-?:,[]{}#&*!|>'\"%@`"); + var isInvalidPlainScalarCharacter = analyzer.IsWhiteBreakOrZero() || analyzer.Check("-?:,[]{}#&*!|>'\"%@`") || CheckJsonComment(); var isPlainScalar = !isInvalidPlainScalarCharacter || @@ -560,6 +566,11 @@ private void FetchNextToken() throw new SyntaxErrorException(start, end, "While scanning for the next token, found character that cannot start any token."); } + private bool CheckJsonComment() + { + return JsonComments && analyzer.Check('/') && analyzer.Check('/', 1); + } + private bool CheckWhiteSpace() { return analyzer.Check(' ') || ((flowLevel > 0 || !simpleKeyAllowed) && analyzer.Check('\t')); @@ -636,13 +647,23 @@ private void ScanToNextToken() private void ProcessComment() { - if (analyzer.Check('#')) + // Additional check for JSON-style comments (//) to support YAML with comments in JSON style + // See https://github.com/aaubry/YamlDotNet/issues/1052 and https://github.com/yaml/www.yaml.org/issues/196 + bool isYamlComment = analyzer.Check('#'); + bool isJsonComment = !isYamlComment && CheckJsonComment(); + + if (isYamlComment || isJsonComment) { var start = cursor.Mark(); - // Eat '#' + // Eat '#' or '//' Skip(); + if (isJsonComment) + { + Skip(); + } + // Eat leading whitespace while (analyzer.IsSpace()) { @@ -816,11 +837,11 @@ private void FetchDirective() } // Eat the rest of the line including any comments. - - while (analyzer.IsWhite()) - { - Skip(); - } + SkipWhitespaces(); + // while (analyzer.IsWhite()) + // { + // Skip(); + // } ProcessComment(); @@ -969,7 +990,7 @@ private void FetchFlowCollectionEnd(bool isSequenceToken) Token? token, errorToken = null; if (isSequenceToken) { - if (analyzer.Check('#')) + if (analyzer.Check('#') || CheckJsonComment()) { errorToken = new Error("While scanning a flow sequence end, found invalid comment after ']'.", start, start); } @@ -1025,7 +1046,7 @@ private void FetchFlowEntry() Skip(); var end = cursor.Mark(); - if (analyzer.Check('#')) + if (analyzer.Check('#') || CheckJsonComment()) { tokens.Enqueue(new Error("While scanning a flow entry, found invalid comment after comma.", start, end)); return; @@ -1551,17 +1572,18 @@ private Scalar ScanBlockScalar(bool isLiteral) // Check if there is a comment without whitespace after block scalar indicator (yaml-test-suite: X4QW). - if (analyzer.Check('#')) + if (analyzer.Check('#') || CheckJsonComment()) { throw new SyntaxErrorException(start, cursor.Mark(), "While scanning a block scalar, found a comment without whtespace after '>' indicator."); } // Eat whitespaces and comments to the end of the line. - while (analyzer.IsWhite()) - { - Skip(); - } + SkipWhitespaces(); + // while (analyzer.IsWhite()) + // { + // Skip(); + // } ProcessComment(); @@ -1787,7 +1809,7 @@ private void FetchQuotedScalar(bool isSingleQuoted) tokens.Enqueue(scalar); // Check if there is a comment subsequently after double-quoted scalar without space. - if (!isSingleQuoted && analyzer.Check('#')) + if (!isSingleQuoted && (analyzer.Check('#') || CheckJsonComment())) { var start = cursor.Mark(); tokens.Enqueue(new Error("While scanning a flow sequence end, found invalid comment after double-quoted scalar.", start, start)); @@ -2148,7 +2170,7 @@ private Scalar ScanPlainScalar(ref bool isMultiline) // Check for a comment. - if (analyzer.Check('#')) + if (analyzer.Check('#') || CheckJsonComment()) { if (indent < 0 && flowLevel == 0) {