Java/JS regex support for escapes within charclasses, JS regex null escape, identity escapes & legacy octal escape

core/src/main/antlr4/org/evomaster/core/parser/RegexEcma262.g4

@@ -86,40 +86,46 @@ bracketQuantifierRange
 atom
  : patternCharacter+
  | DOT
- | AtomEscape
+ | atomEscape
  | characterClass
  | PAREN_open disjunction PAREN_close
  //TODO
 // | '(' '?' ':' disjunction ')'
  ;
 
 
-
 //TODO
-fragment CharacterEscape
- : ControlEscape
- | 'c' ControlLetter
- | HexEscapeSequence
- | UnicodeEscapeSequence
- //| IdentityEscape
+CharacterEscape
+ : SLASH ControlEscape
+ | SLASH HexEscapeSequence
+ | SLASH UnicodeEscapeSequence
+ | SLASH OctalEscapeSequence // legacy octal escapes are deprecated, but this also works for null escape (\u0000)
+ | SLASH IdentityEscape
+ ;
+
+//TODO backreferences
+
+ControlLetterExtendedEscape
+ // This handles both control letter escapes (\ca, \cZ, etc.) and literal interpretations of \c.
+ // As in JS: "\c" + [^a-zA-Z]? is taken literally as "\c" + [^a-zA-Z]? outside charclasses
+ // while "\c" + [^a-zA-Z0-9_]? is taken literally as "\c" + [^a-zA-Z0-9_]? within charclasses.
+ // Therefore, as all characters following "\c" (or none) are permitted we accept "\c" + .? here
+ // and handle each case in visitor.
+ : SLASH 'c' .?   // matches \c, \c<anything>
  ;
 
 fragment ControlEscape
  //one of f n r t v
  : [fnrtv]
  ;
 
-fragment ControlLetter
- : [a-zA-Z]
+fragment IdentityEscape
+ // In JS escape sequences that are not one of the above (excluding backreferences) become identity escapes:
+ // they represent the character that follows the backslash. (e.g.: "\a" becomes "a")
+ // see: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Regular_expressions/Character_escape#:~:text=identity%20escapes
+ : ~[dDsSwWfnrtvxuc0-9]
  ;
 
-
-//TODO
-//fragment IdentityEscape ::
-//SourceCharacter but not IdentifierPart
-//<ZWJ>
-//<ZWNJ>
-
 //TODO
 //DecimalEscape
 // //[lookahead ∉ DecimalDigit]
@@ -170,26 +176,29 @@ classAtomNoDash
  //SourceCharacter but not one of \ or ] or -
  //TODO
  //: ~[-\]\\]
-// | '\\' ClassEscape
- : BaseChar
+ : classEscape
+ | BaseChar
  | DecimalDigit
  | COMMA | CARET | DOLLAR | SLASH | DOT | STAR | PLUS | QUESTION
  | PAREN_open | PAREN_close | BRACKET_open | BRACE_open | BRACE_close | OR;
 
 
-//TODO
-//ClassEscape
-// : CharacterClassEscape
-//// | DecimalEscape
-//// | 'b'
-// //| CharacterEscape
-// ;
+// TODO
+classEscape
+ : controlLetterExtendedEscape // this needs to be first so that we can accept things like \c and \c0 within charclasses
+ | atomEscape
+// | SLASH 'b'
+ ;
 
 decimalDigits
  : DecimalDigit+
  ;
 
-
+controlLetterExtendedEscape
+ // we need this as a parser rule because differentiating between being inside a charclass or outside is important
+ // as behavior changes in each case
+ : ControlLetterExtendedEscape
+ ;
 
 //------ LEXER ------------------------------
 // Lexer rules have first letter in upper-case
@@ -199,16 +208,17 @@ DecimalDigit
  ;
 
 
-AtomEscape
- : '\\' CharacterClassEscape
+atomEscape
+ : CharacterClassEscape
  //TODO
 // | '\\' DecimalEscape
- | '\\' CharacterEscape
+ | CharacterEscape
+ | controlLetterExtendedEscape
  ;
 
-fragment CharacterClassEscape
+CharacterClassEscape
  //one of d D s S w W
- : [dDsSwW]
+ : SLASH [dDsSwW]
  ;
 
 
@@ -236,6 +246,12 @@ BaseChar
  : ~[0-9,^$\\.*+?()[\]{}|-]
  ;
 
+fragment OctalEscapeSequence
+ : OctalDigit
+ | OctalDigit OctalDigit
+ | [0-3] OctalDigit OctalDigit
+;
+
 fragment UnicodeEscapeSequence
  : 'u' HexDigit HexDigit HexDigit HexDigit
  ;
@@ -248,6 +264,10 @@ fragment HexDigit:
  [a-fA-F0-9]
  ;
 
+fragment OctalDigit:
+ [0-7]
+ ;
+
 //TODO
 //DecimalIntegerLiteral
 // : '0'

core/src/main/antlr4/org/evomaster/core/parser/RegexJava.g4

@@ -88,7 +88,7 @@ atom
  : quote
  | patternCharacter+
  | DOT
- | AtomEscape
+ | atomEscape
  | characterClass
  | PAREN_open disjunction PAREN_close
  // These two rules are added to handle the . and + symbols in emails
@@ -119,17 +119,34 @@ quoteChar
 ;
 
 //TODO
-fragment CharacterEscape
- : ControlEscape
- | 'c' ControlLetter
- | HexEscapeSequence
- | UnicodeEscapeSequence
- | OctalEscapeSequence
- | 'p' BRACE_open PosixCharacterClassLabel BRACE_close // this is only implemented in Java at the moment as on JS this
-                                                       // is allowed only while certain flags are enabled
+CharacterEscape
+ : SLASH ControlEscape
+ | SLASH 'c' ControlLetter
+ | SLASH HexEscapeSequence
+ | SLASH UnicodeEscapeSequence
+ | SLASH OctalEscapeSequence
+ | SLASH ('p' | 'P') BRACE_open PCharacterClassEscapeLabel BRACE_close // this is only implemented in Java at the moment
+                                                        // as on JS this is allowed only while certain flags are enabled
+
  //| IdentityEscape
  ;
 
+// TODO missing \p escapes
+fragment PCharacterClassEscapeLabel
+ : PosixCharacterClassLabel
+ | UnicodeCategoriesLabel
+// | UnicodeScriptsLabel // https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html#usc
+// | UnicodeBlocksLabel // https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html#ubc
+// | UnicodeBinaryProperiesLabel // https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html#ubpc
+// | javalangCharacterClassesLabel // https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html#jcc
+ ;
+
+// TODO missing Unicode categories labels and implementations
+// https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html#ucc
+fragment UnicodeCategoriesLabel
+ : 'Pe'
+ ;
+
 // basic US-ASCII only predefined POSIX character classes
 // https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html#:~:text=character%3A%20%5B%5E%5Cw%5D-,POSIX,-character%20classes%20(US
 fragment PosixCharacterClassLabel
@@ -215,27 +232,28 @@ classAtomNoDash
  //SourceCharacter but not one of \ or ] or -
  //TODO
  //: ~[-\]\\]
-// | '\\' ClassEscape
- : BaseChar
+ : classEscape
+ | BaseChar
  | DecimalDigit
  | COMMA | CARET | DOLLAR | SLASH | DOT | STAR | PLUS | QUESTION
  | PAREN_open | PAREN_close | BRACKET_open | BRACE_open | BRACE_close | OR | E | Q
  | ESCAPED_DOT | ESCAPED_PLUS;
 
-
-//TODO
-//ClassEscape
-// : CharacterClassEscape
-//// | DecimalEscape
-//// | 'b'
-// //| CharacterEscape
-// ;
-
 decimalDigits
  : DecimalDigit+
  ;
 
+classEscape
+ : atomEscape
+// | SLASH 'b'
+ ;
 
+atomEscape
+ : CharacterClassEscape
+ | CharacterEscape
+// TODO
+// | '\\' DecimalEscape
+ ;
 
 //------ LEXER ------------------------------
 // Lexer rules have first letter in upper-case
@@ -244,19 +262,11 @@ DecimalDigit
  : [0-9]
  ;
 
-
-AtomEscape
- : '\\' CharacterClassEscape
- //TODO
-// | '\\' DecimalEscape
- | '\\' CharacterEscape
- ;
-
-fragment CharacterClassEscape
+CharacterClassEscape
  //one of d D s S w W v V h H
- // v, V, h and H are java8 exclusive, they represent vertical spaces and horizaontal spaces respectively
- // see https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html for more information
- : [dDsSwWvVhH]
+  // v, V, h and H are java8 exclusive, they represent vertical spaces and horizaontal spaces respectively
+  // see https://docs.oracle.com/javase/8/docs/api/java/util/regex/Pattern.html for more information
+  : SLASH [dDsSwWvVhH]
  ;