Add UTF-8 byte order mark (BOM) to the rewrite-json Lexer/Parser.

openrewrite · Jan 18, 2022 · aa59e6a · aa59e6a
1 parent 6576255
commit aa59e6a
Show file tree

Hide file tree

Showing 22 changed files with 970 additions and 949 deletions.
diff --git a/rewrite-json/src/main/antlr/JSON5.g4 b/rewrite-json/src/main/antlr/JSON5.g4
@@ -145,3 +145,5 @@ fragment NEWLINE
 WS
    : [ \t\n\r\u00A0\uFEFF\u2003] + -> skip
    ;
+
+UTF_8_BOM : '\uFEFF' -> skip;
diff --git a/rewrite-json/src/main/antlr/JsonPathLexer.tokens b/rewrite-json/src/main/antlr/JsonPathLexer.tokens
@@ -23,6 +23,7 @@ RPAREN=22
 QUESTION=23
 Identifier=24
 WS=25
+UTF_8_BOM=26
 '@'=3
 '..'=4
 '.'=5
@@ -44,3 +45,4 @@ WS=25
 '('=21
 ')'=22
 '?'=23
+'\uFEFF'=26
diff --git a/rewrite-json/src/main/java/org/openrewrite/json/internal/grammar/JSON5.interp b/rewrite-json/src/main/java/org/openrewrite/json/internal/grammar/JSON5.interp
@@ -15,6 +15,7 @@ null
 null
 null
 null
+'\uFEFF'
 
 token symbolic names:
 null
@@ -33,6 +34,7 @@ NUMERIC_LITERAL
 SYMBOL
 IDENTIFIER
 WS
+UTF_8_BOM
 
 rule names:
 json5
@@ -45,4 +47,4 @@ number
 
 
 atn:
-[3, 24715, 42794, 33075, 47597, 16764, 15335, 30598, 22884, 3, 17, 76, 4, 2, 9, 2, 4, 3, 9, 3, 4, 4, 9, 4, 4, 5, 9, 5, 4, 6, 9, 6, 4, 7, 9, 7, 4, 8, 9, 8, 3, 2, 5, 2, 18, 10, 2, 3, 2, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 7, 3, 26, 10, 3, 12, 3, 14, 3, 29, 11, 3, 3, 3, 5, 3, 32, 10, 3, 3, 3, 3, 3, 3, 3, 3, 3, 5, 3, 38, 10, 3, 3, 4, 3, 4, 3, 4, 3, 4, 3, 5, 3, 5, 3, 6, 3, 6, 3, 6, 3, 6, 3, 6, 5, 6, 51, 10, 6, 3, 7, 3, 7, 3, 7, 3, 7, 7, 7, 57, 10, 7, 12, 7, 14, 7, 60, 11, 7, 3, 7, 5, 7, 63, 10, 7, 3, 7, 3, 7, 3, 7, 3, 7, 5, 7, 69, 10, 7, 3, 8, 5, 8, 72, 10, 8, 3, 8, 3, 8, 3, 8, 2, 2, 9, 2, 4, 6, 8, 10, 12, 14, 2, 4, 4, 2, 12, 12, 16, 16, 3, 2, 13, 14, 2, 80, 2, 17, 3, 2, 2, 2, 4, 37, 3, 2, 2, 2, 6, 39, 3, 2, 2, 2, 8, 43, 3, 2, 2, 2, 10, 50, 3, 2, 2, 2, 12, 68, 3, 2, 2, 2, 14, 71, 3, 2, 2, 2, 16, 18, 5, 10, 6, 2, 17, 16, 3, 2, 2, 2, 17, 18, 3, 2, 2, 2, 18, 19, 3, 2, 2, 2, 19, 20, 7, 2, 2, 3, 20, 3, 3, 2, 2, 2, 21, 22, 7, 3, 2, 2, 22, 27, 5, 6, 4, 2, 23, 24, 7, 4, 2, 2, 24, 26, 5, 6, 4, 2, 25, 23, 3, 2, 2, 2, 26, 29, 3, 2, 2, 2, 27, 25, 3, 2, 2, 2, 27, 28, 3, 2, 2, 2, 28, 31, 3, 2, 2, 2, 29, 27, 3, 2, 2, 2, 30, 32, 7, 4, 2, 2, 31, 30, 3, 2, 2, 2, 31, 32, 3, 2, 2, 2, 32, 33, 3, 2, 2, 2, 33, 34, 7, 5, 2, 2, 34, 38, 3, 2, 2, 2, 35, 36, 7, 3, 2, 2, 36, 38, 7, 5, 2, 2, 37, 21, 3, 2, 2, 2, 37, 35, 3, 2, 2, 2, 38, 5, 3, 2, 2, 2, 39, 40, 5, 8, 5, 2, 40, 41, 7, 6, 2, 2, 41, 42, 5, 10, 6, 2, 42, 7, 3, 2, 2, 2, 43, 44, 9, 2, 2, 2, 44, 9, 3, 2, 2, 2, 45, 51, 7, 12, 2, 2, 46, 51, 5, 14, 8, 2, 47, 51, 5, 4, 3, 2, 48, 51, 5, 12, 7, 2, 49, 51, 7, 11, 2, 2, 50, 45, 3, 2, 2, 2, 50, 46, 3, 2, 2, 2, 50, 47, 3, 2, 2, 2, 50, 48, 3, 2, 2, 2, 50, 49, 3, 2, 2, 2, 51, 11, 3, 2, 2, 2, 52, 53, 7, 7, 2, 2, 53, 58, 5, 10, 6, 2, 54, 55, 7, 4, 2, 2, 55, 57, 5, 10, 6, 2, 56, 54, 3, 2, 2, 2, 57, 60, 3, 2, 2, 2, 58, 56, 3, 2, 2, 2, 58, 59, 3, 2, 2, 2, 59, 62, 3, 2, 2, 2, 60, 58, 3, 2, 2, 2, 61, 63, 7, 4, 2, 2, 62, 61, 3, 2, 2, 2, 62, 63, 3, 2, 2, 2, 63, 64, 3, 2, 2, 2, 64, 65, 7, 8, 2, 2, 65, 69, 3, 2, 2, 2, 66, 67, 7, 7, 2, 2, 67, 69, 7, 8, 2, 2, 68, 52, 3, 2, 2, 2, 68, 66, 3, 2, 2, 2, 69, 13, 3, 2, 2, 2, 70, 72, 7, 15, 2, 2, 71, 70, 3, 2, 2, 2, 71, 72, 3, 2, 2, 2, 72, 73, 3, 2, 2, 2, 73, 74, 9, 3, 2, 2, 74, 15, 3, 2, 2, 2, 11, 17, 27, 31, 37, 50, 58, 62, 68, 71]
+[3, 24715, 42794, 33075, 47597, 16764, 15335, 30598, 22884, 3, 18, 76, 4, 2, 9, 2, 4, 3, 9, 3, 4, 4, 9, 4, 4, 5, 9, 5, 4, 6, 9, 6, 4, 7, 9, 7, 4, 8, 9, 8, 3, 2, 5, 2, 18, 10, 2, 3, 2, 3, 2, 3, 3, 3, 3, 3, 3, 3, 3, 7, 3, 26, 10, 3, 12, 3, 14, 3, 29, 11, 3, 3, 3, 5, 3, 32, 10, 3, 3, 3, 3, 3, 3, 3, 3, 3, 5, 3, 38, 10, 3, 3, 4, 3, 4, 3, 4, 3, 4, 3, 5, 3, 5, 3, 6, 3, 6, 3, 6, 3, 6, 3, 6, 5, 6, 51, 10, 6, 3, 7, 3, 7, 3, 7, 3, 7, 7, 7, 57, 10, 7, 12, 7, 14, 7, 60, 11, 7, 3, 7, 5, 7, 63, 10, 7, 3, 7, 3, 7, 3, 7, 3, 7, 5, 7, 69, 10, 7, 3, 8, 5, 8, 72, 10, 8, 3, 8, 3, 8, 3, 8, 2, 2, 9, 2, 4, 6, 8, 10, 12, 14, 2, 4, 4, 2, 12, 12, 16, 16, 3, 2, 13, 14, 2, 80, 2, 17, 3, 2, 2, 2, 4, 37, 3, 2, 2, 2, 6, 39, 3, 2, 2, 2, 8, 43, 3, 2, 2, 2, 10, 50, 3, 2, 2, 2, 12, 68, 3, 2, 2, 2, 14, 71, 3, 2, 2, 2, 16, 18, 5, 10, 6, 2, 17, 16, 3, 2, 2, 2, 17, 18, 3, 2, 2, 2, 18, 19, 3, 2, 2, 2, 19, 20, 7, 2, 2, 3, 20, 3, 3, 2, 2, 2, 21, 22, 7, 3, 2, 2, 22, 27, 5, 6, 4, 2, 23, 24, 7, 4, 2, 2, 24, 26, 5, 6, 4, 2, 25, 23, 3, 2, 2, 2, 26, 29, 3, 2, 2, 2, 27, 25, 3, 2, 2, 2, 27, 28, 3, 2, 2, 2, 28, 31, 3, 2, 2, 2, 29, 27, 3, 2, 2, 2, 30, 32, 7, 4, 2, 2, 31, 30, 3, 2, 2, 2, 31, 32, 3, 2, 2, 2, 32, 33, 3, 2, 2, 2, 33, 34, 7, 5, 2, 2, 34, 38, 3, 2, 2, 2, 35, 36, 7, 3, 2, 2, 36, 38, 7, 5, 2, 2, 37, 21, 3, 2, 2, 2, 37, 35, 3, 2, 2, 2, 38, 5, 3, 2, 2, 2, 39, 40, 5, 8, 5, 2, 40, 41, 7, 6, 2, 2, 41, 42, 5, 10, 6, 2, 42, 7, 3, 2, 2, 2, 43, 44, 9, 2, 2, 2, 44, 9, 3, 2, 2, 2, 45, 51, 7, 12, 2, 2, 46, 51, 5, 14, 8, 2, 47, 51, 5, 4, 3, 2, 48, 51, 5, 12, 7, 2, 49, 51, 7, 11, 2, 2, 50, 45, 3, 2, 2, 2, 50, 46, 3, 2, 2, 2, 50, 47, 3, 2, 2, 2, 50, 48, 3, 2, 2, 2, 50, 49, 3, 2, 2, 2, 51, 11, 3, 2, 2, 2, 52, 53, 7, 7, 2, 2, 53, 58, 5, 10, 6, 2, 54, 55, 7, 4, 2, 2, 55, 57, 5, 10, 6, 2, 56, 54, 3, 2, 2, 2, 57, 60, 3, 2, 2, 2, 58, 56, 3, 2, 2, 2, 58, 59, 3, 2, 2, 2, 59, 62, 3, 2, 2, 2, 60, 58, 3, 2, 2, 2, 61, 63, 7, 4, 2, 2, 62, 61, 3, 2, 2, 2, 62, 63, 3, 2, 2, 2, 63, 64, 3, 2, 2, 2, 64, 65, 7, 8, 2, 2, 65, 69, 3, 2, 2, 2, 66, 67, 7, 7, 2, 2, 67, 69, 7, 8, 2, 2, 68, 52, 3, 2, 2, 2, 68, 66, 3, 2, 2, 2, 69, 13, 3, 2, 2, 2, 70, 72, 7, 15, 2, 2, 71, 70, 3, 2, 2, 2, 71, 72, 3, 2, 2, 2, 72, 73, 3, 2, 2, 2, 73, 74, 9, 3, 2, 2, 74, 15, 3, 2, 2, 2, 11, 17, 27, 31, 37, 50, 58, 62, 68, 71]
diff --git a/rewrite-json/src/main/java/org/openrewrite/json/internal/grammar/JSON5.tokens b/rewrite-json/src/main/java/org/openrewrite/json/internal/grammar/JSON5.tokens
@@ -13,9 +13,11 @@ NUMERIC_LITERAL=12
 SYMBOL=13
 IDENTIFIER=14
 WS=15
+UTF_8_BOM=16
 '{'=1
 ','=2
 '}'=3
 ':'=4
 '['=5
 ']'=6
+'\uFEFF'=16
diff --git a/rewrite-json/src/main/java/org/openrewrite/json/internal/grammar/JSON5BaseListener.java b/rewrite-json/src/main/java/org/openrewrite/json/internal/grammar/JSON5BaseListener.java
@@ -13,9 +13,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-// Generated from /Users/jon/Projects/github/openrewrite/rewrite/rewrite-json/src/main/antlr/JSON5.g4 by ANTLR 4.9.2
 package org.openrewrite.json.internal.grammar;
-
+// Generated from /Users/pat/dev/openrewrite/rewrite/rewrite-json/src/main/antlr/JSON5.g4 by ANTLR 4.9.2
 import org.antlr.v4.runtime.ParserRuleContext;
 import org.antlr.v4.runtime.tree.ErrorNode;
 import org.antlr.v4.runtime.tree.TerminalNode;

diff --git a/rewrite-json/src/main/java/org/openrewrite/json/internal/grammar/JSON5BaseVisitor.java b/rewrite-json/src/main/java/org/openrewrite/json/internal/grammar/JSON5BaseVisitor.java
@@ -13,8 +13,8 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-// Generated from /Users/jon/Projects/github/openrewrite/rewrite/rewrite-json/src/main/antlr/JSON5.g4 by ANTLR 4.9.2
 package org.openrewrite.json.internal.grammar;
+// Generated from /Users/pat/dev/openrewrite/rewrite/rewrite-json/src/main/antlr/JSON5.g4 by ANTLR 4.9.2
 import org.antlr.v4.runtime.tree.AbstractParseTreeVisitor;
 
 /**

diff --git a/rewrite-json/src/main/java/org/openrewrite/json/internal/grammar/JSON5Lexer.interp b/rewrite-json/src/main/java/org/openrewrite/json/internal/grammar/JSON5Lexer.interp