use ints not chars for C# (#3567)

* use ints not chars for C# * oops. remove 'quotes' * regen from XPathLexer.g4 * simplify ATN with bypass alts mechanism in Java. * Change string to int[] for serialized ATN for C#; removed unneeded `use System` from XPathLexer.g4; regen that grammar.
antlr · Mar 5, 2022 · de4470b · de4470b
1 parent 80a6a56
commit de4470b
Show file tree

Hide file tree

Showing 10 changed files with 63 additions and 96 deletions.
diff --git a/runtime/CSharp/src/Atn/ATNDeserializer.cs b/runtime/CSharp/src/Atn/ATNDeserializer.cs
@@ -33,10 +33,10 @@ public ATNDeserializer(ATNDeserializationOptions deserializationOptions)
             this.deserializationOptions = deserializationOptions;
         }
 
-		char[] data;
+		int[] data;
 		int p;
 
-        public virtual ATN Deserialize(char[] data)
+        public virtual ATN Deserialize(int[] data)
         {
 	        this.data = data;
 			CheckVersion ();

diff --git a/runtime/CSharp/src/Misc/RuleDependencyChecker.cs b/runtime/CSharp/src/Misc/RuleDependencyChecker.cs
@@ -330,12 +330,12 @@ private static void GetElementDependencies(ICustomAttributeProvider annotatedEle
 
         private static RuleDependencyChecker.RuleRelations ExtractRuleRelations(TypeInfo recognizer)
         {
-            string serializedATN = GetSerializedATN(recognizer);
+            int[] serializedATN = GetSerializedATN(recognizer);
             if (serializedATN == null)
             {
                 return null;
             }
-            ATN atn = new ATNDeserializer().Deserialize(serializedATN.ToCharArray());
+            ATN atn = new ATNDeserializer().Deserialize(serializedATN);
             RuleDependencyChecker.RuleRelations relations = new RuleDependencyChecker.RuleRelations(atn.ruleToStartState.Length);
             foreach (ATNState state in atn.states)
             {
@@ -356,11 +356,11 @@ private static RuleDependencyChecker.RuleRelations ExtractRuleRelations(TypeInfo
             return relations;
         }
 
-        private static string GetSerializedATN(TypeInfo recognizerClass)
+        private static int[] GetSerializedATN(TypeInfo recognizerClass)
         {
             FieldInfo serializedAtnField = recognizerClass.DeclaredFields.First(i => i.Name == "_serializedATN");
             if (serializedAtnField != null)
-                return (string)serializedAtnField.GetValue(null);
+                return (int[])serializedAtnField.GetValue(null);
 
             if (recognizerClass.BaseType != null)
                 return GetSerializedATN(recognizerClass.BaseType.GetTypeInfo());

diff --git a/runtime/CSharp/src/Parser.cs b/runtime/CSharp/src/Parser.cs
@@ -90,7 +90,8 @@ public virtual void ExitEveryRule(ParserRuleContext ctx)
         /// bypass alternatives.
         /// </summary>
         /// <seealso cref="Antlr4.Runtime.Atn.ATNDeserializationOptions.GenerateRuleBypassTransitions()"/>
-        private static readonly IDictionary<string, ATN> bypassAltsAtnCache = new Dictionary<string, ATN>();
+//        private static readonly IDictionary<string, ATN> bypassAltsAtnCache = new Dictionary<string, ATN>();
+        private ATN bypassAltsAtnCache;
 
         /// <summary>The error handling strategy for the parser.</summary>
         /// <remarks>
@@ -562,22 +563,20 @@ public virtual ITokenFactory TokenFactory
         [return: NotNull]
         public virtual ATN GetATNWithBypassAlts()
         {
-            string serializedAtn = SerializedAtn;
+            int[] serializedAtn = SerializedAtn;
             if (serializedAtn == null)
             {
                 throw new NotSupportedException("The current parser does not support an ATN with bypass alternatives.");
             }
-            lock (bypassAltsAtnCache)
+            lock (this)
             {
-                ATN result = bypassAltsAtnCache.Get(serializedAtn);
-                if (result == null)
-                {
-                    ATNDeserializationOptions deserializationOptions = new ATNDeserializationOptions();
-                    deserializationOptions.GenerateRuleBypassTransitions = true;
-                    result = new ATNDeserializer(deserializationOptions).Deserialize(serializedAtn.ToCharArray());
-                    bypassAltsAtnCache.Put(serializedAtn, result);
+                if ( bypassAltsAtnCache!=null ) {
+                    return bypassAltsAtnCache;
                 }
-                return result;
+                ATNDeserializationOptions deserializationOptions = new ATNDeserializationOptions();
+                deserializationOptions.GenerateRuleBypassTransitions = true;
+                bypassAltsAtnCache = new ATNDeserializer(deserializationOptions).Deserialize(serializedAtn);
+                return bypassAltsAtnCache;
             }
         }
 

diff --git a/runtime/CSharp/src/Recognizer.cs b/runtime/CSharp/src/Recognizer.cs
@@ -131,7 +131,7 @@ public virtual int GetTokenType(string tokenName)
         /// <p>For interpreters, we don't know their serialized ATN despite having
         /// created the interpreter from it.</p>
         /// </remarks>
-        public virtual string SerializedAtn
+        public virtual int[] SerializedAtn
         {
             [return: NotNull]
             get

diff --git a/runtime/CSharp/src/Tree/Pattern/ParseTreePatternMatcher.cs b/runtime/CSharp/src/Tree/Pattern/ParseTreePatternMatcher.cs
@@ -301,7 +301,11 @@ public virtual ParseTreePattern Compile(string pattern, int patternRuleIndex)
             IList<IToken> tokenList = Tokenize(pattern);
             ListTokenSource tokenSrc = new ListTokenSource(tokenList);
             CommonTokenStream tokens = new CommonTokenStream(tokenSrc);
-            ParserInterpreter parserInterp = new ParserInterpreter(parser.GrammarFileName, parser.Vocabulary, Arrays.AsList(parser.RuleNames), parser.GetATNWithBypassAlts(), tokens);
+            ParserInterpreter parserInterp = new ParserInterpreter(parser.GrammarFileName,
+                                                                   parser.Vocabulary,
+                                                                   Arrays.AsList(parser.RuleNames),
+                                                                   parser.GetATNWithBypassAlts(),
+                                                                   tokens);
             IParseTree tree = null;
             try
             {

diff --git a/runtime/CSharp/src/Tree/Xpath/XPathLexer.cs b/runtime/CSharp/src/Tree/Xpath/XPathLexer.cs
@@ -19,9 +19,6 @@
 // Ambiguous reference in cref attribute
 #pragma warning disable 419
 
-
-using System;
-
 using System;
 using System.IO;
 using System.Text;
@@ -46,7 +43,7 @@ public const int
 	};
 
 	public static readonly string[] ruleNames = {
-		"Anywhere", "Root", "Wildcard", "Bang", "ID", "NameChar", "NameStartChar",
+		"Anywhere", "Root", "Wildcard", "Bang", "ID", "NameChar", "NameStartChar", 
 		"String"
 	};
 
@@ -64,7 +61,7 @@ public XPathLexer(ICharStream input, TextWriter output, TextWriter errorOutput)
 		null, null, null, "'//'", "'/'", "'*'", "'!'"
 	};
 	private static readonly string[] _SymbolicNames = {
-		null, "TokenRef", "RuleRef", "Anywhere", "Root", "Wildcard", "Bang", "ID",
+		null, "TokenRef", "RuleRef", "Anywhere", "Root", "Wildcard", "Bang", "ID", 
 		"String"
 	};
 	public static readonly IVocabulary DefaultVocabulary = new Vocabulary(_LiteralNames, _SymbolicNames);
@@ -86,7 +83,7 @@ public override IVocabulary Vocabulary
 
 	public override string[] ModeNames { get { return modeNames; } }
 
-	public override string SerializedAtn { get { return new string(_serializedATN); } }
+	public override int[] SerializedAtn { get { return _serializedATN; } }
 
 	static XPathLexer() {
 		decisionToDFA = new DFA[_ATN.NumberOfDecisions];
@@ -101,61 +98,34 @@ public override void Action(RuleContext _localctx, int ruleIndex, int actionInde
 	}
 	private void ID_action(RuleContext _localctx, int actionIndex) {
 		switch (actionIndex) {
-		case 0:
+		case 0: 
 						String text = Text;
-						if ( Char.IsUpper(text[0]) )
+						if ( Char.IsUpper(text[0]) ) 
 							Type = TokenRef;
-						else
+						else 
 							Type = RuleRef;
 						 break;
 		}
 	}
 
-	private static char[] _serializedATN = {
-		'\x4', '\0', '\b', '\x32', '\x6', '\xFFFF', '\x2', '\0', '\a', '\0', '\x2',
-		'\x1', '\a', '\x1', '\x2', '\x2', '\a', '\x2', '\x2', '\x3', '\a', '\x3',
-		'\x2', '\x4', '\a', '\x4', '\x2', '\x5', '\a', '\x5', '\x2', '\x6', '\a',
-		'\x6', '\x2', '\a', '\a', '\a', '\x1', '\0', '\x1', '\0', '\x1', '\0',
-		'\x1', '\x1', '\x1', '\x1', '\x1', '\x2', '\x1', '\x2', '\x1', '\x3',
-		'\x1', '\x3', '\x1', '\x4', '\x1', '\x4', '\x5', '\x4', '\x1D', '\b',
-		'\x4', '\n', '\x4', '\f', '\x4', ' ', '\t', '\x4', '\x1', '\x4', '\x1',
-		'\x4', '\x1', '\x5', '\x1', '\x5', '\x3', '\x5', '&', '\b', '\x5', '\x1',
-		'\x6', '\x1', '\x6', '\x1', '\a', '\x1', '\a', '\x5', '\a', ',', '\b',
-		'\a', '\n', '\a', '\f', '\a', '/', '\t', '\a', '\x1', '\a', '\x1', '\a',
-		'\x1', '-', '\0', '\b', '\x1', '\x3', '\x3', '\x4', '\x5', '\x5', '\a',
-		'\x6', '\t', '\a', '\v', '\0', '\r', '\0', '\xF', '\b', '\x1', '\0', '\x2',
-		'\x5', '\0', '\x30', '\x39', '_', '_', '\xB7', '\xB7', '\x300', '\x36F',
-		'\x203F', '\x2040', '\r', '\0', '\x41', 'Z', '\x61', 'z', '\xC0', '\xD6',
-		'\xD8', '\xF6', '\xF8', '\x2FF', '\x370', '\x37D', '\x37F', '\x1FFF',
-		'\x200C', '\x200D', '\x2070', '\x218F', '\x2C00', '\x2FEF', '\x3001',
-		'\xD7FF', '\xF900', '\xFDCF', '\xFDF0', '\xFFFD', '\0', '\x32', '\0',
-		'\x1', '\x1', '\0', '\0', '\0', '\0', '\x3', '\x1', '\0', '\0', '\0',
-		'\0', '\x5', '\x1', '\0', '\0', '\0', '\0', '\a', '\x1', '\0', '\0', '\0',
-		'\0', '\t', '\x1', '\0', '\0', '\0', '\0', '\xF', '\x1', '\0', '\0', '\0',
-		'\x1', '\x11', '\x1', '\0', '\0', '\0', '\x3', '\x14', '\x1', '\0', '\0',
-		'\0', '\x5', '\x16', '\x1', '\0', '\0', '\0', '\a', '\x18', '\x1', '\0',
-		'\0', '\0', '\t', '\x1A', '\x1', '\0', '\0', '\0', '\v', '%', '\x1', '\0',
-		'\0', '\0', '\r', '\'', '\x1', '\0', '\0', '\0', '\xF', ')', '\x1', '\0',
-		'\0', '\0', '\x11', '\x12', '\x5', '/', '\0', '\0', '\x12', '\x13', '\x5',
-		'/', '\0', '\0', '\x13', '\x2', '\x1', '\0', '\0', '\0', '\x14', '\x15',
-		'\x5', '/', '\0', '\0', '\x15', '\x4', '\x1', '\0', '\0', '\0', '\x16',
-		'\x17', '\x5', '*', '\0', '\0', '\x17', '\x6', '\x1', '\0', '\0', '\0',
-		'\x18', '\x19', '\x5', '!', '\0', '\0', '\x19', '\b', '\x1', '\0', '\0',
-		'\0', '\x1A', '\x1E', '\x3', '\r', '\x6', '\0', '\x1B', '\x1D', '\x3',
-		'\v', '\x5', '\0', '\x1C', '\x1B', '\x1', '\0', '\0', '\0', '\x1D', ' ',
-		'\x1', '\0', '\0', '\0', '\x1E', '\x1C', '\x1', '\0', '\0', '\0', '\x1E',
-		'\x1F', '\x1', '\0', '\0', '\0', '\x1F', '!', '\x1', '\0', '\0', '\0',
-		' ', '\x1E', '\x1', '\0', '\0', '\0', '!', '\"', '\x6', '\x4', '\0', '\0',
-		'\"', '\n', '\x1', '\0', '\0', '\0', '#', '&', '\x3', '\r', '\x6', '\0',
-		'$', '&', '\a', '\0', '\0', '\0', '%', '#', '\x1', '\0', '\0', '\0', '%',
-		'$', '\x1', '\0', '\0', '\0', '&', '\f', '\x1', '\0', '\0', '\0', '\'',
-		'(', '\a', '\x1', '\0', '\0', '(', '\xE', '\x1', '\0', '\0', '\0', ')',
-		'-', '\x5', '\'', '\0', '\0', '*', ',', '\t', '\0', '\0', '\0', '+', '*',
-		'\x1', '\0', '\0', '\0', ',', '/', '\x1', '\0', '\0', '\0', '-', '.',
-		'\x1', '\0', '\0', '\0', '-', '+', '\x1', '\0', '\0', '\0', '.', '\x30',
-		'\x1', '\0', '\0', '\0', '/', '-', '\x1', '\0', '\0', '\0', '\x30', '\x31',
-		'\x5', '\'', '\0', '\0', '\x31', '\x10', '\x1', '\0', '\0', '\0', '\x4',
-		'\0', '\x1E', '%', '-', '\x1', '\x1', '\x4', '\0',
+	private static int[] _serializedATN = {
+		4,0,8,50,6,65535,2,0,7,0,2,1,7,1,2,2,7,2,2,3,7,3,2,4,7,4,2,5,7,5,2,6,7,
+		6,2,7,7,7,1,0,1,0,1,0,1,1,1,1,1,2,1,2,1,3,1,3,1,4,1,4,5,4,29,8,4,10,4,
+		12,4,32,9,4,1,4,1,4,1,5,1,5,3,5,38,8,5,1,6,1,6,1,7,1,7,5,7,44,8,7,10,7,
+		12,7,47,9,7,1,7,1,7,1,45,0,8,1,3,3,4,5,5,7,6,9,7,11,0,13,0,15,8,1,0,2,
+		5,0,48,57,95,95,183,183,768,879,8255,8256,13,0,65,90,97,122,192,214,216,
+		246,248,767,880,893,895,8191,8204,8205,8304,8591,11264,12271,12289,55295,
+		63744,64975,65008,65533,0,50,0,1,1,0,0,0,0,3,1,0,0,0,0,5,1,0,0,0,0,7,1,
+		0,0,0,0,9,1,0,0,0,0,15,1,0,0,0,1,17,1,0,0,0,3,20,1,0,0,0,5,22,1,0,0,0,
+		7,24,1,0,0,0,9,26,1,0,0,0,11,37,1,0,0,0,13,39,1,0,0,0,15,41,1,0,0,0,17,
+		18,5,47,0,0,18,19,5,47,0,0,19,2,1,0,0,0,20,21,5,47,0,0,21,4,1,0,0,0,22,
+		23,5,42,0,0,23,6,1,0,0,0,24,25,5,33,0,0,25,8,1,0,0,0,26,30,3,13,6,0,27,
+		29,3,11,5,0,28,27,1,0,0,0,29,32,1,0,0,0,30,28,1,0,0,0,30,31,1,0,0,0,31,
+		33,1,0,0,0,32,30,1,0,0,0,33,34,6,4,0,0,34,10,1,0,0,0,35,38,3,13,6,0,36,
+		38,7,0,0,0,37,35,1,0,0,0,37,36,1,0,0,0,38,12,1,0,0,0,39,40,7,1,0,0,40,
+		14,1,0,0,0,41,45,5,39,0,0,42,44,9,0,0,0,43,42,1,0,0,0,44,47,1,0,0,0,45,
+		46,1,0,0,0,45,43,1,0,0,0,46,48,1,0,0,0,47,45,1,0,0,0,48,49,5,39,0,0,49,
+		16,1,0,0,0,4,0,30,37,45,1,1,4,0
 	};
 
 	public static readonly ATN _ATN =

diff --git a/runtime/CSharp/src/Tree/Xpath/XPathLexer.g4 b/runtime/CSharp/src/Tree/Xpath/XPathLexer.g4
@@ -1,9 +1,5 @@
 lexer grammar XPathLexer;
 
-@header {
-using System; 
-}
-
 tokens { TokenRef, RuleRef }
 
 /*

diff --git a/runtime/Java/src/org/antlr/v4/runtime/Parser.java b/runtime/Java/src/org/antlr/v4/runtime/Parser.java
@@ -80,13 +80,14 @@ public void exitEveryRule(ParserRuleContext ctx) {
 	}
 
 	/**
-	 * This field maps from the serialized ATN string to the deserialized {@link ATN} with
-	 * bypass alternatives.
+	 * This field holds the deserialized {@link ATN} with bypass alternatives, created
+	 * lazily upon first demand. In 4.10 I changed from map<serializedATNstring, ATN>
+	 * since we only need one per parser object and also it complicates other targets
+	 * that don't use ATN strings.
 	 *
 	 * @see ATNDeserializationOptions#isGenerateRuleBypassTransitions()
 	 */
-	private static final Map<String, ATN> bypassAltsAtnCache =
-		new WeakHashMap<String, ATN>();
+	private ATN bypassAltsAtnCache;
 
 	/**
 	 * The error handling strategy for the parser. The default value is a new
@@ -445,16 +446,14 @@ public ATN getATNWithBypassAlts() {
 			throw new UnsupportedOperationException("The current parser does not support an ATN with bypass alternatives.");
 		}
 
-		synchronized (bypassAltsAtnCache) {
-			ATN result = bypassAltsAtnCache.get(serializedAtn);
-			if (result == null) {
-				ATNDeserializationOptions deserializationOptions = new ATNDeserializationOptions();
-				deserializationOptions.setGenerateRuleBypassTransitions(true);
-				result = new ATNDeserializer(deserializationOptions).deserialize(serializedAtn.toCharArray());
-				bypassAltsAtnCache.put(serializedAtn, result);
+		synchronized (this) {
+			if ( bypassAltsAtnCache!=null ) {
+				return bypassAltsAtnCache;
 			}
-
-			return result;
+			ATNDeserializationOptions deserializationOptions = new ATNDeserializationOptions();
+			deserializationOptions.setGenerateRuleBypassTransitions(true);
+			bypassAltsAtnCache = new ATNDeserializer(deserializationOptions).deserialize(serializedAtn.toCharArray());
+			return bypassAltsAtnCache;
 		}
 	}
 

diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/CSharp/CSharp.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/CSharp/CSharp.stg
@@ -292,7 +292,7 @@ public partial class <parser.name> : <superClass; null="Parser"> {
 
 	public override string[] RuleNames { get { return ruleNames; } }
 
-	public override string SerializedAtn { get { return new string(_serializedATN); } }
+	public override int[] SerializedAtn { get { return _serializedATN; } }
 
 	static <parser.name>() {
 		decisionToDFA = new DFA[_ATN.NumberOfDecisions];
@@ -1033,7 +1033,7 @@ public partial class <lexer.name> : <superClass; null="Lexer"> {
 
 	public override string[] ModeNames { get { return modeNames; } }
 
-	public override string SerializedAtn { get { return new string(_serializedATN); } }
+	public override int[] SerializedAtn { get { return _serializedATN; } }
 
 	static <lexer.name>() {
 		decisionToDFA = new DFA[_ATN.NumberOfDecisions];
@@ -1046,10 +1046,9 @@ public partial class <lexer.name> : <superClass; null="Lexer"> {
 }
 >>
 
-
 SerializedATN(model) ::= <<
-private static char[] _serializedATN = {
-	<model.serialized: {s | '<s>'}; separator=",", wrap>
+private static int[] _serializedATN = {
+	<model.serialized: {s | <s>}; separator=",", wrap>
 };
 
 public static readonly ATN _ATN =

diff --git a/tool/src/org/antlr/v4/codegen/target/CSharpTarget.java b/tool/src/org/antlr/v4/codegen/target/CSharpTarget.java
@@ -173,7 +173,7 @@ private void reportError(STMessage msg) {
 
 	@Override
 	public boolean isATNSerializedAsInts() {
-		return false;
+		return true;
 	}
 
 	@Override