Merge pull request #120 from amaanq/ident-and-toplevel

Ident and toplevel
tree-sitter · Jun 6, 2023 · 3a3a8ee · 3a3a8ee
2 parents 8867111 + 88c19a2
commit 3a3a8ee
Show file tree

Hide file tree

Showing 3 changed files with 27,675 additions and 27,063 deletions.
diff --git a/grammar.js b/grammar.js
@@ -15,11 +15,6 @@ const
   comparative_operators = ['==', '!=', '<', '<=', '>', '>='],
   assignment_operators = multiplicative_operators.concat(additive_operators).map(operator => operator + '=').concat('='),
 
-  unicodeLetter = /\p{L}/,
-  unicodeDigit = /[0-9]/,
-  unicodeChar = /./,
-  unicodeValue = unicodeChar,
-  letter = choice(unicodeLetter, '_'),
 
   newline = '\n',
   terminator = choice(newline, ';'),
@@ -84,12 +79,9 @@ module.exports = grammar({
     [$.qualified_type, $._expression],
     [$.generic_type, $._expression],
     [$.generic_type, $._simple_type],
-    [$.parameter_declaration, $.type_arguments],
     [$.parameter_declaration, $._simple_type, $._expression],
     [$.parameter_declaration, $.generic_type, $._expression],
     [$.parameter_declaration, $._expression],
-    [$.func_literal, $.function_type],
-    [$.function_type],
     [$.parameter_declaration, $._simple_type],
   ],
 
@@ -102,12 +94,15 @@ module.exports = grammar({
   ],
 
   rules: {
-    source_file: $ => repeat(choice(
+    source_file: $ => seq(
+      repeat(choice(
       // Unlike a Go compiler, we accept statements at top-level to enable
       // parsing of partial code snippets in documentation (see #63).
       seq($._statement, terminator),
-      seq($._top_level_declaration, optional(terminator)),
-    )),
+      seq($._top_level_declaration, terminator),
+      )),
+      optional($._top_level_declaration),
+    ),
 
     _top_level_declaration: $ => choice(
       $.package_clause,
@@ -829,10 +824,7 @@ module.exports = grammar({
       field('name', $._type_identifier)
     ),
 
-    identifier: $ => token(seq(
-      letter,
-      repeat(choice(letter, unicodeDigit))
-    )),
+    identifier: _ => /[_\p{XID_Start}][_\p{XID_Continue}]*/,
 
     _type_identifier: $ => alias($.identifier, $.type_identifier),
     _field_identifier: $ => alias($.identifier, $.field_identifier),

diff --git a/src/grammar.json b/src/grammar.json
@@ -3,42 +3,42 @@
   "word": "identifier",
   "rules": {
     "source_file": {
-      "type": "REPEAT",
-      "content": {
-        "type": "CHOICE",
-        "members": [
-          {
-            "type": "SEQ",
+      "type": "SEQ",
+      "members": [
+        {
+          "type": "REPEAT",
+          "content": {
+            "type": "CHOICE",
             "members": [
               {
-                "type": "SYMBOL",
-                "name": "_statement"
-              },
-              {
-                "type": "CHOICE",
+                "type": "SEQ",
                 "members": [
                   {
-                    "type": "STRING",
-                    "value": "\n"
+                    "type": "SYMBOL",
+                    "name": "_statement"
                   },
                   {
-                    "type": "STRING",
-                    "value": ";"
+                    "type": "CHOICE",
+                    "members": [
+                      {
+                        "type": "STRING",
+                        "value": "\n"
+                      },
+                      {
+                        "type": "STRING",
+                        "value": ";"
+                      }
+                    ]
                   }
                 ]
-              }
-            ]
-          },
-          {
-            "type": "SEQ",
-            "members": [
-              {
-                "type": "SYMBOL",
-                "name": "_top_level_declaration"
               },
               {
-                "type": "CHOICE",
+                "type": "SEQ",
                 "members": [
+                  {
+                    "type": "SYMBOL",
+                    "name": "_top_level_declaration"
+                  },
                   {
                     "type": "CHOICE",
                     "members": [
@@ -51,16 +51,25 @@
                         "value": ";"
                       }
                     ]
-                  },
-                  {
-                    "type": "BLANK"
                   }
                 ]
               }
             ]
           }
-        ]
-      }
+        },
+        {
+          "type": "CHOICE",
+          "members": [
+            {
+              "type": "SYMBOL",
+              "name": "_top_level_declaration"
+            },
+            {
+              "type": "BLANK"
+            }
+          ]
+        }
+      ]
     },
     "_top_level_declaration": {
       "type": "CHOICE",
@@ -4332,50 +4341,8 @@
       ]
     },
     "identifier": {
-      "type": "TOKEN",
-      "content": {
-        "type": "SEQ",
-        "members": [
-          {
-            "type": "CHOICE",
-            "members": [
-              {
-                "type": "PATTERN",
-                "value": "\\p{L}"
-              },
-              {
-                "type": "STRING",
-                "value": "_"
-              }
-            ]
-          },
-          {
-            "type": "REPEAT",
-            "content": {
-              "type": "CHOICE",
-              "members": [
-                {
-                  "type": "CHOICE",
-                  "members": [
-                    {
-                      "type": "PATTERN",
-                      "value": "\\p{L}"
-                    },
-                    {
-                      "type": "STRING",
-                      "value": "_"
-                    }
-                  ]
-                },
-                {
-                  "type": "PATTERN",
-                  "value": "[0-9]"
-                }
-              ]
-            }
-          }
-        ]
-      }
+      "type": "PATTERN",
+      "value": "[_\\p{XID_Start}][_\\p{XID_Continue}]*"
     },
     "_type_identifier": {
       "type": "ALIAS",
@@ -6809,10 +6776,6 @@
       "generic_type",
       "_simple_type"
     ],
-    [
-      "parameter_declaration",
-      "type_arguments"
-    ],
     [
       "parameter_declaration",
       "_simple_type",
@@ -6827,13 +6790,6 @@
       "parameter_declaration",
       "_expression"
     ],
-    [
-      "func_literal",
-      "function_type"
-    ],
-    [
-      "function_type"
-    ],
     [
       "parameter_declaration",
       "_simple_type"