add e2e pyodide test!!

Now just to make it machine independent and update CI

tests/conftest.py

@@ -85,7 +85,10 @@ def fn(*args):
         assert fpath.exists()
 
         if runtime == "pyodide":
-            raise NotImplementedError(f"uri_for: {runtime=}")
+            # Pyodide cannot see the whole file system, so this needs to be made relative to
+            # the workspace's parent folder
+            path = str(fpath).replace(str(WORKSPACE_DIR.parent), "")
+            return uris.from_fs_path(path)
 
         elif runtime == "wasi":
             # WASI cannot see the whole filesystem, so this needs to be made relative to
@@ -124,6 +127,32 @@ async def fn(server_name: str):
     return fn
 
 
+def get_client_for_pyodide_server(uri_fixture):
+    """Return a client configured to communicate with a server running under Pyodide.
+
+    This assumes that the pyodide environment has already been bootstrapped.
+    """
+
+    async def fn(server_name: str):
+        client = LanguageClient("pygls-test-suite", "v1")
+
+        PYODIDE_DIR = REPO_DIR / "tests" / "pyodide"
+        server_py = str(SERVER_DIR / server_name)
+
+        await client.start_io("node", str(PYODIDE_DIR / "run_server.js"), server_py)
+
+        response = await client.initialize(
+            types.InitializeParams(
+                capabilities=types.ClientCapabilities(),
+                root_uri=uri_fixture(""),
+            )
+        )
+        assert response is not None
+        return client, response
+
+    return fn
+
+
 def get_client_for_wasi_server(uri_fixture):
     """Return a client configured to communicate with a server running under WASI.
 
@@ -161,9 +190,12 @@ def get_client_for(runtime, uri_for):
     It's the consuming fixture's responsibility to stop the client.
     """
     # TODO: Add TCP/WS support.
-    if runtime == "pyodide":
+    if runtime not in {"cpython", "pyodide", "wasi"}:
         raise NotImplementedError(f"get_client_for: {runtime=}")
 
+    elif runtime == "pyodide":
+        return get_client_for_pyodide_server(uri_for)
+
     elif runtime == "wasi":
         return get_client_for_wasi_server(uri_for)
 

tests/pyodide/.gitignore

@@ -0,0 +1,2 @@
+*.log
+node_modules/

tests/pyodide/bootstrap.py

@@ -0,0 +1,96 @@
+import pathlib
+import subprocess
+import sys
+from typing import Optional
+
+# Common paths
+REPO = pathlib.Path(__file__).parent.parent.parent
+PYODIDE_DIR = REPO / "tests" / "pyodide"
+PYODIDE_INDEX = PYODIDE_DIR / "node_modules" / "pyodide"
+
+
+def build_pygls():
+    """Build pygls' whl file and place it in pyodide's local index."""
+    run(
+        sys.executable,
+        "-m",
+        "build",
+        "--wheel",
+        "--outdir",
+        str(PYODIDE_INDEX),
+        cwd=str(REPO),
+    )
+
+
+def install_pyodide():
+    """Install pyodide and related node dependencies."""
+    run("npm", "ci", cwd=str(PYODIDE_DIR))
+
+
+def download_dependencies():
+    """Download pygls' dependencies so that we have the wheels locally for pyodide to
+    use."""
+    requirements = PYODIDE_DIR / "requirements.txt"
+
+    run(
+        "poetry",
+        "export",
+        "-f",
+        "requirements.txt",
+        "--output",
+        str(requirements),
+        cwd=str(REPO),
+    )
+
+    # Ensure that pip uses packages compatible with the pyodide runtime.
+    run(
+        "pip",
+        "download",
+        "--no-deps",
+        "--python-version",
+        "3.11",  # The version of Python pyodide compiled to WASM
+        "--implementation",
+        "py",  # Use only pure python packages.
+        "-r",
+        str(requirements),
+        "--dest",
+        str(PYODIDE_INDEX),
+        cwd=str(REPO),
+    )
+
+
+def run(*cmd, cwd: Optional[str] = None, capture: bool = False) -> Optional[str]:
+    """Run a command."""
+
+    result = subprocess.run(cmd, cwd=cwd, capture_output=capture)
+    if result.returncode != 0:
+        if capture:
+            sys.stdout.buffer.write(result.stdout)
+            sys.stdout.flush()
+            sys.stderr.buffer.write(result.stderr)
+            sys.stderr.flush()
+
+        sys.exit(result.returncode)
+
+    if capture:
+        return result.stdout.decode("utf8").strip()
+
+    return None
+
+
+def main():
+    """Bootstrap the pyodide environment."""
+    install_pyodide()
+
+    # NOTE: Disabled for now as it's non-trivial to get mircopip to look in the local
+    # folder in the general case - we'd need to implement PyPi's JSON API!
+    #
+    # download_dependencies()
+
+    build_pygls()
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/pyodide/package.json

@@ -0,0 +1,10 @@
+{
+    "name": "pyodide_tests",
+    "version": "0.0.0",
+    "description": "Simple wrapper that executes pygls servers in Pyodide",
+    "main": "run_server.js",
+    "author": "openlawlibrary",
+    "dependencies": {
+        "pyodide": "^0.24.1"
+    }
+}

tests/pyodide/run_server.js

@@ -0,0 +1,86 @@
+const fs = require('fs');
+const path = require('path')
+const { loadPyodide } = require('pyodide');
+
+const consoleLog = console.log
+
+const WORKSPACE = path.join(__dirname, "..", "..", "examples", "servers", "workspace")
+
+// Create a file to log pyodide output to.
+const logFile = fs.createWriteStream("pyodide.log")
+
+function writeToFile(...args) {
+    logFile.write(args[0] + `\n`);
+}
+
+// Load the workspace into the pyodide runtime.
+//
+// Unlike WASI, there is no "just works" solution for exposing the workspace/ folder
+// to the runtime - it's up to us to manually copy it into pyodide's in-memory filesystem.
+function loadWorkspace(pyodide) {
+    const FS = pyodide.FS
+
+    // Create a folder for the workspace to be copied into.
+    FS.mkdir('/workspace')
+
+    const workspace = fs.readdirSync(WORKSPACE)
+    workspace.forEach((file) => {
+
+        try {
+            const filename = "/" + path.join("workspace", file)
+            // consoleLog(`${file} -> ${filename}`)
+
+            const stream = FS.open(filename, 'w+')
+            const data = fs.readFileSync(path.join(WORKSPACE, file))
+
+            FS.write(stream, data, 0, data.length, 0)
+            FS.close(stream)
+        } catch (err) {
+            consoleLog(err)
+        }
+    })
+}
+
+async function runServer(serverCode) {
+    // Annoyingly, while we can redirect stderr/stdout to a file during this setup stage
+    // it doesn't prevent `micropip.install` from indirectly writing to console.log.
+    //
+    // Internally, `micropip.install` calls `pyodide.loadPackage` and doesn't expose loadPacakge's
+    // options for redirecting output i.e. messageCallback.
+    //
+    // So instead, we override console.log globally.
+    console.log = writeToFile
+    const pyodide = await loadPyodide({
+        // stdin:
+        stderr: writeToFile,
+    })
+
+    loadWorkspace(pyodide)
+
+    await pyodide.loadPackage("micropip")
+    const micropip = pyodide.pyimport("micropip")
+    // TODO: Make machine independent
+    await micropip.install("file:///var/home/alex/Projects/pygls-next/tests/pyodide/node_modules/pyodide/pygls-1.2.1-py3-none-any.whl")
+
+    // Restore the original console.log
+    console.log = consoleLog
+    await pyodide.runPythonAsync(serverCode)
+}
+
+if (process.argv.length < 3) {
+    console.error("Missing server.py file")
+    process.exit(1)
+}
+
+const serverCode = fs.readFileSync(process.argv[2], 'utf8')
+let returnCode = 0
+logFile.once('open', (fd) => {
+    runServer(serverCode).then(() => {
+        logFile.end();
+        process.exit(0)
+    }).catch(err => {
+        logFile.write(`Error in server process\n${err}`)
+        logFile.end();
+        process.exit(1);
+    })
+})