Upgrade python version

.github/workflows/cibuildwheel.yml

@@ -0,0 +1,124 @@
+name: cibuildwheel
+# https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#concurrency
+# https://docs.github.com/en/developers/webhooks-and-events/events/github-event-types#pullrequestevent
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.number }}-${{ github.event.type }}
+  cancel-in-progress: true
+
+on: [push, pull_request]
+
+jobs:
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    env:
+      min-numpy-version: "1.19.5"
+      min-numpy-hash: "51/60/3f0fe5b7675a461d96b9d6729beecd3532565743278a9c3fe6dd09697fa7"
+      CIBW_ARCHS_MACOS: "x86_64 universal2 arm64"
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-18.04, windows-latest, macos-10.15]
+        cibw_archs: ["auto"]
+        include: 
+          - os: ubuntu-18.04
+            cibw_archs: "aarch64"
+
+    steps:
+      - name: Set up QEMU
+        if: matrix.cibw_archs == 'aarch64'
+        uses: docker/setup-qemu-action@v1
+        with:
+          platforms: arm64
+
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - uses: actions/setup-python@v2
+        name: Install Python
+        with:
+          python-version: '3.10'
+
+      - uses: actions/cache@v2
+        id: numpy-cache
+        with:
+          path: numpy-aarch64-cache/
+          key: numpy-${{ matrix.cibw_archs }}-cache-${{ env.min-numpy-version }}
+
+      - name: Install cibuildwheel
+        run: |
+          python -m pip install cibuildwheel==2.3.1
+
+      - name: Build minimum NumPy for aarch64
+        if: matrix.cibw_archs == 'aarch64' && steps.numpy-cache.outputs.cache-hit != 'true'
+        run: |
+          wget https://files.pythonhosted.org/packages/${{ env.min-numpy-hash }}/numpy-${{ env.min-numpy-version }}.zip
+          unzip numpy-${{ env.min-numpy-version }}.zip
+          cd numpy-${{ env.min-numpy-version }}
+          python -m cibuildwheel --output-dir ../numpy-aarch64-cache
+        env:
+          CIBW_BUILD: "cp36-* cp37-* cp38-* cp39-* cp310-*"
+          CIBW_ARCHS: aarch64
+
+      - name: Build wheels
+        uses: pypa/cibuildwheel@v2.3.1
+
+      - uses: actions/upload-artifact@v2
+        with:
+          path: ./wheelhouse/*.whl
+
+  build_sdist:
+    name: Build source distribution
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+
+    - uses: actions/setup-python@v2
+      name: Install Python
+      with:
+        python-version: '3.7'
+
+    - name: Build sdist
+      run: |
+        python -m pip install --upgrade pip setuptools build wheel
+        python -m build --sdist -o dist/
+
+    - uses: actions/upload-artifact@v2
+      with:
+        path: dist/*.tar.gz
+
+  # upload_test_pypi:
+  #   needs: [build_wheels, build_sdist]
+  #   runs-on: ubuntu-latest
+  #   # upload to Test PyPI for every commit on main branch
+  #   if: github.event_name == 'push' && github.event.ref == 'refs/heads/main'
+  #   steps:
+  #     - uses: actions/download-artifact@v2
+  #       with:
+  #         name: artifact
+  #         path: dist
+  #     - uses: pypa/gh-action-pypi-publish@master
+  #       with:
+  #         user: __token__
+  #         password: ${{ secrets.LABGRAPH_TEST_PYPI_TOKEN }}
+  #         repository_url: https://test.pypi.org/legacy/
+
+  # upload_pypi:
+  #   needs: [build_wheels, build_sdist]
+  #   runs-on: ubuntu-latest
+  #   # upload to PyPI on every tag starting with 'v'
+  #   if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')
+  #   # alternatively, to publish when a GitHub Release is created, use the following rule:
+  #   # if: github.event_name == 'release' && github.event.action == 'published'
+  #   steps:
+  #   - uses: actions/download-artifact@v2
+  #     with:
+  #       name: artifact
+  #       path: dist
+  #   - uses: pypa/gh-action-pypi-publish@master
+  #     with:
+  #       user: __token__
+  #       password: ${{ secrets.LABGRAPH_PYPI_TOKEN }}

Dockerfile

@@ -12,7 +12,7 @@ SHELL ["/bin/bash", "--login", "-c"]
 RUN g++ --version
 
 # Install Python, Java, wget, vim
-RUN yum install -y python2 python36 python36-devel wget java-1.8.0-openjdk \
+RUN yum install -y python2 python3 python3-devel wget java-1.8.0-openjdk \
     java-1.8.0-openjdk-devel vim
 
 # Install Ant
@@ -32,7 +32,7 @@ ENV JAVA_HOME="/usr/lib/jvm/java-1.8.0-openjdk"
 # Build Buck
 WORKDIR "/opt/buck"
 RUN ant
-Run ln -s /opt/buck/bin/buck /usr/bin/buck
+RUN ln -s /opt/buck/bin/buck /usr/bin/buck
 
 # Install Watchman
 WORKDIR "/opt/watchman"
@@ -45,25 +45,117 @@ RUN cp lib/* /usr/local/lib
 RUN chmod 755 /usr/local/bin/watchman
 RUN chmod 2777 /usr/local/var/run/watchman
 
+# Unpack the static python libraries from the manylinux image.
+WORKDIR "/opt/_internal/"
+RUN XZ_OPT=-9e tar -xf static-libs-for-embedding-only.tar.xz cpython-*/lib/libpython*.a
+
 # Copy LabGraph files
-WORKDIR "/opt/labgraph"
+WORKDIR "/opt/labgraph/"
+
+# Copy labgraph into the container
 COPY . .
 
+# Create a user to act as the builder.
+# This is done to prevent root installs with pip.
+RUN useradd -m -r builder && \
+    chown -R builder /opt/labgraph
+USER builder
+
+# Build LabGraph Wheel
+RUN python3.6 -m pip install build && \
+ python3.6 -m build --sdist --wheel 
+# Build LabGraph Wheel
+RUN sed -i 's/3.6/3.7/' /opt/labgraph/third-party/python/DEFS && \
+ python3.7 -m pip install build && \
+ python3.7 -m build --sdist --wheel 
+# Build LabGraph Wheel
+RUN sed -i 's/3.7/3.8/' /opt/labgraph/third-party/python/DEFS && \
+ python3.8 -m pip install build && \
+ python3.8 -m build --sdist --wheel
+# Build LabGraph Wheel
+RUN sed -i 's/3.8/3.9/' /opt/labgraph/third-party/python/DEFS && \
+ python3.9 -m pip install build && \
+ python3.9 -m build --sdist --wheel
 # Build LabGraph Wheel
-RUN python3.6 setup_py36.py install --user
-RUN python3.6 setup_py36.py sdist bdist_wheel
-RUN python3.6 -m pip install auditwheel
-RUN auditwheel repair dist/*whl -w dist/
-
-# Test LabGraph
-RUN python3.6 -m pytest --pyargs -v labgraph._cthulhu
-RUN python3.6 -m pytest --pyargs -v labgraph.events
-RUN python3.6 -m pytest --pyargs -v labgraph.graphs
-RUN python3.6 -m pytest --pyargs -v labgraph.loggers
-RUN python3.6 -m pytest --pyargs -v labgraph.messages
-RUN python3.6 -m pytest --pyargs -v labgraph.runners.tests.test_process_manager
-RUN python3.6 -m pytest --pyargs -v labgraph.runners.tests.test_aligner
-RUN python3.6 -m pytest --pyargs -v labgraph.runners.tests.test_cpp
-RUN python3.6 -m pytest --pyargs -v labgraph.runners.tests.test_exception
-RUN python3.6 -m pytest --pyargs -v labgraph.runners.tests.test_launch
-RUN python3.6 -m pytest --pyargs -v labgraph.runners.tests.test_runner
+RUN sed -i 's/3.9/3.10/' /opt/labgraph/third-party/python/DEFS && \
+ python3.10 -m pip install build && \
+ python3.10 -m build --sdist --wheel --no-isolation
+
+# Build wheels for each python version
+RUN find dist/*whl -exec auditwheel repair {} -w dist/ \;
+
+# Test LabGraph for python3.6
+RUN python3.6 -m pip install \
+ dist/labgraph-2.0.0-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl && \
+ python3.6 -m pytest --pyargs -v labgraph._cthulhu && \
+ python3.6 -m pytest --pyargs -v labgraph.events && \
+ python3.6 -m pytest --pyargs -v labgraph.graphs && \
+ python3.6 -m pytest --pyargs -v labgraph.loggers && \
+ python3.6 -m pytest --pyargs -v labgraph.messages && \
+ python3.6 -m pytest --pyargs -v labgraph.runners.tests.test_process_manager && \
+ python3.6 -m pytest --pyargs -v labgraph.runners.tests.test_aligner && \
+ python3.6 -m pytest --pyargs -v labgraph.runners.tests.test_cpp && \
+ python3.6 -m pytest --pyargs -v labgraph.runners.tests.test_exception && \
+ python3.6 -m pytest --pyargs -v labgraph.runners.tests.test_launch && \
+ python3.6 -m pytest --pyargs -v labgraph.runners.tests.test_runner
+
+# Test LabGraph for python3.7
+RUN python3.7 -m pip install \
+ dist/labgraph-2.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl && \
+ python3.7 -m pytest --pyargs -v labgraph._cthulhu && \
+ python3.7 -m pytest --pyargs -v labgraph.events && \
+ python3.7 -m pytest --pyargs -v labgraph.graphs && \
+ python3.7 -m pytest --pyargs -v labgraph.loggers && \
+ python3.7 -m pytest --pyargs -v labgraph.messages && \
+ python3.7 -m pytest --pyargs -v labgraph.runners.tests.test_process_manager && \
+ python3.7 -m pytest --pyargs -v labgraph.runners.tests.test_aligner && \
+ python3.7 -m pytest --pyargs -v labgraph.runners.tests.test_cpp && \
+ python3.7 -m pytest --pyargs -v labgraph.runners.tests.test_exception && \
+ python3.7 -m pytest --pyargs -v labgraph.runners.tests.test_launch && \
+ python3.7 -m pytest --pyargs -v labgraph.runners.tests.test_runner
+
+# Test LabGraph for python3.8
+RUN python3.8 -m pip install \
+ dist/labgraph-2.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl && \
+ python3.8 -m pytest --pyargs -v labgraph._cthulhu && \
+ python3.8 -m pytest --pyargs -v labgraph.events && \
+ python3.8 -m pytest --pyargs -v labgraph.graphs && \
+ python3.8 -m pytest --pyargs -v labgraph.loggers && \
+ python3.8 -m pytest --pyargs -v labgraph.messages && \
+ python3.8 -m pytest --pyargs -v labgraph.runners.tests.test_process_manager && \
+ python3.8 -m pytest --pyargs -v labgraph.runners.tests.test_aligner && \
+ python3.8 -m pytest --pyargs -v labgraph.runners.tests.test_cpp && \
+ python3.8 -m pytest --pyargs -v labgraph.runners.tests.test_exception && \
+ python3.8 -m pytest --pyargs -v labgraph.runners.tests.test_launch && \
+ python3.8 -m pytest --pyargs -v labgraph.runners.tests.test_runner
+
+# Test LabGraph for python3.9
+RUN python3.9 -m pip install \
+ dist/labgraph-2.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl && \
+ python3.9 -m pytest --pyargs -v labgraph._cthulhu && \
+ python3.9 -m pytest --pyargs -v labgraph.events && \
+ python3.9 -m pytest --pyargs -v labgraph.graphs && \
+ python3.9 -m pytest --pyargs -v labgraph.loggers && \
+ python3.9 -m pytest --pyargs -v labgraph.messages && \
+ python3.9 -m pytest --pyargs -v labgraph.runners.tests.test_process_manager && \
+ python3.9 -m pytest --pyargs -v labgraph.runners.tests.test_aligner && \
+ python3.9 -m pytest --pyargs -v labgraph.runners.tests.test_cpp && \
+ python3.9 -m pytest --pyargs -v labgraph.runners.tests.test_exception && \
+ python3.9 -m pytest --pyargs -v labgraph.runners.tests.test_launch && \
+ python3.9 -m pytest --pyargs -v labgraph.runners.tests.test_runner
+
+# Test LabGraph for python3.10
+RUN python3.10 -m pip install \
+ dist/labgraph-2.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl && \
+ python3.10 -m pytest --pyargs -v labgraph._cthulhu && \
+ python3.10 -m pytest --pyargs -v labgraph.events && \
+ python3.10 -m pytest --pyargs -v labgraph.graphs && \
+ python3.10 -m pytest --pyargs -v labgraph.loggers && \
+ python3.10 -m pytest --pyargs -v labgraph.messages && \
+ python3.10 -m pytest --pyargs -v labgraph.runners.tests.test_process_manager && \
+ python3.10 -m pytest --pyargs -v labgraph.runners.tests.test_aligner && \
+ python3.10 -m pytest --pyargs -v labgraph.runners.tests.test_cpp && \
+ python3.10 -m pytest --pyargs -v labgraph.runners.tests.test_exception && \
+ python3.10 -m pytest --pyargs -v labgraph.runners.tests.test_launch && \
+ python3.10 -m pytest --pyargs -v labgraph.runners.tests.test_runner
+

buck_ext.py

@@ -12,10 +12,10 @@
 
 CONFIG_FILE = {
     "Windows": "win.buckconfig",
-}.get(platform.system(), "unix.buckconfig")
+    "Darwin" : "unix.buckconfig",
+    "Linux" : "manylinux.buckconfig",
+}[platform.system()]
 
-if platform.system() == 'Linux' and platform.linux_distribution()[0] == 'CentOS Linux':
-    CONFIG_FILE = "manylinux.buckconfig"
 
 class BuckExtension(Extension):
     def __init__(self, name: str, target: str) -> None:

extensions/labgraph_protocol/README.md → extensions/labgraph_protocol/labgraph_protocol/README.md

@@ -12,17 +12,15 @@ Make sure to install labgraph before proceeding
 
 ```
 cd labgraph/extensions/labgraph_protocol
-# HACK: PyQt5-sip has to be installed before PyQt5
-pip install PyQt5-sip==4.19.18
-python setup.py install
+python -m pip install .
 ```
 
 ### Testing:
 
 To make sure things are working you can run any of the following examples
 
 ```
-python -m extensions.labgraph_protocol.labgraph_protocol.examples PROTOCOL_NAME
+python -m labgraph_protocol.examples PROTOCOL_NAME
 ```
 `PROTOCOL_NAME` can be any of:
 - `audio_player`

extensions/labgraph_protocol/setup.py

@@ -11,10 +11,10 @@
     packages=find_packages(),
     python_requires=">=3.6",
     install_requires=[
-        "dataclasses==0.6",
+        "dataclasses>=0.6",
         "labgraph>=2.0.0",
-        "PyQt5-sip==4.19.18",
-        "PyQt5==5.13.0",
+        "PyQt5-sip>=4.19.18",
+        "PyQt5>=5.13.0",
     ],
     include_package_data=True,
 )

extensions/labgraph_viz/labgraph_viz/README.md

@@ -12,20 +12,20 @@ Make sure to install labgraph before proceeding
 
 ```
 cd labgraph/extensions/labgraph_viz
-python setup.py install
+python -m pip install .
 ```
 
 ### Testing:
 
 To make sure things are working you can run any of the following examples
 
 ```
-python -m extensions.labgraph_viz.labgraph_viz.application_example
-python -m extensions.labgraph_viz.labgraph_viz.bar_plot_example
-python -m extensions.labgraph_viz.labgraph_viz.heat_map_example
-python -m extensions.labgraph_viz.labgraph_viz.line_plot_example
-python -m extensions.labgraph_viz.labgraph_viz.scatter_plot_example
-python -m extensions.labgraph_viz.labgraph_viz.spatial_plot_example
+python -m labgraph_viz.examples.application_example
+python -m labgraph_viz.examples.bar_plot_example
+python -m labgraph_viz.examples.heat_map_example
+python -m labgraph_viz.examples.line_plot_example
+python -m labgraph_viz.examples.scatter_plot_example
+python -m labgraph_viz.examples.spatial_plot_example
 ```
 
 ### Contributers:

extensions/labgraph_viz/labgraph_viz/application/application.py

@@ -50,7 +50,7 @@ def run_plot(self) -> None:
         else:
             for plot in self.plots:
                 plot.stop()
-    
+
     # Using a very simple layout
     # You can subclass and override this method to use another layout
     def setup_window(self) -> None:

extensions/labgraph_viz/labgraph_viz/examples/application_example.py

@@ -47,8 +47,8 @@ class Generator(lg.Node):
     async def generate_noise(self) -> lg.AsyncPublisher:
         while True:
             yield self.BARPLOT_OUTPUT, BarPlotMessage(
-                domain=np.arange(self.config.num_features + 1),
-                range=np.random.rand(self.config.num_features),
+                domain=np.arange(self.config.num_features + 1, dtype=np.int32),
+                range=np.random.rand(self.config.num_features).astype(np.float64),
             )
             await asyncio.sleep(1 / self.config.sample_rate)
 
@@ -107,7 +107,7 @@ def setup(self) -> None:
 
         # Add plots to application
         self.application.plots = [self.line_plot, self.bar_plot]
-    
+
     # Connect the Generator outputs to the Plot inputs
     def connections(self) -> lg.Connections:
         return (

extensions/labgraph_viz/labgraph_viz/examples/bar_plot_example.py

@@ -38,8 +38,8 @@ class Generator(lg.Node):
     async def generate_noise(self) -> lg.AsyncPublisher:
         while True:
             yield self.OUTPUT, RandomMessage(
-                domain=np.arange(self.config.num_features + 1),
-                range=np.random.rand(self.config.num_features),
+                domain=np.arange(self.config.num_features + 1, dtype=np.int32),
+                range=np.random.rand(self.config.num_features).astype(np.float64),
             )
             await asyncio.sleep(1 / self.config.sample_rate)