python setup

torch-mlir pip dep buildifier use python 3.10 llvm/llvm-project#75963 fix fixes --test_output=errors for showing errors on GHA cleanup minor add py_test fix fix paths fix add python tests to test_suite headers enable python lit tests buildifier clean disk space fix bump actions versions fix fix fix update requirements minor try stablehlo without test_output cleanup specify shell disk space cleanup trial fix more fixes fix fix
cruise-automation · Feb 11, 2024 · 11f3e27 · 11f3e27
1 parent b0d474e
commit 11f3e27
Show file tree

Hide file tree

Showing 30 changed files with 514 additions and 93 deletions.
diff --git a/.github/actions/setup-build/action.yml b/.github/actions/setup-build/action.yml
@@ -0,0 +1,64 @@
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+# Also available under a BSD-style license. See LICENSE.
+
+name: "Setup build environment"
+description: "Setup build environment"
+
+inputs:
+  cache-prefix:
+    required: true
+    default: 'mlir-tcp'
+
+runs:
+  # This is a composite action - has a list of steps to execute.
+  using: "composite"
+
+  steps:
+    # https://github.com/jlumbroso/free-disk-space/blob/main/action.yml
+    - name: Free disk space
+      shell: bash
+      run: |
+        df -h
+        sudo swapoff -a
+        sudo rm -f /swapfile
+        sudo rm -f /mnt/swapfile
+        free -h
+        sudo rm -rf /usr/share/dotnet
+        sudo rm -rf /usr/local/lib/android
+        sudo rm -rf /opt/ghc
+        sudo rm -rf "/usr/local/share/boost"
+        sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+        sudo apt-get remove -y '^aspnetcore-.*' || echo "::warning::The command [sudo apt-get remove -y '^aspnetcore-.*'] failed to complete successfully. Proceeding..."
+        sudo apt-get remove -y '^dotnet-.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y '^dotnet-.*' --fix-missing] failed to complete successfully. Proceeding..."
+        sudo apt-get remove -y '^llvm-.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y '^llvm-.*' --fix-missing] failed to complete successfully. Proceeding..."
+        sudo apt-get remove -y 'php.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y 'php.*' --fix-missing] failed to complete successfully. Proceeding..."
+        sudo apt-get remove -y '^mongodb-.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y '^mongodb-.*' --fix-missing] failed to complete successfully. Proceeding..."
+        sudo apt-get remove -y '^mysql-.*' --fix-missing || echo "::warning::The command [sudo apt-get remove -y '^mysql-.*' --fix-missing] failed to complete successfully. Proceeding..."
+        sudo apt-get remove -y azure-cli google-chrome-stable firefox powershell mono-devel libgl1-mesa-dri --fix-missing || echo "::warning::The command [sudo apt-get remove -y azure-cli google-chrome-stable firefox powershell mono-devel libgl1-mesa-dri --fix-missing] failed to complete successfully. Proceeding..."
+        sudo apt-get remove -y google-cloud-sdk --fix-missing || echo "::debug::The command [sudo apt-get remove -y google-cloud-sdk --fix-missing] failed to complete successfully. Proceeding..."
+        sudo apt-get remove -y google-cloud-cli --fix-missing || echo "::debug::The command [sudo apt-get remove -y google-cloud-cli --fix-missing] failed to complete successfully. Proceeding..."
+        sudo apt-get autoremove -y || echo "::warning::The command [sudo apt-get autoremove -y] failed to complete successfully. Proceeding..."
+        sudo apt-get clean || echo "::warning::The command [sudo apt-get clean] failed to complete successfully. Proceeding..."
+        docker rmi $(docker image ls -aq)
+        sudo docker image prune --all --force || true
+        df -h
+
+    # Continually update cache even if there's a "hit" during
+    # restore to avoid the cache going stale over time
+    # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache
+    - name: Setup cache for bazel
+      uses: actions/cache@v4
+      with:
+        path: ~/.cache/bazel
+        key: ${{ inputs.cache-prefix }}-bazel-build-cache-${{ runner.os }}-${{ github.sha }}
+        restore-keys: |
+          ${{ inputs.cache-prefix }}-bazel-build-cache-${{ runner.os }}
+
+    - name: Create bazel cache dir when not found
+      shell: bash
+      run: |
+        if [ ! -d "${HOME}/.cache/bazel" ]; then
+          mkdir -p "${HOME}/.cache/bazel"
+        fi
diff --git a/.github/workflows/bazelBuildAndTestLlvm.yml b/.github/workflows/bazelBuildAndTestLlvm.yml
@@ -37,24 +37,12 @@ jobs:
 
     steps:
     - name: Checkout mlir-tcp
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
-    # Continually update cache even if there's a "hit" during
-    # restore to avoid the cache going stale over time
-    # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache
-    - name: Setup cache for bazel
-      uses: actions/cache@v3
+    - name: Setup workspace
+      uses: ./.github/actions/setup-build
       with:
-        path: ~/.cache/bazel
-        key: llvm-project-bazel-build-cache-${{ runner.os }}-${{ github.sha }}
-        restore-keys: |
-          llvm-project-bazel-build-cache-${{ runner.os }}
-
-    - name: Create bazel cache dir when not found
-      run: |
-        if [ ! -d "${HOME}/.cache/bazel" ]; then
-          mkdir -p "${HOME}/.cache/bazel"
-        fi
+        cache-prefix: 'llvm-project'
 
     - name: Build docker image
       run: |
@@ -72,4 +60,4 @@ jobs:
                    -v "$(pwd)":"/opt/src/mlir-tcp" \
                    -v "${HOME}/.cache/bazel":"${HOME}/.cache/bazel" \
                    mlir-tcp:ci \
-                   bazel test @llvm-project//mlir/...
+                   bazel test --test_output=errors @llvm-project//mlir/...
diff --git a/.github/workflows/bazelBuildAndTestStablehlo.yml b/.github/workflows/bazelBuildAndTestStablehlo.yml
@@ -37,24 +37,12 @@ jobs:
 
     steps:
     - name: Checkout mlir-tcp
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
-    # Continually update cache even if there's a "hit" during
-    # restore to avoid the cache going stale over time
-    # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache
-    - name: Setup cache for bazel
-      uses: actions/cache@v3
+    - name: Setup workspace
+      uses: ./.github/actions/setup-build
       with:
-        path: ~/.cache/bazel
-        key: stablehlo-bazel-build-cache-${{ runner.os }}-${{ github.sha }}
-        restore-keys: |
-          stablehlo-bazel-build-cache-${{ runner.os }}
-
-    - name: Create bazel cache dir when not found
-      run: |
-        if [ ! -d "${HOME}/.cache/bazel" ]; then
-          mkdir -p "${HOME}/.cache/bazel"
-        fi
+        cache-prefix: 'stablehlo'
 
     - name: Build docker image
       run: |
@@ -72,4 +60,4 @@ jobs:
                    -v "$(pwd)":"/opt/src/mlir-tcp" \
                    -v "${HOME}/.cache/bazel":"${HOME}/.cache/bazel" \
                    mlir-tcp:ci \
-                   bazel test @stablehlo//...
+                   bazel test --test_output=errors @stablehlo//...
diff --git a/.github/workflows/bazelBuildAndTestTcp.yml b/.github/workflows/bazelBuildAndTestTcp.yml
@@ -30,24 +30,12 @@ jobs:
 
     steps:
     - name: Checkout mlir-tcp
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
-    # Continually update cache even if there's a "hit" during
-    # restore to avoid the cache going stale over time
-    # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache
-    - name: Setup cache for bazel
-      uses: actions/cache@v3
+    - name: Setup workspace
+      uses: ./.github/actions/setup-build
       with:
-        path: ~/.cache/bazel
-        key: mlir-tcp-bazel-build-cache-${{ runner.os }}-${{ github.sha }}
-        restore-keys: |
-          mlir-tcp-bazel-build-cache-${{ runner.os }}
-
-    - name: Create bazel cache dir when not found
-      run: |
-        if [ ! -d "${HOME}/.cache/bazel" ]; then
-          mkdir -p "${HOME}/.cache/bazel"
-        fi
+        cache-prefix: 'mlir-tcp'
 
     - name: Build docker image
       run: |
@@ -97,4 +85,4 @@ jobs:
                    -v "$(pwd)":"/opt/src/mlir-tcp" \
                    -v "${HOME}/.cache/bazel":"${HOME}/.cache/bazel" \
                    mlir-tcp:ci \
-                   bazel test //test/...
+                   bazel test --test_output=errors //...
diff --git a/.github/workflows/bazelBuildAndTestTorchmlir.yml b/.github/workflows/bazelBuildAndTestTorchmlir.yml
@@ -37,24 +37,12 @@ jobs:
 
     steps:
     - name: Checkout mlir-tcp
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
-    # Continually update cache even if there's a "hit" during
-    # restore to avoid the cache going stale over time
-    # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache
-    - name: Setup cache for bazel
-      uses: actions/cache@v3
+    - name: Setup workspace
+      uses: ./.github/actions/setup-build
       with:
-        path: ~/.cache/bazel
-        key: torch-mlir-bazel-build-cache-${{ runner.os }}-${{ github.sha }}
-        restore-keys: |
-          torch-mlir-bazel-build-cache-${{ runner.os }}
-
-    - name: Create bazel cache dir when not found
-      run: |
-        if [ ! -d "${HOME}/.cache/bazel" ]; then
-          mkdir -p "${HOME}/.cache/bazel"
-        fi
+        cache-prefix: 'torch-mlir'
 
     - name: Build docker image
       run: |
@@ -72,4 +60,4 @@ jobs:
                    -v "$(pwd)":"/opt/src/mlir-tcp" \
                    -v "${HOME}/.cache/bazel":"${HOME}/.cache/bazel" \
                    mlir-tcp:ci \
-                   bazel test @torch-mlir//...
+                   bazel test --test_output=errors @torch-mlir//...
diff --git a/BUILD b/BUILD
@@ -11,6 +11,11 @@ package(
     ],
 )
 
+exports_files([
+    "requirements.txt",
+    "requirements_lock.txt",
+])
+
 td_library(
     name = "TcpDialectTdFiles",
     srcs = [

diff --git a/README.md b/README.md
@@ -28,7 +28,7 @@ bazel build //:tcp-opt
 
 3. To run TCP lit and aot compile tests:
 ```shell
-bazel test //test/...
+bazel test //...
 ```
 
 We welcome contributions to `mlir-tcp`. If you do contribute, please finalize your PR with clang-format and bazel buildifier to ensure the C++ sources and BUILD files are formatted consistently:
@@ -60,6 +60,11 @@ The following CI workflows are automatically triggered anytime upstream dependen
 - [![Bazel Build and Test (torch-mlir)](https://github.com/cruise-automation/mlir-tcp/actions/workflows/bazelBuildAndTestTorchmlir.yml/badge.svg)](https://github.com/cruise-automation/mlir-tcp/actions/workflows/bazelBuildAndTestTorchmlir.yml)
 - [![Bazel Build and Test (stablehlo)](https://github.com/cruise-automation/mlir-tcp/actions/workflows/bazelBuildAndTestStablehlo.yml/badge.svg)](https://github.com/cruise-automation/mlir-tcp/actions/workflows/bazelBuildAndTestStablehlo.yml)
 
+To use newer `torch-mlir` and/or `torch` python packages in our hermetic python sandbox, just regenerate `requirements_lock.txt` as follows:
+```shell
+truncate -s 0 requirements_lock.txt
+bazel run //tools/pip:requirements.update
+```
 
 ## Debugging Guide
 

diff --git a/WORKSPACE b/WORKSPACE
@@ -26,8 +26,47 @@ torch_mlir_configure(name = "torch-mlir")
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 
+# -------------------------- #
+#    Hermetic Python Setup   #
+# -------------------------- #
+
+RULES_PYTHON_VERSION = "0.29.0"
+
+RULES_PYTHON_SHA256 = "d71d2c67e0bce986e1c5a7731b4693226867c45bfe0b7c5e0067228a536fc580"
+
+http_archive(
+    name = "rules_python",
+    sha256 = RULES_PYTHON_SHA256,
+    strip_prefix = "rules_python-{}".format(RULES_PYTHON_VERSION),
+    url = "https://github.com/bazelbuild/rules_python/releases/download/{}/rules_python-{}.tar.gz".format(RULES_PYTHON_VERSION, RULES_PYTHON_VERSION),
+)
+
+load("@rules_python//python:repositories.bzl", "py_repositories")
+
+py_repositories()
+
+load("@rules_python//python:repositories.bzl", "python_register_toolchains")
+
+python_register_toolchains(
+    name = "python_3_10",
+    python_version = "3.10",
+)
+
+load("@python_3_10//:defs.bzl", "interpreter")
+load("@rules_python//python:pip.bzl", "pip_parse")
+
+pip_parse(
+    name = "pip_deps",
+    python_interpreter_target = interpreter,
+    requirements_lock = "//:requirements_lock.txt",
+)
+
+load("@pip_deps//:requirements.bzl", "install_deps")
+
+install_deps()
+
 # --------------------------- #
-#    Buildifier dependencies  #
+#    Buildifier Dependencies  #
 # --------------------------- #
 
 # https://github.com/bazelbuild/buildtools/blob/master/buildifier/README.md
@@ -84,10 +123,10 @@ http_archive(
     ],
 )
 
-# ----------------------------- #
-#    Compile Commands Extractor #
-#    for Bazel (clangd)         #
-# ----------------------------- #
+# ------------------------------------ #
+#    Bazel Compile Commands Extractor  #
+#    for clangd                        #
+# ------------------------------------ #
 
 # https://github.com/hedronvision/bazel-compile-commands-extractor/blob/main/README.md
 

diff --git a/requirements.txt b/requirements.txt
@@ -0,0 +1,3 @@
+-f https://github.com/llvm/torch-mlir-release/releases/expanded_assets/dev-wheels
+torch
+torch-mlir