Fix openr.thrift Python Module Build

Dockerfile

@@ -1,9 +1,10 @@
 FROM ubuntu:20.04
 
+
 # Install tools needed for development
 RUN apt update && \
     apt upgrade --yes && \
-    apt install --yes build-essential git libssl-dev m4 python3-pip
+    apt install --yes build-essential cython3 git libssl-dev m4 python3-pip
 
 # Copy needed source
 RUN mkdir /src
@@ -17,16 +18,20 @@ RUN cd /src && build/build_openr.sh && chmod 644 /etc/openr.conf
 RUN mkdir /opt/bin && cp /src/build/docker_openr_helper.sh /opt/bin
 
 # Install `breeze` OpenR CLI
-RUN pip3 --no-cache-dir install --upgrade pip setuptools wheel
-RUN PATH="${PATH}:/opt/facebook/fbthrift/bin" ; \
-    cd /src/openr/py/ && \
-    python3 setup.py install
+RUN apt install g++-10 --yes  # We need g++-10 or higher for coroutines which are used in folly::coro
+# TODO Move these files into build/
+COPY cython_compile.py /src/build/cython_compile.py
+RUN git clone https://github.com/cython/cython
+COPY setup.py /src/openr/py/setup.py
+RUN cd /src && build/build_breeze.sh
+RUN cp -r /src/build/lib.linux-x86_64-3.8 /breeze-build
 
 # Cleanup all we can to keep container as lean as possible
+# TODO: We can use Dockerfile stages instead
 RUN apt remove --yes build-essential git libssl-dev m4 && \
     apt autoremove --yes && \
     rm -rf /src /tmp/* /var/lib/apt/lists/*
 
 CMD ["/opt/bin/docker_openr_helper.sh"]
-# Expose OpenR Thrift port - Recommended to run OpenR on host network ...
+# Expose OpenR Thrift port
 EXPOSE 2018/tcp

README.md

@@ -118,7 +118,7 @@ Note: the `build_openr.sh` script will run this step for you
 - Manually you can drive `getdeps.py` to install elsewhere
   - refer to `build_openr.sh`
 
-#### Installing Python Libraries
+#### Installing Python Libraries + CLI
 
 You will need python `pip` or `setuptools` to build and install python modules.
 All library dependencies will be automatically installed except the
@@ -136,11 +136,13 @@ sudo python setup.py install
 
 ### Docker Building / Usage
 
-OpenR now has a `Dockerfile`. It uses `gendeps.py` to build all dependencies + OpenR.
-It also installs the OpenR CLI `breeze` into the container.
+OpenR now has two `Dockerfile`s. It uses `gendeps.py` to build all dependencies + OpenR. The two files are:
+
+- `Dockerfile` - Full OpenR daemon + `breeze` CLI
+- `Dockerfile_breeze` - Only `breeze` CLI
 
 ```console
- docker build --network host .
+docker build [-f Dockerfile_breeze] --network host .
 ```
 
 #### Running

build/build_breeze.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+#
+# Copyright (c) 2014-present, Facebook, Inc.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+source "$(dirname "$0")/common.sh"
+
+alias thrift1=/opt/facebook/fbthrift/bin/thrift1
+
+# Step 1: Stage compilation
+
+# folly Cython
+mkdir ./folly
+cp -r \
+/tmp/fbcode_builder_getdeps-ZsrcZbuildZfbcode_builder-root/repos/github.51.al-facebook-folly.git/folly/python/* ./folly
+
+# fb303 thrift
+mkdir -p ./fb303-thrift
+cp -r  /opt/facebook/fb303/include/thrift-files/fb303 ./fb303-thrift/
+
+# fbzmq thrift
+mkdir -p ./fbzmq-thrift/fbzmq
+cp -r  /opt/facebook/fbzmq/include/fbzmq/service ./fbzmq-thrift/fbzmq/
+
+# fbthrift Cython and thrift
+cp -r  /opt/facebook/fbthrift/include/thrift/lib/ ./thrift
+touch ./thrift/py3/__init__.py
+touch ./thrift/__init__.py
+
+mkdir /src/fbthrift-thrift
+cp -r \
+/tmp/fbcode_builder_getdeps-ZsrcZbuildZfbcode_builder-root/repos/github.51.al-facebook-fbthrift.git/thrift/lib/thrift/* \
+/src/fbthrift-thrift
+
+mkdir -p /src/thrift/py3
+chown -R root /tmp/fbcode_builder_getdeps-ZsrcZbuildZfbcode_builder-root/repos/github.51.al-facebook-fbthrift.git/thrift/lib/py3/
+cp -r \
+/tmp/fbcode_builder_getdeps-ZsrcZbuildZfbcode_builder-root/repos/github.51.al-facebook-fbthrift.git/thrift/lib/py3/* \
+/src/thrift/py3
+touch /src/thrift/__init__.py
+
+# Open/R thrift
+mkdir -p ./openr-thrift/openr
+cp -r /src/openr/if/ ./openr-thrift/openr/
+
+# Neteng thrift
+mkdir -p neteng-thrift/configerator/structs/neteng/config/
+cp -r \
+/tmp/fbcode_builder_getdeps-ZsrcZbuildZfbcode_builder-root/repos/github.51.al-facebook-openr.git/configerator/structs/neteng/config/routing_policy.thrift \
+neteng-thrift/configerator/structs/neteng/config/
+
+# HACK TO FIX CYTHON .pxd
+cp /cython/Cython/Includes/libcpp/utility.pxd /usr/lib/python3/dist-packages/Cython/Includes/libcpp/utility.pxd
+
+# Step 2. Generate mstch_cpp2 and mstch_py3 bindings
+
+python3 /src/build/gen.py
+
+# HACK TO FIX fbthrift-py/gen-cpp2/
+echo " " > /src/fbthrift-thrift/gen-cpp2/metadata_metadata.h
+echo " " > /src/fbthrift-thrift/gen-cpp2/metadata_types.h
+echo " " > /src/fbthrift-thrift/gen-cpp2/metadata_types_custom_protocol.h
+
+# Step 3. Generate clients.cpp
+
+python3 /src/build/cython_compile.py
+
+# Step 4. Compile .so
+
+CC="/usr/bin/gcc-10" \
+CXX="/usr/bin/g++-10" \
+CFLAGS="-I. -Iopenr-thrift -Ifb303-thrift -Ifbzmq-thrift -Ineteng-thrift -std=c++20 -fcoroutines " \
+CFLAGS="$CFLAGS -w -D_CPPLIB_VER=20" \
+CXXFLAGS="$CFLAGS" \
+python3 openr/py/setup.py build -j10

build/build_openr.sh

@@ -4,29 +4,16 @@
 # This source code is licensed under the MIT license found in the
 # LICENSE file in the root directory of this source tree.
 
-errorCheck() {
-  return_code=$?
-  if [ $return_code -ne 0 ]; then
-    echo "[ERROR]: $1"
-    exit $return_code
-  fi
-}
-
-GETDEPS="$(dirname "$0")/fbcode_builder/getdeps.py"
-INSTALL_PREFIX="/opt/facebook"
-PYTHON3=$(command -v python3)
-
-if [ "$PYTHON3" == "" ]; then
-  echo "ERROR: No \`python3\` in PATH"
-  exit 1
-fi
+source "$(dirname "$0")/common.sh"
 
 # TODO: Get apt deb installing working - dies in Docker container with libzstd-dev fixed
-# python3 "$GETDEPS" --allow-system-packages install-system-deps --recursive openr
+# "$PYTHON3" "$GETDEPS" --allow-system-packages install-system-deps --recursive openr
 # errorCheck "Failed to install-system-deps for openr"
 
-python3 "$GETDEPS" --allow-system-packages build --src-dir=. --no-tests --install-prefix "$INSTALL_PREFIX" openr
+"$PYTHON3" "$GETDEPS" --allow-system-packages build --no-tests --install-prefix "$INSTALL_PREFIX" \
+--extra-cmake-defines "$EXTRA_CMAKE_DEFINES" openr
 errorCheck "Failed to build openr"
 
-python3 "$GETDEPS" fixup-dyn-deps --strip --src-dir=. openr _artifacts/linux  --project-install-prefix openr:"$INSTALL_PREFIX" --final-install-prefix "$INSTALL_PREFIX"
+# TODO: Maybe fix src-dir to be absolute reference to dirname $0's parent
+"$PYTHON3" "$GETDEPS" fixup-dyn-deps --strip --src-dir=. openr _artifacts/linux  --project-install-prefix openr:"$INSTALL_PREFIX" --final-install-prefix "$INSTALL_PREFIX"
 errorCheck "Failed to fixup-dyn-deps for openr"

build/common.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+#
+# Copyright (c) 2014-present, Facebook, Inc.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+GETDEPS="$(dirname "$0")/fbcode_builder/getdeps.py"
+INSTALL_PREFIX="/opt/facebook"
+PYTHON3=$(command -v python3)
+PIP3=$(command -v pip3)
+EXTRA_CMAKE_DEFINES='{"CMAKE_POSITION_INDEPENDENT_CODE": "ON"}'
+
+errorCheck() {
+  return_code=$?
+  if [ $return_code -ne 0 ]; then
+    echo "[ERROR]: $1"
+    exit $return_code
+  fi
+}
+
+if [ "$PYTHON3" == "" ]; then
+  echo "ERROR: No \`python3\` in PATH"
+  exit 1
+fi
+
+if [ "$PIP3" == "" ]; then
+  echo "ERROR: No \`pip3\` in PATH"
+  exit 2
+fi

build/cython_compile.py

@@ -0,0 +1,35 @@
+import os
+from subprocess import Popen, check_call
+
+thrift_files = []
+procs = []
+for root, dirs, files in os.walk('openr-thrift'):
+    for f in files:
+        if f.endswith(".pyx"):
+            thrift_file = os.path.join(root, f)
+            cmd = \
+                [
+                    "cython3",
+                    "--fast-fail",
+                    "-3",
+                    "--cplus",
+                    thrift_file,
+                    "-o",
+                    root,
+                    "-I.",
+                    "-I/src",
+                    "-I/usr/lib/python3/dist-packages/Cython/Includes",
+                    "-I/src/fbthrift-thrift/gen-py3",
+                    "-I/src/fb303-thrift/fb303/thrift/gen-py3",
+                    "-I/src/neteng-thrift/configerator/structs/neteng/config/gen-py3",
+                ]
+            print(f"Generating cython module {f}")
+            procs += [Popen( cmd)]
+
+print("Waiting for cython generation to finish...")
+failures = 0
+for proc in procs:
+    proc.wait()
+    if proc.returncode != 0:
+        failures += 1
+print(f"{len(procs) - failures}/{len(procs)} succeeded")

build/fbcode_builder/fbcode_builder.py

@@ -414,6 +414,14 @@ def cmake_configure(self, name, cmake_path='..'):
             'BUILD_SHARED_LIBS': 'ON',
             'CMAKE_INSTALL_PREFIX': self.option('prefix'),
         }
+
+        # Hacks to add thriftpy3 support
+        if 'BUILD_THRIFT_PY3' in os.environ and 'folly' in name:
+            cmake_defines['PYTHON_EXTENSIONS'] = 'True'
+
+        if 'BUILD_THRIFT_PY3' in os.environ and 'fbthrift' in name:
+            cmake_defines['thriftpy3'] = 'ON'
+
         cmake_defines.update(
             self.option('{0}:cmake_defines'.format(name), {})
         )

build/gen.py

@@ -0,0 +1,54 @@
+import os
+from subprocess import check_call
+
+def generate_thrift_files():
+    """
+    Get list of all thrift files (absolute path names) and then generate
+    python definitions for all thrift files.
+    """
+
+    thrift_dirs = [
+        "openr-thrift",
+        "fb303-thrift",
+        "fbzmq-thrift",
+        "fbthrift-thrift",
+        "neteng-thrift",
+    ]
+    generators = ['mstch_cpp2', 'py', 'mstch_py3']
+    includes = [
+        "openr-thrift",
+        "fb303-thrift",
+        "fbzmq-thrift",
+        "neteng-thrift",
+        "fbthrift-thrift",
+        ".",
+    ]
+
+    # Find .thrift files
+    thrift_files = []
+    for thrift_dir in thrift_dirs:
+        for root, dirs, files in os.walk(thrift_dir):
+            for file in files:
+                if file.endswith(".thrift"):
+                     thrift_files += [os.path.join(root, file)]
+
+    # Generate cpp and python
+    for gen in generators:
+        cmd = ["/opt/facebook/fbthrift/bin/thrift1", '--gen', gen]
+        for include in includes:
+            cmd += ["-I", f"{include}"]
+        for thrift_file in thrift_files:
+            check_call([*cmd, '-o', os.path.join(os.path.dirname(thrift_file)), str(thrift_file)])
+
+    # Add __init__.py for compiling cython modules
+    for include in includes:
+        for root, dirs, files in os.walk(f"{include}"):
+            for f in files:
+                check_call(
+                    [
+                        "touch",
+                        os.path.join(root, os.path.dirname(f),  '__init__.py')
+                    ]
+                )
+
+generate_thrift_files()