From b5e8bf61b4accaf15cda70e4e97e297992983b72 Mon Sep 17 00:00:00 2001 From: Joe Evans Date: Tue, 21 Dec 2021 22:11:05 -0800 Subject: [PATCH] [v1.x] Backport #20782, #20783 and #20786 from v1.9.x branch, #19945 from master (#20785) * [v1.9.x] Fix CD for pypi wheel version (#20782) * Change env variable used to detect whether it's a release build or not. RELEASE_BUILD is already setup in the CD pipeline. * Refine how variable is compared to trigger release builds. * Check for 'true' exclusively, to prevent unwanted release builds. * Make sure we pass any environment variables on the build.py command line to the docker run command. * [v1.9.x] Fix aarch64 cd pipeline (#20783) * Add wheel to installed pip packages on aarch64, so we can build the wheel in the CD pipeline. * Better support for aarch64 docker container cache. * Only import machine() function from platform module, as platform variable is already widely used. * Only build containers with aarch64 in name on aarch64. * Make all arch defaults to local machine type. * Retry getting test data to prevent CI tests from failing - increase connect timeout from 10 to 60sec, increase retry delay from 0 (setting duplicated) to 30, increase retry attempts to 5. * Update website for v1.9.x branch. (#20786) * [website] Fix broken website for master version (#19945) * fix website * fix header * fix index * remove locall test * fix main.scss Co-authored-by: Wei Chu Co-authored-by: waytrue17 <52505574+waytrue17@users.noreply.github.com> Co-authored-by: Wei Chu --- ci/build.py | 24 ++++++------- ci/docker/install/requirements_aarch64 | 1 + ci/docker_cache.py | 3 +- cpp-package/example/get_data.sh | 7 ++-- .../themes/mx-theme/mxtheme/header_top.html | 17 ++++----- docs/static_site/src/404.html | 34 ++++++++---------- docs/static_site/src/_config.yml | 3 +- docs/static_site/src/_config_beta.yml | 3 +- docs/static_site/src/_config_prod.yml | 3 +- .../_includes/get_started/get_started.html | 7 ++-- .../get_started/linux/python/cpu/pip.md | 11 ++++-- .../get_started/linux/python/gpu/pip.md | 9 ++++- .../src/_includes/get_started/pip_snippet.md | 2 +- docs/static_site/src/_includes/header.html | 7 ++-- docs/static_site/src/_layouts/default.html | 4 +-- docs/static_site/src/_layouts/home.html | 4 +-- docs/static_site/src/_layouts/page.html | 33 ++++++++--------- docs/static_site/src/_layouts/page_api.html | 33 ++++++++--------- .../src/_layouts/page_category.html | 33 ++++++++--------- .../src/_layouts/page_landing_tutorials.html | 33 ++++++++--------- docs/static_site/src/_layouts/post.html | 33 ++++++++--------- docs/static_site/src/assets/main.scss | 4 +-- docs/static_site/src/index.html | 35 +++++++++---------- .../src/pages/get_started/download.md | 3 +- .../dependencies/make_shared_dependencies.sh | 7 ++-- tools/pip/setup.py | 2 +- 26 files changed, 175 insertions(+), 180 deletions(-) diff --git a/ci/build.py b/ci/build.py index 06a747515182..1998f58ee272 100755 --- a/ci/build.py +++ b/ci/build.py @@ -34,6 +34,7 @@ import shutil import signal import subprocess +from platform import machine from itertools import chain from subprocess import check_call, check_output from typing import * @@ -46,7 +47,7 @@ DOCKER_COMPOSE_FILES = set(['docker/build.centos7']) # keywords to identify arm-based dockerfiles -AARCH_FILE_KEYWORDS = ['armv', 'aarch64'] +AARCH_FILE_KEYWORDS = ['aarch64'] def get_dockerfiles_path(): return "docker" @@ -60,7 +61,7 @@ def get_docker_compose_platforms(path: str = get_dockerfiles_path()): return platforms -def get_platforms(path: str = get_dockerfiles_path(), arch='x86') -> List[str]: +def get_platforms(path: str = get_dockerfiles_path(), arch=machine()) -> List[str]: """Get a list of platforms given our dockerfiles""" dockerfiles = glob.glob(os.path.join(path, "Dockerfile.*")) dockerfiles = set(filter(lambda x: x[-1] != '~', dockerfiles)) @@ -68,7 +69,7 @@ def get_platforms(path: str = get_dockerfiles_path(), arch='x86') -> List[str]: files = files - DOCKER_COMPOSE_FILES files.update(["build."+x for x in get_docker_compose_platforms()]) arm_files = set(filter(lambda x: any(y in x for y in AARCH_FILE_KEYWORDS), files)) - if arch == 'x86': + if arch == 'x86_64': files = files - arm_files elif arch == 'aarch64': files = arm_files @@ -273,15 +274,10 @@ def container_run(platform: str, # mount mxnet/build for storing build '-v', "{}:/work/build".format(local_build_folder), '-v', "{}:/work/ccache".format(local_ccache_dir), - '-u', '{}:{}'.format(os.getuid(), os.getgid()), - '-e', 'CCACHE_MAXSIZE={}'.format(environment['CCACHE_MAXSIZE']), - # temp dir should be local and not shared - '-e', 'CCACHE_TEMPDIR={}'.format(environment['CCACHE_TEMPDIR']), - # this path is inside the container as /work/ccache is mounted - '-e', "CCACHE_DIR={}".format(environment['CCACHE_DIR']), - # a container-scoped log, useful for ccache verification. - '-e', "CCACHE_LOGFILE={}".format(environment['CCACHE_LOGFILE']), + '-u', '{}:{}'.format(os.getuid(), os.getgid()) ] + for e in environment.keys(): + docker_arg_list += ['-e', '{}={}'.format(e, environment[e])] docker_arg_list += [tag] docker_arg_list.extend(command) @@ -305,7 +301,7 @@ def docker_run_cmd(cmd): return 0 -def list_platforms(arch='x86') -> str: +def list_platforms(arch=machine()) -> str: return "\nSupported platforms:\n{}".format('\n'.join(get_platforms(arch=arch))) @@ -362,8 +358,8 @@ def main() -> int: type=str) parser.add_argument("-A", "--architecture", - help="Architecture of images to build (x86 or aarch64). Default is x86.", - default='x86', + help="Architecture of images to build (x86_64 or aarch64). Default is current machine type.", + default=machine(), dest='architecture') parser.add_argument("-b", "--build-only", diff --git a/ci/docker/install/requirements_aarch64 b/ci/docker/install/requirements_aarch64 index 327a78f7d960..1cfead0b41fc 100644 --- a/ci/docker/install/requirements_aarch64 +++ b/ci/docker/install/requirements_aarch64 @@ -30,3 +30,4 @@ astroid==2.3.3 # pylint and astroid need to be aligned requests<2.19.0,>=2.18.4 setuptools coverage +wheel diff --git a/ci/docker_cache.py b/ci/docker_cache.py index f9d5b81dff2b..a51203480d7e 100755 --- a/ci/docker_cache.py +++ b/ci/docker_cache.py @@ -30,6 +30,7 @@ import subprocess import re import sys +from platform import machine from typing import * import build as build_util @@ -225,7 +226,7 @@ def script_name() -> str: args = parser.parse_args() - platforms = build_util.get_platforms() + platforms = build_util.get_platforms(arch=machine()) if "dkr.ecr" in args.docker_registry: _ecr_login(args.docker_registry) diff --git a/cpp-package/example/get_data.sh b/cpp-package/example/get_data.sh index fda69ce2f087..ae427565ec55 100755 --- a/cpp-package/example/get_data.sh +++ b/cpp-package/example/get_data.sh @@ -33,11 +33,10 @@ download () { fi echo "Downloading ${URL} ..." - local CURL_OPTIONS="--connect-timeout 10 \ + local CURL_OPTIONS="--connect-timeout 60 \ --max-time 300 \ - --retry-delay 10 \ - --retry 3 \ - --retry-delay 0 \ + --retry-delay 30 \ + --retry 5 \ --location \ --silent" curl ${CURL_OPTIONS} ${URL} -o ${GZ_FILE_NAME} diff --git a/docs/python_docs/themes/mx-theme/mxtheme/header_top.html b/docs/python_docs/themes/mx-theme/mxtheme/header_top.html index 0b54d3c217bc..eaba20ba6fb1 100644 --- a/docs/python_docs/themes/mx-theme/mxtheme/header_top.html +++ b/docs/python_docs/themes/mx-theme/mxtheme/header_top.html @@ -1,6 +1,6 @@