Merge remote-tracking branch 'upstream/v1.7.x' into v1.7.x

.codecov.yml

@@ -4,6 +4,9 @@ codecov:
     require_ci_to_pass: yes
 
 coverage:
+  status:
+    project: off
+    patch: off
   precision: 2
   round: down
   range: "70...100"

.gitignore

@@ -121,6 +121,10 @@ cmake_install.cmake
 # Mac OS X
 .DS_Store
 
+# Windows
+windows_package.7z
+windows_package
+
 #Notebook Automated Test
 !tests/nightly/test_tutorial_config.txt
 !tests/nightly/TestNotebook

LICENSE

@@ -216,7 +216,7 @@
     The following components are provided under an Apache 2.0 license.
 
     1. MXNet Cpp-package - For details, /cpp-package/LICENSE
-         Copyright (c) 2015-2016 by Contributors 
+         Copyright (c) 2015-2016 by Contributors
     2. MXNet rcnn - For details, see, example/rcnn/LICENSE
          Copyright (c) 2014, 2015, The Regents of the University of California (Regents)
     3. MXNet scala-package - For details, see, scala-package/LICENSE
@@ -226,9 +226,9 @@
     5. 3rdparty/dlpack - For details, see, 3rdparty/dlpack/LICENSE
          Copyright 2017 by Contributors
     6. 3rdparty/dmlc-core - For details, see, 3rdparty/dmlc-core/LICENSE
-         Copyright (c) 2015 by Contributors 
+         Copyright (c) 2015 by Contributors
          Copyright 2015 by dmlc-core developers
-         Copyright by Contributors 
+         Copyright by Contributors
     7. 3rdparty/mshadow - For details, see, 3rdparty/mshadow/LICENSE
          Copyright (c) 2014-2016 by Contributors
          Copyright by Contributors
@@ -237,7 +237,7 @@
          Copyright 2018 by Contributors
          Copyright (c) 2018 by Xilinx, Contributors
     9. 3rdparty/tvm/dmlc-core - For details, see, 3rdparty/tvm/3rdparty/dmlc-core/LICENSE
-         Copyright (c) 2015 by Contributors 
+         Copyright (c) 2015 by Contributors
     10. 3rdparty/tvm/dlpack - For details, see, 3rdparty/tvm/3rdparty/dlpack/LICENSE
          Copyright (c) 2015-2017 by Contributors
          Copyright by Contributors
@@ -343,6 +343,12 @@
     11. Numpy einsum operator - For details, see src/operator/numpy/np_einsum_op-inl.h
          Copyright (c) 2005-2019, NumPy Developers.
          Copyright (c) 2019, The Apache Software Foundation.
+    12. Numpy einsum operator - For details, see src/operator/numpy/np_einsum_path_op-inl.h
+         Copyright (c) 2005-2019, NumPy Developers.
+         Copyright (c) 2019, The Apache Software Foundation.
+    13. Numpy einsum operator - For details, see src/operator/numpy/np_einsum_op.cc
+         Copyright (c) 2005-2019, NumPy Developers.
+         Copyright (c) 2019, The Apache Software Foundation.
 
     =======================================================================================
     2-clause BSD licenses
@@ -385,14 +391,18 @@
     5. im2col.cuh - For details, see, src/operator/nn/im2col.cuh
          Copyright (c) 2014-2017 The Regents of the University of California (Regents)
          Copyright (c) 2014-2017, the respective contributors
-
     6. deformable_im2col.h - For details, see, src/operator/contrib/nn/deformable_im2col.h
          Copyright (c) 2014-2017 The Regents of the University of California (Regents)
          Copyright (c) 2014-2017, the respective contributors
-
     7. deformable_im2col.cuh - For details, see, src/operator/contrib/nn/deformable_im2col.cuh
          Copyright (c) 2014-2017 The Regents of the University of California (Regents)
          Copyright (c) 2014-2017, the respective contributors
+    8. modulated_deformable_im2col.h - For details, see, src/operator/contrib/nn/modulated_deformable_im2col.h
+         Copyright (c) 2014-2017 The Regents of the University of California (Regents)
+         Copyright (c) 2014-2017, the respective contributors
+    9. modulated_deformable_im2col.cuh - For details, see, src/operator/contrib/nn/modulated_deformable_im2col.cuh
+         Copyright (c) 2014-2017 The Regents of the University of California (Regents)
+         Copyright (c) 2014-2017, the respective contributors
 
 
     COPYRIGHT
@@ -667,7 +677,7 @@
     <none yet>
 
     =======================================================================================
-    
+
     13. FindJeMalloc.cmake
     For details, see cmake/Modules/FindJeMalloc.cmake
 
@@ -690,14 +700,14 @@
     # execute, and transmit the Software, and to prepare derivative works of the
     # Software, and to permit third-parties to whom the Software is furnished to
     # do so, all subject to the following:
-    # 
+    #
     # The copyright notices in the Software and this entire statement, including
     # the above license grant, this restriction and the following disclaimer,
     # must be included in all copies of the Software, in whole or in part, and
     # all derivative works of the Software, unless such copies or derivative
     # works are solely in the form of machine-executable object code generated by
     # a source language processor.
-    # 
+    #
     # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
     # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
     # FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
@@ -706,7 +716,7 @@
     # ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
     # DEALINGS IN THE SOFTWARE.
 
-    Now it has been changed by Apache MXNet project. Modifications have the following license: 
+    Now it has been changed by Apache MXNet project. Modifications have the following license:
 
     # Licensed to the Apache Software Foundation (ASF) under one
     # or more contributor license agreements.  See the NOTICE file
@@ -724,8 +734,8 @@
     # KIND, either express or implied.  See the License for the
     # specific language governing permissions and limitations
     # under the License.
-    
-    
+
+
     # Copyright (c)      2014 Thomas Heller
     # Copyright (c) 2007-2012 Hartmut Kaiser
     # Copyright (c) 2010-2011 Matt Anderson
@@ -848,28 +858,28 @@
     /*
      A C-program for MT19937, with initialization improved 2002/1/26.
      Coded by Takuji Nishimura and Makoto Matsumoto.
-    
+
      Before using, initialize the state by using init_genrand(seed)
      or init_by_array(init_key, key_length).
-    
+
      Copyright (C) 1997 - 2002, Makoto Matsumoto and Takuji Nishimura,
      All rights reserved.
-    
+
      Redistribution and use in source and binary forms, with or without
      modification, are permitted provided that the following conditions
      are met:
-    
+
      1. Redistributions of source code must retain the above copyright
      notice, this list of conditions and the following disclaimer.
-    
+
      2. Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
-    
+
      3. The names of its contributors may not be used to endorse or promote
      products derived from this software without specific prior written
      permission.
-    
+
      THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
      "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
      LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
@@ -881,8 +891,8 @@
      LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
      NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
      SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-    
-    
+
+
      Any feedback is very welcome.
      http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/emt.html
      email: m-mat @ math.sci.hiroshima-u.ac.jp (remove space)
@@ -922,28 +932,28 @@
     For details, see /3rdparty/tvm/3rdparty/rang/LICENSE
 
     This is free and unencumbered software released into the public domain.
-    
+
     Anyone is free to copy, modify, publish, use, compile, sell, or
     distribute this software, either in source code form or as a compiled
     binary, for any purpose, commercial or non-commercial, and by any
     means.
-    
+
     In jurisdictions that recognize copyright laws, the author or authors
     of this software dedicate any and all copyright interest in the
     software to the public domain. We make this dedication for the benefit
     of the public at large and to the detriment of our heirs and
     successors. We intend this dedication to be an overt act of
     relinquishment in perpetuity of all present and future rights to this
     software under copyright law.
-    
+
     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
     MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
     IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
     OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
     ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
     OTHER DEALINGS IN THE SOFTWARE.
-    
+
     For more information, please refer to <http://unlicense.org>
 
     20. Font-Awesome, SIL Open Font License(OFL)

ci/build_windows.py

@@ -33,13 +33,15 @@
 import zipfile
 from distutils.dir_util import copy_tree
 from enum import Enum
-from subprocess import check_call
+from subprocess import check_call, call
 
 from util import *
 
 KNOWN_VCVARS = {
+    # https://gitlab.kitware.com/cmake/cmake/issues/18920
     'VS 2015': r'C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\x86_amd64\vcvarsx86_amd64.bat',
-    'VS 2017': r'C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsx86_amd64.bat'
+    'VS 2017': r'C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsx86_amd64.bat',
+    'VS 2019': r'C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars64.bat',
 }
 
 
@@ -54,6 +56,8 @@ class BuildFlavour(Enum):
 
 CMAKE_FLAGS = {
     'WIN_CPU': (
+        '-DCMAKE_C_COMPILER=cl '
+        '-DCMAKE_CXX_COMPILER=cl '
         '-DUSE_CUDA=OFF '
         '-DUSE_CUDNN=OFF '
         '-DENABLE_CUDA_RTC=OFF '
@@ -67,6 +71,8 @@ class BuildFlavour(Enum):
         '-DCMAKE_BUILD_TYPE=Release')
 
     , 'WIN_CPU_MKLDNN': (
+        '-DCMAKE_C_COMPILER=cl '
+        '-DCMAKE_CXX_COMPILER=cl '
         '-DUSE_CUDA=OFF '
         '-DUSE_CUDNN=OFF '
         '-DENABLE_CUDA_RTC=OFF '
@@ -80,6 +86,8 @@ class BuildFlavour(Enum):
         '-DCMAKE_BUILD_TYPE=Release')
 
     , 'WIN_CPU_MKLDNN_MKL': (
+        '-DCMAKE_C_COMPILER=cl '
+        '-DCMAKE_CXX_COMPILER=cl '
         '-DUSE_CUDA=OFF '
         '-DUSE_CUDNN=OFF '
         '-DENABLE_CUDA_RTC=OFF '
@@ -93,6 +101,8 @@ class BuildFlavour(Enum):
         '-DCMAKE_BUILD_TYPE=Release')
 
     , 'WIN_CPU_MKL': (
+        '-DCMAKE_C_COMPILER=cl '
+        '-DCMAKE_CXX_COMPILER=cl '
         '-DUSE_CUDA=OFF '
         '-DUSE_CUDNN=OFF '
         '-DENABLE_CUDA_RTC=OFF '
@@ -106,6 +116,8 @@ class BuildFlavour(Enum):
         '-DCMAKE_BUILD_TYPE=Release')
 
     , 'WIN_GPU': (
+        '-DCMAKE_C_COMPILER=cl '
+        '-DCMAKE_CXX_COMPILER=cl '
         '-DUSE_CUDA=ON '
         '-DUSE_CUDNN=ON '
         '-DENABLE_CUDA_RTC=ON '
@@ -120,6 +132,8 @@ class BuildFlavour(Enum):
         '-DCMAKE_BUILD_TYPE=Release')
 
     , 'WIN_GPU_MKLDNN': (
+        '-DCMAKE_C_COMPILER=cl '
+        '-DCMAKE_CXX_COMPILER=cl '
         '-DUSE_CUDA=ON '
         '-DUSE_CUDNN=ON '
         '-DENABLE_CUDA_RTC=ON '
@@ -140,38 +154,40 @@ def windows_build(args):
     logging.info("Using vcvars environment:\n{}".format(args.vcvars))
 
     path = args.output
-    os.makedirs(path, exist_ok=True)
 
-    mxnet_root = get_mxnet_root()
-    logging.info("Found MXNet root: {}".format(mxnet_root))
+    # cuda thrust + VS 2019 is flaky: try multiple times if fail
+    MAXIMUM_TRY = 5
+    build_try = 0
+
+    while build_try < MAXIMUM_TRY:
+        if os.path.exists(path):
+            shutil.rmtree(path)
+        os.makedirs(path, exist_ok=True)
 
-    url = 'https://github.com/Kitware/CMake/releases/download/v3.16.1/cmake-3.16.1-win64-x64.zip'
-    with tempfile.TemporaryDirectory() as tmpdir:
-        cmake_file_path = download_file(url, tmpdir)
-        with zipfile.ZipFile(cmake_file_path, 'r') as zip_ref:
-            # Create $tmpdir\cmake-3.16.1-win64-x64\bin\cmake.exe
-            zip_ref.extractall(tmpdir)
+        mxnet_root = get_mxnet_root()
+        logging.info("Found MXNet root: {}".format(mxnet_root))
 
         with remember_cwd():
             os.chdir(path)
-            cmd = "\"{}\" && {} -G \"NMake Makefiles JOM\" {} {}".format(
-                args.vcvars,
-                os.path.join(tmpdir, 'cmake-3.16.1-win64-x64', 'bin', 'cmake.exe'),
-                CMAKE_FLAGS[args.flavour], mxnet_root)
+            cmd = "\"{}\" && cmake -GNinja {} {}".format(args.vcvars,
+                                                         CMAKE_FLAGS[args.flavour],
+                                                         mxnet_root)
             logging.info("Generating project with CMake:\n{}".format(cmd))
             check_call(cmd, shell=True)
 
-            cmd = "\"{}\" && jom".format(args.vcvars)
-            logging.info("Building with jom:\n{}".format(cmd))
+            cmd = "\"{}\" && ninja".format(args.vcvars)
+            logging.info("Building:\n{}".format(cmd))
 
             t0 = int(time.time())
-            check_call(cmd, shell=True)
-
-            logging.info(
-                "Build flavour: {} complete in directory: \"{}\"".format(
-                    args.flavour, os.path.abspath(path)))
-            logging.info("Build took {}".format(
-                datetime.timedelta(seconds=int(time.time() - t0))))
+            ret = call(cmd, shell=True)
+
+            if ret != 0:
+                build_try += 1
+                logging.info("{} build(s) have failed".format(build_try))
+            else:
+                logging.info("Build flavour: {} complete in directory: \"{}\"".format(args.flavour, os.path.abspath(path)))
+                logging.info("Build took {}".format(datetime.timedelta(seconds=int(time.time() - t0))))
+                break
     windows_package(args)
 
 
@@ -233,7 +249,7 @@ def main():
 
     parser.add_argument("--vcvars",
         help="vcvars batch file location, typically inside vs studio install dir",
-        default=KNOWN_VCVARS['VS 2015'],
+        default=KNOWN_VCVARS['VS 2019'],
         type=str)
 
     parser.add_argument("--arch",
@@ -258,7 +274,7 @@ def main():
         if 'OpenCV_DIR' not in os.environ:
             os.environ["OpenCV_DIR"] = "C:\\Program Files\\OpenCV-v3.4.1\\build"
         if 'CUDA_PATH' not in os.environ:
-            os.environ["CUDA_PATH"] = "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v9.2"
+            os.environ["CUDA_PATH"] = "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v10.2"
         if 'MKL_ROOT' not in os.environ:
             os.environ["MKL_ROOT"] = "C:\\Program Files (x86)\\IntelSWTools\\compilers_and_libraries\\windows\\mkl"
         windows_build(args)

ci/docker/install/centos7_python.sh

@@ -29,4 +29,5 @@ yum -y install python36u
 # Install PIP
 curl "https://bootstrap.pypa.io/get-pip.py" -o "get-pip.py"
 python3.6 get-pip.py
-pip3 install nose pylint numpy nose-timer requests h5py scipy==1.2.3
+# Restrict numpy version to < 1.19.0 due to https://github.com/apache/incubator-mxnet/issues/18600
+pip3 install nose pylint 'numpy>1.16.0,<1.19.0' nose-timer requests h5py scipy==1.2.3

ci/docker/install/requirements

@@ -26,7 +26,7 @@ h5py==2.8.0rc1
 mock==2.0.0
 nose==1.3.7
 nose-timer==0.7.3
-numpy>1.16.0,<2.0.0
+numpy>1.16.0,<1.19.0  # Restrict numpy version to < 1.19.0 due to https://github.com/apache/incubator-mxnet/issues/18600
 pylint==2.3.1  # pylint and astroid need to be aligned
 astroid==2.3.3  # pylint and astroid need to be aligned
 requests<2.19.0,>=2.18.4

ci/jenkins/Jenkinsfile_full

@@ -21,7 +21,7 @@
 // See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
 
 // timeout in minutes
-def max_time = 30
+def max_time = 60
 
 def buildJobs = [
     'centos-cpu',

ci/jenkins/Jenkinsfile_sanity

@@ -21,7 +21,7 @@
 // See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
 
 // timeout in minutes
-max_time = 180
+max_time = 60
 
 node('utility') {
   // Loading the utilities requires a node context unfortunately