From 561fc5ec2f350b9b791f0331ac44a5aed4c2c5ef Mon Sep 17 00:00:00 2001 From: Naveen Swamy Date: Mon, 4 Jun 2018 23:38:22 -0700 Subject: [PATCH 1/4] changes made to publish to maven --- Makefile | 18 ++- .../assembly/linux-x86_64-cpu/pom.xml | 31 ++++- .../assembly/linux-x86_64-gpu/pom.xml | 28 +++++ scala-package/assembly/osx-x86_64-cpu/pom.xml | 36 +++++- scala-package/assembly/pom.xml | 2 +- scala-package/core/pom.xml | 6 +- scala-package/dev/compile-mxnet-backend.sh | 107 ++++++++++++++++++ scala-package/examples/pom.xml | 6 +- scala-package/infer/pom.xml | 4 +- scala-package/init-native/osx-x86_64/pom.xml | 4 +- scala-package/init-native/pom.xml | 2 +- scala-package/init/pom.xml | 24 +++- scala-package/macros/pom.xml | 6 +- scala-package/native/osx-x86_64-cpu/pom.xml | 4 +- scala-package/native/pom.xml | 2 +- scala-package/pom.xml | 14 ++- scala-package/spark/pom.xml | 4 +- 17 files changed, 259 insertions(+), 39 deletions(-) create mode 100755 scala-package/dev/compile-mxnet-backend.sh diff --git a/Makefile b/Makefile index 3317fc53b11b..9442b9083320 100644 --- a/Makefile +++ b/Makefile @@ -576,6 +576,8 @@ scalaclean: scalapkg: (cd $(ROOTDIR)/scala-package; \ mvn package -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \ + -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ + -Dmxnetdir="$(ROOTDIR)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ -Dcurrent_libdir="$(ROOTDIR)/lib" \ -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") @@ -583,12 +585,16 @@ scalapkg: scalatest: (cd $(ROOTDIR)/scala-package; \ mvn verify -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \ + -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ + -Dmxnetdir="$(ROOTDIR)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a" $(SCALA_TEST_ARGS)) scalainstall: (cd $(ROOTDIR)/scala-package; \ mvn install -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -DskipTests -Dcxx="$(CXX)" \ + -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ + -Dmxnetdir="$(ROOTDIR)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") @@ -596,23 +602,25 @@ scalarelease-dryrun: (cd $(ROOTDIR)/scala-package; \ mvn release:clean release:prepare -DdryRun=true -DautoVersionSubmodules=true \ -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \ - -Darguments=""-DskipTests\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") + -Darguments=""-Dmxnetdir=\""$(ROOTDIR)\""\ -Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") -scalarelease: +scalarelease-prepare: (cd $(ROOTDIR)/scala-package; \ mvn release:clean release:prepare -DautoVersionSubmodules=true \ -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \ - -Darguments=""-DskipTests\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") + -Darguments=""-Dmxnetdir=\""$(ROOTDIR)\""\ -Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") scalarelease-perform: (cd $(ROOTDIR)/scala-package; \ - mvn release:clean release:prepare -DautoVersionSubmodules=true \ + mvn release:perform -DautoVersionSubmodules=true \ -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \ - -Darguments=""-DskipTests\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") + -Darguments=""-Dmxnetdir=\""$(ROOTDIR)\""\ -Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") scaladeploy: (cd $(ROOTDIR)/scala-package; \ mvn deploy -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests -Dcxx="$(CXX)" \ + -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ + -Dmxnetdir="$(ROOTDIR)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") diff --git a/scala-package/assembly/linux-x86_64-cpu/pom.xml b/scala-package/assembly/linux-x86_64-cpu/pom.xml index 869c14cf7cdb..cb2de87a1cff 100644 --- a/scala-package/assembly/linux-x86_64-cpu/pom.xml +++ b/scala-package/assembly/linux-x86_64-cpu/pom.xml @@ -36,18 +36,39 @@ - org.apache.maven.plugins - maven-deploy-plugin - false + org.codehaus.mojo + flatten-maven-plugin + - false + + remove + + + + + flatten + process-resources + + flatten + + + + + flatten.clean + clean + + clean + + + org.apache.maven.plugins maven-deploy-plugin + false - true + false diff --git a/scala-package/assembly/linux-x86_64-gpu/pom.xml b/scala-package/assembly/linux-x86_64-gpu/pom.xml index 386072abb13b..d417b0fe3b46 100644 --- a/scala-package/assembly/linux-x86_64-gpu/pom.xml +++ b/scala-package/assembly/linux-x86_64-gpu/pom.xml @@ -35,6 +35,34 @@ + + org.codehaus.mojo + flatten-maven-plugin + + + + remove + + + + + + flatten + process-resources + + flatten + + + + + flatten.clean + clean + + clean + + + + org.apache.maven.plugins maven-deploy-plugin diff --git a/scala-package/assembly/osx-x86_64-cpu/pom.xml b/scala-package/assembly/osx-x86_64-cpu/pom.xml index e0dd5fec3387..6d89c16ad8f2 100644 --- a/scala-package/assembly/osx-x86_64-cpu/pom.xml +++ b/scala-package/assembly/osx-x86_64-cpu/pom.xml @@ -4,7 +4,7 @@ org.apache.mxnet mxnet-full-parent_2.11 - 1.2.0 + 1.2.0-SNAPSHOT ../pom.xml @@ -16,23 +16,51 @@ org.apache.mxnet mxnet-core_${scala.binary.version} - 1.2.0 + 1.2.0-SNAPSHOT org.apache.mxnet libmxnet-scala-osx-x86_64-cpu - 1.2.0 + 1.2.0-SNAPSHOT jnilib org.apache.mxnet mxnet-infer_${scala.binary.version} - 1.2.0 + 1.2.0-SNAPSHOT + + org.codehaus.mojo + flatten-maven-plugin + + + + remove + + + + + + flatten + process-resources + + flatten + + + + + flatten.clean + clean + + clean + + + + org.apache.maven.plugins maven-deploy-plugin diff --git a/scala-package/assembly/pom.xml b/scala-package/assembly/pom.xml index 8d2df25a8df0..2c2a4fc8cd24 100644 --- a/scala-package/assembly/pom.xml +++ b/scala-package/assembly/pom.xml @@ -4,7 +4,7 @@ org.apache.mxnet mxnet-parent_2.11 - 1.2.0 + 1.2.0-SNAPSHOT ../pom.xml diff --git a/scala-package/core/pom.xml b/scala-package/core/pom.xml index 711a5fb508d9..509e7326dc34 100644 --- a/scala-package/core/pom.xml +++ b/scala-package/core/pom.xml @@ -4,7 +4,7 @@ org.apache.mxnet mxnet-parent_2.11 - 1.2.0 + 1.2.0-SNAPSHOT ../pom.xml @@ -76,13 +76,13 @@ org.apache.mxnet mxnet-init_${scala.binary.version} - 1.2.0 + 1.2.0-SNAPSHOT provided org.apache.mxnet mxnet-macros_${scala.binary.version} - 1.2.0 + 1.2.0-SNAPSHOT provided diff --git a/scala-package/dev/compile-mxnet-backend.sh b/scala-package/dev/compile-mxnet-backend.sh new file mode 100755 index 000000000000..7a04b77a853b --- /dev/null +++ b/scala-package/dev/compile-mxnet-backend.sh @@ -0,0 +1,107 @@ +#!/bin/bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# (Yizhi) This is mainly inspired by the script in apache/spark. +# I did some modificaiton to get it with our project. +# + +set -e +echo "Compiling MXNet Backend, Hang tight!....." + +if [[ ($# -ne 2) || ( $1 == "--help") || $1 == "-h" ]]; then + echo "Usage: [-h|--help] " 1>&2 + exit 1 +fi + +PLATFORM=$1 +MXNETDIR=$2 + +# below routine shamelessly copied from +# https://github.com/apache/incubator-mxnet/blob/master/setup-utils/install-mxnet-osx-python.sh + +chkret() { + cmd=$* + echo "$cmd" + $cmd + ret=$? + if [[ ${ret} != 0 ]]; then + echo " " + echo "ERROR: Return value non-zero for: $cmd" + echo " " + exit 1 + fi +} # chkret() + +UNAME=`uname -s` +chkret pushd $MXNETDIR +chkret git submodule update --init --recursive + +# don't want to overwrite an existing config file +cp make/config.mk ./config.mk + +if [[ $PLATFORM == "osx-x86_64-cpu" ]]; +then + echo "Building MXNet Backend on MAC OS" + echo "ADD_CFLAGS += -I/usr/local/opt/opencv@2/include" >> ./config.mk + echo "ADD_CFLAGS += -I/usr/local/opt/openblas/include" >> ./config.mk + echo "ADD_LDFLAGS += -L/usr/local/opt/opencv@2/lib" >> ./config.mk + echo "ADD_LDFLAGS += -L/usr/local/opt/openblas/lib" >> ./config.mk + echo "USE_OPENMP = 0" >> ./config.mk + echo "USE_LAPACK_PATH = /usr/local/opt/lapack/lib" >> ./config.mk + make -j$(sysctl -n hw.ncpu) +elif [[ $PLATFORM == "linux-x86_64-cpu" ]]; +then + echo "Building MXNet Backend on Linux CPU" + echo "ADD_CFLAGS += -I/usr/local/include/opencv" >> ./config.mk + echo "ADD_LDFLAGS += -L/usr/local/lib" >> ./config.mk + echo "USE_OPENCV=1" >> ./config.mk + echo "USE_OPENMP=1" >> ./config.mk + echo "USE_BLAS=openblas" >> ./config.mk + echo "USE_LAPACK=1" >> ./config.mk + echo "USE_DIST_KVSTORE=1" >> ./config.mk + echo "USE_S3=1" >> ./config.mk + make -j$(cat /proc/cpuinfo | awk '/^processor/{print $3}' | tail -1) +elif [[ $PLATFORM == "linux-x86_64-gpu" ]] +then + echo "Building MXNet Backend on Linux GPU" + echo "Building MXNet Backend on Linux CPU" + echo "ADD_CFLAGS += -I/usr/local/include/opencv" >> ./config.mk + echo "ADD_LDFLAGS += -L/usr/local/lib" >> ./config.mk + echo "USE_OPENCV=1" >> ./config.mk + echo "USE_OPENMP=1" >> ./config.mk + echo "USE_BLAS=openblas" >> ./config.mk + echo "USE_LAPACK=1" >> ./config.mk + echo "USE_DIST_KVSTORE=1" >> ./config.mk + echo "USE_S3=1" >> ./config.mk + echo "USE_CUDA=1" >> ./config.mk + echo "USE_CUDNN=1" >> ./config.mk + echo "ADD_CFLAGS += -I/usr/local/cuda/include" >> ./config.mk + echo "ADD_LDFLAGS += -L/usr/local/cuda/lib64/ " >> ./config.mk + #update th nccl version approriately + echo "ADD_LDFLAGS += -L/lib/nccl/cuda-9.0/lib " >> ./config.mk + eval "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/lib/nccl/cuda-9.0/lib" + eval "export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH" + make -j$(cat /proc/cpuinfo | awk '/^processor/{print $3}' | tail -1) + echo "Building MXNet Backend on Linux GPU" +else + echo "MY ALIEN OVERLOADS HAVE NOT TOLD WHAT TO DO FOR INVALID INPUT !!!" + echo "Currently supported platforms: osx-x86_64-cpu or linux-x86_64-cpu or linux-x86_64-gpu" +fi +chkret popd +echo "done building MXNet Backend" +exit 0 diff --git a/scala-package/examples/pom.xml b/scala-package/examples/pom.xml index 721f8824aa0f..1d187caf2eb3 100644 --- a/scala-package/examples/pom.xml +++ b/scala-package/examples/pom.xml @@ -4,7 +4,7 @@ org.apache.mxnet mxnet-parent_2.11 - 1.2.0 + 1.2.0-SNAPSHOT ../pom.xml @@ -150,13 +150,13 @@ org.apache.mxnet mxnet-core_${scala.binary.version} - 1.2.0 + 1.2.0-SNAPSHOT provided org.apache.mxnet mxnet-infer_${scala.binary.version} - 1.2.0 + 1.2.0-SNAPSHOT provided diff --git a/scala-package/infer/pom.xml b/scala-package/infer/pom.xml index 56a35072888f..929a45e0d2f7 100644 --- a/scala-package/infer/pom.xml +++ b/scala-package/infer/pom.xml @@ -4,7 +4,7 @@ mxnet-parent_2.11 org.apache.mxnet - 1.2.0 + 1.2.0-SNAPSHOT ../pom.xml @@ -76,7 +76,7 @@ org.apache.mxnet mxnet-core_${scala.binary.version} - 1.2.0 + 1.2.0-SNAPSHOT provided diff --git a/scala-package/init-native/osx-x86_64/pom.xml b/scala-package/init-native/osx-x86_64/pom.xml index f2b2b3d8c096..54581e60b57c 100644 --- a/scala-package/init-native/osx-x86_64/pom.xml +++ b/scala-package/init-native/osx-x86_64/pom.xml @@ -4,7 +4,7 @@ org.apache.mxnet mxnet-scala-init-native-parent - 1.2.0 + 1.2.0-SNAPSHOT ../pom.xml @@ -18,7 +18,7 @@ org.apache.mxnet mxnet-init_${scala.binary.version} - 1.2.0 + 1.2.0-SNAPSHOT jar compile diff --git a/scala-package/init-native/pom.xml b/scala-package/init-native/pom.xml index 3a56d5892597..e7834a094405 100644 --- a/scala-package/init-native/pom.xml +++ b/scala-package/init-native/pom.xml @@ -4,7 +4,7 @@ org.apache.mxnet mxnet-parent_2.11 - 1.2.0 + 1.2.0-SNAPSHOT ../pom.xml diff --git a/scala-package/init/pom.xml b/scala-package/init/pom.xml index 35ab07dfab4a..5fb5cbb6319a 100644 --- a/scala-package/init/pom.xml +++ b/scala-package/init/pom.xml @@ -4,8 +4,8 @@ org.apache.mxnet mxnet-parent_2.11 - 1.2.0 - + 1.2.0-SNAPSHOT + ../pom.xml mxnet-init_2.11 @@ -32,8 +32,27 @@ + + + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + + compile-mxnet-backend + compile + + exec + + + bash + ${project.parent.basedir}/dev/compile-mxnet-backend.sh ${build.platform} ${project.parent.basedir}/../ + + + + org.apache.maven.plugins maven-deploy-plugin @@ -43,5 +62,4 @@ - diff --git a/scala-package/macros/pom.xml b/scala-package/macros/pom.xml index e692d95f6052..1b33561a18b8 100644 --- a/scala-package/macros/pom.xml +++ b/scala-package/macros/pom.xml @@ -4,7 +4,7 @@ org.apache.mxnet mxnet-parent_2.11 - 1.2.0 + 1.2.0-SNAPSHOT ../pom.xml @@ -51,13 +51,13 @@ org.apache.mxnet mxnet-init_${scala.binary.version} - 1.2.0 + 1.2.0-SNAPSHOT provided org.apache.mxnet libmxnet-init-scala-${platform} - 1.2.0 + 1.2.0-SNAPSHOT provided ${libtype} diff --git a/scala-package/native/osx-x86_64-cpu/pom.xml b/scala-package/native/osx-x86_64-cpu/pom.xml index 01c3114e536a..8a68864627c2 100644 --- a/scala-package/native/osx-x86_64-cpu/pom.xml +++ b/scala-package/native/osx-x86_64-cpu/pom.xml @@ -4,7 +4,7 @@ org.apache.mxnet mxnet-scala-native-parent - 1.2.0 + 1.2.0-SNAPSHOT ../pom.xml @@ -18,7 +18,7 @@ org.apache.mxnet mxnet-core_${scala.binary.version} - 1.2.0 + 1.2.0-SNAPSHOT jar compile diff --git a/scala-package/native/pom.xml b/scala-package/native/pom.xml index 3280ebf33681..594820af6f99 100644 --- a/scala-package/native/pom.xml +++ b/scala-package/native/pom.xml @@ -4,7 +4,7 @@ org.apache.mxnet mxnet-parent_2.11 - 1.2.0 + 1.2.0-SNAPSHOT ../pom.xml diff --git a/scala-package/pom.xml b/scala-package/pom.xml index 63208c5fb121..9ebe8adf066f 100644 --- a/scala-package/pom.xml +++ b/scala-package/pom.xml @@ -8,7 +8,7 @@ org.apache.mxnet mxnet-parent_2.11 - 1.2.0 + 1.2.0-SNAPSHOT MXNet Scala Package - Parent https://github.com/apache/incubator-mxnet/tree/master/scala-package @@ -30,12 +30,14 @@ scm:git:git@github.com:apache/incubator-mxnet.git scm:git:git@github.com:apache/incubator-mxnet.git https://github.com/apache/incubator-mxnet - mxnet-parent_2.11-1.2.0 + HEAD 2.11.8 2.11 + + pom @@ -148,6 +150,14 @@ + + org.apache.maven.plugins + maven-release-plugin + + true + false + + org.apache.maven.plugins maven-deploy-plugin diff --git a/scala-package/spark/pom.xml b/scala-package/spark/pom.xml index 41c090529e10..e2554c444f1b 100644 --- a/scala-package/spark/pom.xml +++ b/scala-package/spark/pom.xml @@ -4,7 +4,7 @@ org.apache.mxnet mxnet-parent_2.11 - 1.2.0 + 1.2.0-SNAPSHOT ../pom.xml @@ -31,7 +31,7 @@ org.apache.mxnet mxnet-core_${scala.binary.version} - 1.2.0 + 1.2.0-SNAPSHOT provided From ce72a3cf21fc60fcfdd8b6596871553590386e96 Mon Sep 17 00:00:00 2001 From: Naveen Swamy Date: Wed, 6 Jun 2018 09:08:05 -0700 Subject: [PATCH 2/4] update VERSION to 1.2.1-SNAPSHOT --- scala-package/assembly/linux-x86_64-cpu/pom.xml | 8 ++++---- scala-package/assembly/linux-x86_64-gpu/pom.xml | 8 ++++---- scala-package/assembly/osx-x86_64-cpu/pom.xml | 8 ++++---- scala-package/assembly/pom.xml | 2 +- scala-package/core/pom.xml | 6 +++--- scala-package/examples/pom.xml | 6 +++--- scala-package/infer/pom.xml | 4 ++-- scala-package/init-native/linux-x86_64/pom.xml | 4 ++-- scala-package/init-native/osx-x86_64/pom.xml | 4 ++-- scala-package/init-native/pom.xml | 2 +- scala-package/init/pom.xml | 2 +- scala-package/macros/pom.xml | 6 +++--- scala-package/native/linux-x86_64-cpu/pom.xml | 4 ++-- scala-package/native/linux-x86_64-gpu/pom.xml | 4 ++-- scala-package/native/osx-x86_64-cpu/pom.xml | 4 ++-- scala-package/native/pom.xml | 2 +- scala-package/pom.xml | 2 +- scala-package/spark/pom.xml | 4 ++-- 18 files changed, 40 insertions(+), 40 deletions(-) diff --git a/scala-package/assembly/linux-x86_64-cpu/pom.xml b/scala-package/assembly/linux-x86_64-cpu/pom.xml index cb2de87a1cff..5a91566d4824 100644 --- a/scala-package/assembly/linux-x86_64-cpu/pom.xml +++ b/scala-package/assembly/linux-x86_64-cpu/pom.xml @@ -6,7 +6,7 @@ org.apache.mxnet mxnet-full-parent_2.11 - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT ../pom.xml @@ -18,18 +18,18 @@ org.apache.mxnet mxnet-core_${scala.binary.version} - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT org.apache.mxnet libmxnet-scala-linux-x86_64-cpu - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT so org.apache.mxnet mxnet-infer_${scala.binary.version} - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT diff --git a/scala-package/assembly/linux-x86_64-gpu/pom.xml b/scala-package/assembly/linux-x86_64-gpu/pom.xml index d417b0fe3b46..bdad3fd1c9f0 100644 --- a/scala-package/assembly/linux-x86_64-gpu/pom.xml +++ b/scala-package/assembly/linux-x86_64-gpu/pom.xml @@ -6,7 +6,7 @@ org.apache.mxnet mxnet-full-parent_2.11 - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT ../pom.xml @@ -18,18 +18,18 @@ org.apache.mxnet mxnet-core_${scala.binary.version} - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT org.apache.mxnet libmxnet-scala-linux-x86_64-gpu - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT so org.apache.mxnet mxnet-infer_${scala.binary.version} - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT diff --git a/scala-package/assembly/osx-x86_64-cpu/pom.xml b/scala-package/assembly/osx-x86_64-cpu/pom.xml index 6d89c16ad8f2..9c9cac309ca7 100644 --- a/scala-package/assembly/osx-x86_64-cpu/pom.xml +++ b/scala-package/assembly/osx-x86_64-cpu/pom.xml @@ -4,7 +4,7 @@ org.apache.mxnet mxnet-full-parent_2.11 - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT ../pom.xml @@ -16,18 +16,18 @@ org.apache.mxnet mxnet-core_${scala.binary.version} - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT org.apache.mxnet libmxnet-scala-osx-x86_64-cpu - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT jnilib org.apache.mxnet mxnet-infer_${scala.binary.version} - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT diff --git a/scala-package/assembly/pom.xml b/scala-package/assembly/pom.xml index 2c2a4fc8cd24..456f453f80a1 100644 --- a/scala-package/assembly/pom.xml +++ b/scala-package/assembly/pom.xml @@ -4,7 +4,7 @@ org.apache.mxnet mxnet-parent_2.11 - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT ../pom.xml diff --git a/scala-package/core/pom.xml b/scala-package/core/pom.xml index 509e7326dc34..fb5ea2bab6fd 100644 --- a/scala-package/core/pom.xml +++ b/scala-package/core/pom.xml @@ -4,7 +4,7 @@ org.apache.mxnet mxnet-parent_2.11 - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT ../pom.xml @@ -76,13 +76,13 @@ org.apache.mxnet mxnet-init_${scala.binary.version} - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT provided org.apache.mxnet mxnet-macros_${scala.binary.version} - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT provided diff --git a/scala-package/examples/pom.xml b/scala-package/examples/pom.xml index 1d187caf2eb3..5ae86f48f89a 100644 --- a/scala-package/examples/pom.xml +++ b/scala-package/examples/pom.xml @@ -4,7 +4,7 @@ org.apache.mxnet mxnet-parent_2.11 - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT ../pom.xml @@ -150,13 +150,13 @@ org.apache.mxnet mxnet-core_${scala.binary.version} - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT provided org.apache.mxnet mxnet-infer_${scala.binary.version} - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT provided diff --git a/scala-package/infer/pom.xml b/scala-package/infer/pom.xml index 929a45e0d2f7..daf4b4782c84 100644 --- a/scala-package/infer/pom.xml +++ b/scala-package/infer/pom.xml @@ -4,7 +4,7 @@ mxnet-parent_2.11 org.apache.mxnet - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT ../pom.xml @@ -76,7 +76,7 @@ org.apache.mxnet mxnet-core_${scala.binary.version} - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT provided diff --git a/scala-package/init-native/linux-x86_64/pom.xml b/scala-package/init-native/linux-x86_64/pom.xml index d3653a5b133f..c35e905606b4 100644 --- a/scala-package/init-native/linux-x86_64/pom.xml +++ b/scala-package/init-native/linux-x86_64/pom.xml @@ -6,7 +6,7 @@ org.apache.mxnet mxnet-scala-init-native-parent - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT ../pom.xml @@ -20,7 +20,7 @@ org.apache.mxnet mxnet-init_${scala.binary.version} - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT jar compile diff --git a/scala-package/init-native/osx-x86_64/pom.xml b/scala-package/init-native/osx-x86_64/pom.xml index 54581e60b57c..812ad643c316 100644 --- a/scala-package/init-native/osx-x86_64/pom.xml +++ b/scala-package/init-native/osx-x86_64/pom.xml @@ -4,7 +4,7 @@ org.apache.mxnet mxnet-scala-init-native-parent - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT ../pom.xml @@ -18,7 +18,7 @@ org.apache.mxnet mxnet-init_${scala.binary.version} - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT jar compile diff --git a/scala-package/init-native/pom.xml b/scala-package/init-native/pom.xml index e7834a094405..55c724dd13ff 100644 --- a/scala-package/init-native/pom.xml +++ b/scala-package/init-native/pom.xml @@ -4,7 +4,7 @@ org.apache.mxnet mxnet-parent_2.11 - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT ../pom.xml diff --git a/scala-package/init/pom.xml b/scala-package/init/pom.xml index 5fb5cbb6319a..3ca834ce453c 100644 --- a/scala-package/init/pom.xml +++ b/scala-package/init/pom.xml @@ -4,7 +4,7 @@ org.apache.mxnet mxnet-parent_2.11 - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT ../pom.xml diff --git a/scala-package/macros/pom.xml b/scala-package/macros/pom.xml index 1b33561a18b8..fee1dd226ce1 100644 --- a/scala-package/macros/pom.xml +++ b/scala-package/macros/pom.xml @@ -4,7 +4,7 @@ org.apache.mxnet mxnet-parent_2.11 - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT ../pom.xml @@ -51,13 +51,13 @@ org.apache.mxnet mxnet-init_${scala.binary.version} - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT provided org.apache.mxnet libmxnet-init-scala-${platform} - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT provided ${libtype} diff --git a/scala-package/native/linux-x86_64-cpu/pom.xml b/scala-package/native/linux-x86_64-cpu/pom.xml index 94c2a744f637..0f2bc84d7f0a 100644 --- a/scala-package/native/linux-x86_64-cpu/pom.xml +++ b/scala-package/native/linux-x86_64-cpu/pom.xml @@ -6,7 +6,7 @@ org.apache.mxnet mxnet-scala-native-parent - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT ../pom.xml @@ -20,7 +20,7 @@ org.apache.mxnet mxnet-core_${scala.binary.version} - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT jar compile diff --git a/scala-package/native/linux-x86_64-gpu/pom.xml b/scala-package/native/linux-x86_64-gpu/pom.xml index 3d2969bdd4bf..8b9f875e945b 100644 --- a/scala-package/native/linux-x86_64-gpu/pom.xml +++ b/scala-package/native/linux-x86_64-gpu/pom.xml @@ -6,7 +6,7 @@ org.apache.mxnet mxnet-scala-native-parent - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT ../pom.xml @@ -20,7 +20,7 @@ org.apache.mxnet mxnet-core_${scala.binary.version} - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT jar compile diff --git a/scala-package/native/osx-x86_64-cpu/pom.xml b/scala-package/native/osx-x86_64-cpu/pom.xml index 8a68864627c2..1296050d2c4d 100644 --- a/scala-package/native/osx-x86_64-cpu/pom.xml +++ b/scala-package/native/osx-x86_64-cpu/pom.xml @@ -4,7 +4,7 @@ org.apache.mxnet mxnet-scala-native-parent - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT ../pom.xml @@ -18,7 +18,7 @@ org.apache.mxnet mxnet-core_${scala.binary.version} - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT jar compile diff --git a/scala-package/native/pom.xml b/scala-package/native/pom.xml index 594820af6f99..a9b2ec93c9d5 100644 --- a/scala-package/native/pom.xml +++ b/scala-package/native/pom.xml @@ -4,7 +4,7 @@ org.apache.mxnet mxnet-parent_2.11 - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT ../pom.xml diff --git a/scala-package/pom.xml b/scala-package/pom.xml index 9ebe8adf066f..fe16af6b103d 100644 --- a/scala-package/pom.xml +++ b/scala-package/pom.xml @@ -8,7 +8,7 @@ org.apache.mxnet mxnet-parent_2.11 - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT MXNet Scala Package - Parent https://github.com/apache/incubator-mxnet/tree/master/scala-package diff --git a/scala-package/spark/pom.xml b/scala-package/spark/pom.xml index e2554c444f1b..8ba4241337b2 100644 --- a/scala-package/spark/pom.xml +++ b/scala-package/spark/pom.xml @@ -4,7 +4,7 @@ org.apache.mxnet mxnet-parent_2.11 - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT ../pom.xml @@ -31,7 +31,7 @@ org.apache.mxnet mxnet-core_${scala.binary.version} - 1.2.0-SNAPSHOT + 1.2.1-SNAPSHOT provided From ddfc0f29868fccbc28c8753aa3ee825299025546 Mon Sep 17 00:00:00 2001 From: Naveen Swamy Date: Wed, 6 Jun 2018 19:49:01 -0700 Subject: [PATCH 3/4] compile backend only when passed flag --- Makefile | 6 +++--- scala-package/dev/compile-mxnet-backend.sh | 10 ++++++++-- scala-package/pom.xml | 2 +- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 9442b9083320..577cfd4deeb5 100644 --- a/Makefile +++ b/Makefile @@ -602,19 +602,19 @@ scalarelease-dryrun: (cd $(ROOTDIR)/scala-package; \ mvn release:clean release:prepare -DdryRun=true -DautoVersionSubmodules=true \ -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \ - -Darguments=""-Dmxnetdir=\""$(ROOTDIR)\""\ -Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") + -Darguments=""-Dcompile\.backend=true -Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") scalarelease-prepare: (cd $(ROOTDIR)/scala-package; \ mvn release:clean release:prepare -DautoVersionSubmodules=true \ -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \ - -Darguments=""-Dmxnetdir=\""$(ROOTDIR)\""\ -Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") + -Darguments=""-Dcompile\.backend=true -Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") scalarelease-perform: (cd $(ROOTDIR)/scala-package; \ mvn release:perform -DautoVersionSubmodules=true \ -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \ - -Darguments=""-Dmxnetdir=\""$(ROOTDIR)\""\ -Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") + -Darguments=""-Dcompile\.backend=true -Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") scaladeploy: (cd $(ROOTDIR)/scala-package; \ diff --git a/scala-package/dev/compile-mxnet-backend.sh b/scala-package/dev/compile-mxnet-backend.sh index 7a04b77a853b..b1bd7f1e1480 100755 --- a/scala-package/dev/compile-mxnet-backend.sh +++ b/scala-package/dev/compile-mxnet-backend.sh @@ -23,13 +23,14 @@ set -e echo "Compiling MXNet Backend, Hang tight!....." -if [[ ($# -ne 2) || ( $1 == "--help") || $1 == "-h" ]]; then - echo "Usage: [-h|--help] " 1>&2 +if [[ ($# -ne 3) || ( $1 == "--help") || $1 == "-h" ]]; then + echo "Usage: [-h|--help] " 1>&2 exit 1 fi PLATFORM=$1 MXNETDIR=$2 +COMPILE_BACKEND=$3 # below routine shamelessly copied from # https://github.com/apache/incubator-mxnet/blob/master/setup-utils/install-mxnet-osx-python.sh @@ -47,6 +48,11 @@ chkret() { fi } # chkret() +if [[$COMPILE_BACKEND == "false"]]; +then + exit 0 +fi + UNAME=`uname -s` chkret pushd $MXNETDIR chkret git submodule update --init --recursive diff --git a/scala-package/pom.xml b/scala-package/pom.xml index fe16af6b103d..9e23e425611a 100644 --- a/scala-package/pom.xml +++ b/scala-package/pom.xml @@ -37,7 +37,7 @@ 2.11.8 2.11 - + false pom From 8a988e260d4cb51d16fc1627fda3520de266e9b6 Mon Sep 17 00:00:00 2001 From: Naveen Swamy Date: Wed, 6 Jun 2018 19:50:08 -0700 Subject: [PATCH 4/4] remove the compile.backend flag --- Makefile | 10 ++-- scala-package/dev/compile-mxnet-backend.sh | 10 +--- scala-package/init/pom.xml | 62 +++++++++++----------- scala-package/pom.xml | 1 - 4 files changed, 37 insertions(+), 46 deletions(-) diff --git a/Makefile b/Makefile index 577cfd4deeb5..4db73195a6fd 100644 --- a/Makefile +++ b/Makefile @@ -577,7 +577,6 @@ scalapkg: (cd $(ROOTDIR)/scala-package; \ mvn package -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \ -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ - -Dmxnetdir="$(ROOTDIR)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ -Dcurrent_libdir="$(ROOTDIR)/lib" \ -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") @@ -586,7 +585,6 @@ scalatest: (cd $(ROOTDIR)/scala-package; \ mvn verify -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -Dcxx="$(CXX)" \ -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ - -Dmxnetdir="$(ROOTDIR)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a" $(SCALA_TEST_ARGS)) @@ -594,7 +592,6 @@ scalainstall: (cd $(ROOTDIR)/scala-package; \ mvn install -P$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) -DskipTests -Dcxx="$(CXX)" \ -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ - -Dmxnetdir="$(ROOTDIR)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") @@ -602,25 +599,24 @@ scalarelease-dryrun: (cd $(ROOTDIR)/scala-package; \ mvn release:clean release:prepare -DdryRun=true -DautoVersionSubmodules=true \ -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \ - -Darguments=""-Dcompile\.backend=true -Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") + -Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") scalarelease-prepare: (cd $(ROOTDIR)/scala-package; \ mvn release:clean release:prepare -DautoVersionSubmodules=true \ -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \ - -Darguments=""-Dcompile\.backend=true -Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") + -Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") scalarelease-perform: (cd $(ROOTDIR)/scala-package; \ mvn release:perform -DautoVersionSubmodules=true \ -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \ - -Darguments=""-Dcompile\.backend=true -Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") + -Darguments=""-Dbuild\.platform=\""$(SCALA_PKG_PROFILE)\""\ -DskipTests\ -Dcflags=\""$(CFLAGS)\""\ -Dcxx=\""$(CXX)\""\ -Dldflags=\""$(LDFLAGS)\""\ -Dlddeps=\""$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a\"""") scaladeploy: (cd $(ROOTDIR)/scala-package; \ mvn deploy -Papache-release,$(SCALA_PKG_PROFILE),$(SCALA_VERSION_PROFILE) \-DskipTests -Dcxx="$(CXX)" \ -Dbuild.platform="$(SCALA_PKG_PROFILE)" \ - -Dmxnetdir="$(ROOTDIR)" \ -Dcflags="$(CFLAGS)" -Dldflags="$(LDFLAGS)" \ -Dlddeps="$(LIB_DEP) $(ROOTDIR)/lib/libmxnet.a") diff --git a/scala-package/dev/compile-mxnet-backend.sh b/scala-package/dev/compile-mxnet-backend.sh index b1bd7f1e1480..43517fc826d8 100755 --- a/scala-package/dev/compile-mxnet-backend.sh +++ b/scala-package/dev/compile-mxnet-backend.sh @@ -24,13 +24,12 @@ set -e echo "Compiling MXNet Backend, Hang tight!....." if [[ ($# -ne 3) || ( $1 == "--help") || $1 == "-h" ]]; then - echo "Usage: [-h|--help] " 1>&2 + echo "Usage: [-h|--help] " 1>&2 exit 1 fi - PLATFORM=$1 MXNETDIR=$2 -COMPILE_BACKEND=$3 + # below routine shamelessly copied from # https://github.com/apache/incubator-mxnet/blob/master/setup-utils/install-mxnet-osx-python.sh @@ -48,11 +47,6 @@ chkret() { fi } # chkret() -if [[$COMPILE_BACKEND == "false"]]; -then - exit 0 -fi - UNAME=`uname -s` chkret pushd $MXNETDIR chkret git submodule update --init --recursive diff --git a/scala-package/init/pom.xml b/scala-package/init/pom.xml index 3ca834ce453c..38bd130fd24a 100644 --- a/scala-package/init/pom.xml +++ b/scala-package/init/pom.xml @@ -30,36 +30,38 @@ linux-x86_64-gpu - - - - - - - org.codehaus.mojo - exec-maven-plugin - 1.6.0 - - - compile-mxnet-backend - compile - - exec - + + apache-release + + + + + org.codehaus.mojo + exec-maven-plugin + 1.6.0 + + + compile-mxnet-backend + compile + + exec + + + bash + ${project.parent.basedir}/dev/compile-mxnet-backend.sh ${build.platform} ${project.parent.basedir}/../ + + + + + + org.apache.maven.plugins + maven-deploy-plugin - bash - ${project.parent.basedir}/dev/compile-mxnet-backend.sh ${build.platform} ${project.parent.basedir}/../ + true - - - - - org.apache.maven.plugins - maven-deploy-plugin - - true - - - - + + + + + diff --git a/scala-package/pom.xml b/scala-package/pom.xml index 9e23e425611a..1643c3682d5f 100644 --- a/scala-package/pom.xml +++ b/scala-package/pom.xml @@ -37,7 +37,6 @@ 2.11.8 2.11 - false pom