Add style-checking for notebooks & refactor Travis config (#1522)

* add installation script for env

* Add run script for test/codestyle

* modify travis file

* fix misprints

* add pytest

* Add basic version checking

* try to fix FAST_VERSION==-1

* try to fix FAST_VERSION==-1[2]

* remove debug info

* fix flake8 problems with ignore list

* continue with flake (break pep8 in matutils)

* fix regexp for grep

* restore matutils

* echo files for flake8

* Add ipynb checking

* special mistakes in .ipynb for testing purposes

* Distinct file checking for ipynb

* remove mistakes from notebooks

.travis.yml

@@ -1,23 +1,22 @@
 sudo: false
+
+cache:
+  apt: true
+  directories:
+  - $HOME/.cache/pip
+  - $HOME/.ccache
+
 dist: trusty
 language: python
-python:
-  - "2.7"
-  - "3.5"
-  - "3.6"
-before_install:
-  - wget 'http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh' -O miniconda.sh
-  - chmod +x miniconda.sh
-  - ./miniconda.sh -b
-  - export PATH=/home/travis/miniconda2/bin:$PATH
-  - conda update --yes conda
-install:
-  - conda create --yes -n gensim-test python=$TRAVIS_PYTHON_VERSION pip atlas numpy==1.11.3 scipy==0.18.1
-  - source activate gensim-test
-  - python setup.py install
-  - pip install .[test]
-script:
-  - pip freeze
-  - python setup.py test
-  - pip install flake8
-  - continuous_integration/travis/flake8_diff.sh
+
+
+matrix:
+  include:
+    - env: PYTHON_VERSION="2.7" NUMPY_VERSION="1.11.3" SCIPY_VERSION="0.18.1" ONLY_CODESTYLE="yes"
+    - env: PYTHON_VERSION="2.7" NUMPY_VERSION="1.11.3" SCIPY_VERSION="0.18.1" ONLY_CODESTYLE="no"
+    - env: PYTHON_VERSION="3.5" NUMPY_VERSION="1.11.3" SCIPY_VERSION="0.18.1" ONLY_CODESTYLE="no"
+    - env: PYTHON_VERSION="3.6" NUMPY_VERSION="1.11.3" SCIPY_VERSION="0.18.1" ONLY_CODESTYLE="no"
+
+
+install: source continuous_integration/travis/install.sh
+script: bash continuous_integration/travis/run.sh

continuous_integration/travis/flake8_diff.sh

@@ -118,7 +118,13 @@ echo '--------------------------------------------------------------------------
 # Excluding vec files since they contain non-utf8 content and flake8 raises exception for non-utf8 input
 # We need the following command to exit with 0 hence the echo in case
 # there is no match
-MODIFIED_FILES="$(git diff --name-only $COMMIT_RANGE -- . ':(exclude)*.vec' || echo "no_match")"
+MODIFIED_PY_FILES="$(git diff --name-only $COMMIT_RANGE | grep '[a-zA-Z0-9]*.py$' || echo "no_match")"
+MODIFIED_IPYNB_FILES="$(git diff --name-only $COMMIT_RANGE | grep '[a-zA-Z0-9]*.ipynb$' || echo "no_match")"
+
+
+echo "*.py files: " $MODIFIED_PY_FILES
+echo "*.ipynb files: " $MODIFIED_IPYNB_FILES
+
 
 check_files() {
     files="$1"
@@ -131,9 +137,19 @@ check_files() {
     fi
 }
 
-if [[ "$MODIFIED_FILES" == "no_match" ]]; then
-    echo "No file has been modified"
+if [[ "$MODIFIED_PY_FILES" == "no_match" ]]; then
+    echo "No .py files has been modified"
 else
-    check_files "$(echo "$MODIFIED_FILES" )" "--ignore=E501,E731,E12,W503 --exclude=*.sh,*.md,*.yml,*.rst,*.ipynb,*.txt,*.csv,*.vec,Dockerfile*,*.c,*.pyx,*.inc,*.html"
+    check_files "$(echo "$MODIFIED_PY_FILES" )" "--ignore=E501,E731,E12,W503"
 fi
 echo -e "No problem detected by flake8\n"
+
+if [[ "$MODIFIED_IPYNB_FILES" == "no_match" ]]; then
+    echo "No .ipynb file has been modified"
+else
+    for fname in ${MODIFIED_IPYNB_FILES}
+    do
+        echo "File: $fname"
+        jupyter nbconvert --to script --stdout $fname | flake8 - --show-source --ignore=E501,E731,E12,W503,E402 --builtins=get_ipython || true
+    done
+fi

continuous_integration/travis/install.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -e
+
+deactivate
+wget 'http://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh' -O miniconda.sh
+chmod +x miniconda.sh && ./miniconda.sh -b
+export PATH=/home/travis/miniconda2/bin:$PATH
+conda update --yes conda
+
+
+conda create --yes -n gensim-test python=$PYTHON_VERSION pip atlas flake8 jupyter numpy==$NUMPY_VERSION scipy==$SCIPY_VERSION && source activate gensim-test
+pip install . && pip install .[test]

continuous_integration/travis/run.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e
+
+pip freeze
+
+if [[ "$ONLY_CODESTYLE" == "yes" ]]; then
+	continuous_integration/travis/flake8_diff.sh
+else
+	python setup.py test
+fi

docs/notebooks/deepir.ipynb

@@ -468,4 +468,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 0
-}
+}