Use nightly wheels where available to run tests and support NumPy 2.0 (…

…#3427) * Use nightly wheels where available to run tests * Use nightly wheels where available to run tests * bump up min numba version * Update nightly-deps-test.yml
scipp · Jun 19, 2024 · 0682656 · 0682656
1 parent 412cec8
commit 0682656
Show file tree

Hide file tree

Showing 17 changed files with 121 additions and 29 deletions.
diff --git a/.github/workflows/nightly-deps-test.yml b/.github/workflows/nightly-deps-test.yml
@@ -0,0 +1,59 @@
+name: Run tests against nightly dependencies
+
+on:
+  schedule:
+    - cron: '0 0 * * 0'
+  workflow_dispatch:
+
+jobs:
+  build_and_test:
+    name: build and test ${{ matrix.variant.os }}, py${{ matrix.python-version }}
+    runs-on: ${{ matrix.variant.os }}
+    defaults:
+      run:
+        shell: bash -l {0}  # required for conda env
+    strategy:
+      fail-fast: false
+      matrix:
+        variant:
+        - {os: ubuntu-22.04, cmake-preset: ci-linux}
+        - {os: macos-11, cmake-preset: ci-macos}
+        - {os: windows-2019, cmake-preset: ci-windows}
+        python-version: ["3.10"]
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+          fetch-depth: 0  # history required so cmake can determine version
+
+      - uses: ilammy/msvc-dev-cmd@v1  # Required to set up MSVC dev environment for Ninja builds.
+
+      - name: Setup conda environment
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          micromamba-version: 1.5.6-0
+          environment-file: .buildconfig/${{ matrix.variant.cmake-preset }}.yml
+          cache-environment: true
+          create-args: python=${{ matrix.python-version }}
+
+      - name: ccache
+        uses: hendrikmuhs/ccache-action@v1.2.12
+        with:
+          key: ${{ matrix.variant.os }}
+
+      - name: Cache conan setup
+        id: conan-cache-key
+        run: |
+          echo "key=$(/bin/date -u "+%Y%m%d")" >> $GITHUB_OUTPUT
+          echo "path=$(conan config home)" >> $GITHUB_OUTPUT
+      - name: Cache conan
+        uses: actions/cache@v4
+        with:
+          path: ${{ steps.conan-cache-key.outputs.path }}
+          key: conan-${{ matrix.variant.os }}-${{ steps.conan-cache-key.outputs.key }}
+
+      - run: cmake --preset ${{ matrix.variant.cmake-preset }}
+      - run: cmake --build --preset build
+      - run: ctest --preset test
+      - run: tox -e nightlydeps -vvv
diff --git a/requirements/asv.txt b/requirements/asv.txt
@@ -15,7 +15,7 @@ build==1.2.1
     # via asv
 distlib==0.3.8
     # via virtualenv
-filelock==3.14.0
+filelock==3.15.1
     # via virtualenv
 importlib-metadata==7.1.0
     # via asv-runner

diff --git a/requirements/base.txt b/requirements/base.txt
@@ -5,5 +5,5 @@
 #
 #    pip-compile-multi
 #
-numpy==1.26.4
+numpy==2.0.0
     # via -r base.in
diff --git a/requirements/build.txt b/requirements/build.txt
@@ -11,12 +11,14 @@ certifi==2024.6.2
     # via requests
 charset-normalizer==3.3.2
     # via requests
-cmake==3.29.5
+cmake==3.29.5.1
     # via -r build.in
 colorama==0.4.6
     # via conan
 conan==1.64.1
     # via -r build.in
+distro==1.8.0
+    # via conan
 fasteners==0.19
     # via conan
 idna==3.7
@@ -49,7 +51,7 @@ six==1.16.0
     #   python-dateutil
 tqdm==4.66.4
     # via conan
-urllib3==1.26.18
+urllib3==1.26.19
     # via
     #   conan
     #   requests
diff --git a/requirements/ci.txt b/requirements/ci.txt
@@ -17,7 +17,7 @@ colorama==0.4.6
     # via tox
 distlib==0.3.8
     # via virtualenv
-filelock==3.14.0
+filelock==3.15.1
     # via
     #   tox
     #   virtualenv
@@ -50,7 +50,7 @@ tomli==2.0.1
     #   tox
 tox==4.15.1
     # via -r ci.in
-urllib3==2.2.1
+urllib3==2.2.2
     # via requests
 virtualenv==20.26.2
     # via tox
diff --git a/requirements/docs.txt b/requirements/docs.txt
@@ -38,15 +38,15 @@ docutils==0.21.2
     #   nbsphinx
     #   pydata-sphinx-theme
     #   sphinx
-fastjsonschema==2.19.1
+fastjsonschema==2.20.0
     # via nbformat
 fonttools==4.53.0
     # via matplotlib
 gitdb==4.0.11
     # via gitpython
 gitpython==3.1.43
     # via -r docs.in
-hypothesis==6.103.1
+hypothesis==6.103.2
     # via -r docs.in
 imagesize==1.4.1
     # via sphinx
@@ -115,7 +115,7 @@ pillow==10.3.0
     # via
     #   ipympl
     #   matplotlib
-plopp==24.5.0
+plopp==24.6.0
     # via -r docs.in
 pydata-sphinx-theme==0.15.3
     # via -r docs.in

diff --git a/requirements/extra.txt b/requirements/extra.txt
@@ -6,8 +6,6 @@
 #    pip-compile-multi
 #
 -r base.txt
-appnope==0.1.4
-    # via ipykernel
 asttokens==2.4.1
     # via stack-data
 certifi==2024.6.2
@@ -42,15 +40,15 @@ jupyter-core==5.7.2
     # via
     #   ipykernel
     #   jupyter-client
-llvmlite==0.42.0
+llvmlite==0.43.0
     # via numba
 matplotlib-inline==0.1.7
     # via
     #   ipykernel
     #   ipython
 nest-asyncio==1.6.0
     # via ipykernel
-numba==0.59.1
+numba==0.60.0
     # via -r extra.in
 packaging==24.1
     # via
@@ -73,7 +71,7 @@ pooch==1.8.2
     # via -r extra.in
 prompt-toolkit==3.0.47
     # via ipython
-psutil==5.9.8
+psutil==6.0.0
     # via ipykernel
 ptyprocess==0.7.0
     # via pexpect
@@ -117,9 +115,9 @@ typing-extensions==4.12.2
     # via ipython
 tzdata==2024.1
     # via pandas
-urllib3==2.2.1
+urllib3==2.2.2
     # via requests
 wcwidth==0.2.13
     # via prompt-toolkit
-xarray==2024.5.0
+xarray==2024.6.0
     # via -r extra.in
diff --git a/requirements/static.txt b/requirements/static.txt
@@ -9,7 +9,7 @@ cfgv==3.4.0
     # via pre-commit
 distlib==0.3.8
     # via virtualenv
-filelock==3.14.0
+filelock==3.15.1
     # via virtualenv
 identify==2.5.36
     # via pre-commit

diff --git a/requirements/test.txt b/requirements/test.txt
@@ -17,7 +17,7 @@ bs4==0.0.2
     # via -r test.in
 execnet==2.1.1
     # via pytest-xdist
-hypothesis==6.103.1
+hypothesis==6.103.2
     # via -r test.in
 iniconfig==2.0.0
     # via pytest

diff --git a/src/scipp/io/hdf5.py b/src/scipp/io/hdf5.py
@@ -431,7 +431,13 @@ def read(group: h5.Group):
 
 
 def _direct_io(cls, convert=None):
-    type_name = cls.__name__
+    # Use fully qualified name for numpy.bool to avoid confusion with builtin bool."
+    # Numpy bools have different names in 1.x and 2.x
+    # TODO: This should be fixed once we drop support for numpy<2
+    if cls.__module__ == 'numpy' and cls.__name__[:4] == 'bool':
+        type_name = 'numpy.bool'
+    else:
+        type_name = cls.__name__
     if convert is None:
         convert = cls
 
@@ -458,6 +464,7 @@ class _HDF5IO:
         'DataGroup': _DataGroupIO,
         'str': _direct_io(str, convert=lambda b: b.decode('utf-8')),
         'ndarray': _direct_io(np.ndarray, convert=lambda x: x),
+        'numpy.bool': _direct_io(np.bool_),
         **{
             cls.__name__: _direct_io(cls)
             for cls in (
@@ -470,15 +477,20 @@ class _HDF5IO:
                 np.float32,
                 np.float64,
                 bool,
-                np.bool_,
                 bytes,
             )
         },
     }
 
     @classmethod
     def write(cls, group: h5.Group, data, **kwargs):
-        name = data.__class__.__name__.replace('View', '')
+        data_cls = data.__class__
+        # Numpy bools have different names in 1.x and 2.x
+        # TODO: This should be fixed once we drop support for numpy<2
+        if data_cls.__module__ == 'numpy' and data_cls.__name__[:4] == 'bool':
+            name = 'numpy.bool'
+        else:
+            name = data_cls.__name__.replace('View', '')
         try:
             handler = cls._handlers[name]
         except KeyError:

diff --git a/tests/binning_test.py b/tests/binning_test.py
@@ -865,7 +865,7 @@ def test_bin_with_explicit_lower_precision_drops_rows_outside_domain():
     x = sc.linspace('x', 0.0, 1.0, 3, unit='m', dtype='float32')
     da = table.bin(x=x)
     size = da.bins.size().sum().value
-    table.coords['x'].values[0] = 2.0 * np.finfo(np.float32).max
+    table.coords['x'].values[0] = np.float64(2.0) * np.finfo(np.float32).max
     da = table.bin(x=x)
     reference = table[1:].bin(x=x)
     assert sc.identical(da, reference)

diff --git a/tests/dtype_test.py b/tests/dtype_test.py
@@ -46,7 +46,7 @@ def test_numpy_comparison():
 
 
 def check_numpy_version_for_comaprison():
-    major, minor, patch = np.__version__.split('.')
+    major, minor, *_ = np.__version__.split('.')
     if int(major) == 1 and int(minor) < 21:
         return True
     return False

diff --git a/tests/lookup_test.py b/tests/lookup_test.py
@@ -100,7 +100,7 @@ def test_nearest_masked_points_replaced_by_fill_value(dtype):
 
 def outofbounds(dtype):
     if dtype in ['float32', 'float64']:
-        return np.NaN
+        return np.nan
     return 0
 
 

diff --git a/tests/reduce_test.py b/tests/reduce_test.py
@@ -35,7 +35,7 @@ def test_reduce_tuple():
 
 
 def test_reduce_nan():
-    var.values[1, 1] = np.NAN
+    var.values[1, 1] = np.nan
     args = _slices(var)
     assert sc.identical(sc.reduce(args).nanmax(), sc.nanmax(var, dim))
     assert sc.identical(sc.reduce(args).nanmin(), sc.nanmin(var, dim))

diff --git a/tests/variable_creation_test.py b/tests/variable_creation_test.py
@@ -6,6 +6,12 @@
 
 import scipp as sc
 
+# https://numpy.org/devdocs/release/2.0.0-notes.html#remove-datetime64-deprecation-warning-when-constructing-with-timezone
+# TODO: This should be removed once we drop support for numpy<2
+raise_np_datetime = (
+    ValueError if np.lib.NumpyVersion(np.__version__) < '2.0.0b1' else UserWarning
+)
+
 
 def _compare_properties(a, b):
     assert a.dims == b.dims
@@ -671,7 +677,7 @@ def test_arange_datetime_from_str_raises_if_step_has_no_unit():
 
 
 def test_arange_datetime_from_str_raises_given_string_with_timezone():
-    with pytest.raises(ValueError, match='timezone'):
+    with pytest.raises(raise_np_datetime, match='timezone'):
         sc.arange(
             't',
             '2022-08-02T06:42:45Z',
@@ -961,13 +967,13 @@ def test_empty_sizes():
 
 @pytest.mark.parametrize('timezone', ['Z', '-05:00', '+02'])
 def test_datetime_raises_given_string_with_timezone(timezone):
-    with pytest.raises(ValueError, match='timezone'):
+    with pytest.raises(raise_np_datetime, match='timezone'):
         sc.datetime(f'2152-11-25T13:13:46{timezone}')
 
 
 @pytest.mark.parametrize('timezone', ['Z', '-05:00', '+02'])
 def test_datetimes_raises_given_string_with_timezone(timezone):
-    with pytest.raises(ValueError, match='timezone'):
+    with pytest.raises(raise_np_datetime, match='timezone'):
         sc.datetimes(dims=['time'], values=[f'2152-11-25T13:13:46{timezone}'], unit='s')
 
 

diff --git a/tests/variable_init_test.py b/tests/variable_init_test.py
@@ -12,7 +12,10 @@
 
 
 def representation_of_native_int():
-    if platform.system() == 'Windows':
+    if (
+        platform.system() == 'Windows'
+        and np.lib.NumpyVersion(np.__version__) < '2.0.0b1'
+    ):
         return sc.DType.int32
     return sc.DType.int64
 

diff --git a/tox.ini b/tox.ini
@@ -121,3 +121,15 @@ description = Generate a stub file for the C++ classes
 skip_install = true
 changedir = tools
 commands = python -m stubgen
+
+[testenv:nightlydeps]
+description = Test using nightly versions of dependencies
+deps = -r requirements/test.in
+pip_pre = true
+setenv =
+  PYTHONPATH = {toxinidir}{/}install
+  JUPYTER_PLATFORM_DIRS = 1
+  PYTHONWARNDEFAULTENCODING = 1
+  PIP_INDEX_URL = {env:PIP_INDEX_URL:https://pypi.anaconda.org/scientific-python-nightly-wheels/simple}
+  PIP_EXTRA_INDEX_URL = {env:PIP_EXTRA_INDEX_URL:https://pypi.org/simple}
+commands = python -m pytest -n auto -v {posargs:tests}