Merge pull request #119 from ecmwf-ifs/naan-ecwam-regression-tests

Add ecWam regression tests

.github/workflows/regression_tests.yml

@@ -33,6 +33,13 @@ jobs:
           path: cloudsc
           ref: develop
 
+      - name: Clone ECWAM
+        uses: actions/checkout@v3
+        with:
+          repository: ecmwf-ifs/ecwam
+          path: ecwam
+          ref: naan-phys-gpu
+
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
@@ -64,11 +71,25 @@ jobs:
           source loki-activate
           pip install cmake
 
-      - name: Run CLOUDSC regression tests
+      - name: Install ECWAM dependencies
+        run: |
+          sudo apt-get -o Acquire::Retries=3 install -y libopenmpi-dev
+          source loki-activate
+          pip install fypp
+
+      - name: Run CLOUDSC and ECWAM regression tests
         env:
           CLOUDSC_DIR: ${{ github.workspace }}/cloudsc
           CLOUDSC_ARCH: ${{ github.workspace }}/cloudsc/arch/github/ubuntu/gnu/9.4.0
+          ECWAM_DIR: ${{ github.workspace }}/ecwam
+          ECWAM_ARCH: ${{ github.workspace }}/ecwam/arch/github/ubuntu/gnu/9.4.0
           OMP_STACKSIZE: 4G
         run: |
+          mkdir -p ~/.ssh/ && touch ~/.ssh/known_hosts
+          cat << 'EOF' > ~/.ssh/known_hosts
+          ${{ secrets.BITBUCKET_HOSTKEY }}
+          EOF
+          eval `ssh-agent -s`
+          ssh-add - <<< '${{ secrets.ECWAM_REPOSITORY_SSH_KEY }}'
           source loki-activate
-          pytest --cov=transformations/transformations transformations/tests -k cloudsc
+          pytest --cov=transformations/transformations transformations/tests -k 'cloudsc or ecwam'

transformations/tests/conftest.py

@@ -5,11 +5,14 @@
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 
+from pathlib import Path
+import shutil
 import pytest
 
 from loki import as_tuple, Frontend
 import loki.frontend
 
+__all__ = ['available_frontends', 'local_loki_setup', 'local_loki_cleanup']
 
 def available_frontends(xfail=None, skip=None):
     """
@@ -68,3 +71,39 @@ def my_test(frontend):
             params += [f]
 
     return params
+
+
+def write_env_launch_script(here, binary, args):
+    # Write a script to source env.sh and launch the binary
+    script = Path(here/f'build/run_{binary}.sh')
+    script.write_text(f"""
+#!/bin/bash
+
+source env.sh >&2
+bin/{binary} {' '.join(args)}
+exit $?
+    """.strip())
+    script.chmod(0o750)
+
+    return script
+
+
+def local_loki_setup(here):
+    lokidir = Path(__file__).parent.parent.parent
+    target = here/'source/loki'
+    backup = here/'source/loki.bak'
+
+    # Do not overwrite any existing Loki copy
+    if target.exists():
+        if backup.exists():
+            shutil.rmtree(backup)
+        shutil.move(target, backup)
+
+    return str(lokidir.resolve()), target, backup
+
+
+def local_loki_cleanup(target, backup):
+    if target.is_symlink():
+        target.unlink()
+    if not target.exists() and backup.exists():
+        shutil.move(backup, target)

transformations/tests/test_cloudsc.py

@@ -10,12 +10,12 @@
 import resource
 from subprocess import CalledProcessError
 from pathlib import Path
-import shutil
 import pandas as pd
 import pytest
-import yaml
 
-from conftest import available_frontends
+from conftest import (
+    available_frontends, write_env_launch_script, local_loki_setup, local_loki_cleanup
+)
 from loki import execute, OMNI, HAVE_FP, HAVE_OMNI, warning
 
 pytestmark = pytest.mark.skipif('CLOUDSC_DIR' not in os.environ, reason='CLOUDSC_DIR not set')
@@ -28,45 +28,21 @@ def fixture_here():
 
 @pytest.fixture(scope='module', name='local_loki_bundle')
 def fixture_local_loki_bundle(here):
-    """Inject ourselves into the CLOUDSC bundle"""
-    lokidir = Path(__file__).parent.parent.parent
-    target = here/'source/loki'
-    backup = here/'source/loki.bak'
-    bundlefile = here/'bundle.yml'
-    local_loki_bundlefile = here/'__bundle_loki.yml'
-
-    # Do not overwrite any existing Loki copy
-    if target.exists():
-        if backup.exists():
-            shutil.rmtree(backup)
-        shutil.move(target, backup)
-
-    # Change bundle to symlink for Loki
-    bundle = yaml.safe_load(bundlefile.read_text())
-    loki_index = [i for i, p in enumerate(bundle['projects']) if 'loki' in p]
-    assert len(loki_index) == 1
-    if 'git' in bundle['projects'][loki_index[0]]['loki']:
-        del bundle['projects'][loki_index[0]]['loki']['git']
-    bundle['projects'][loki_index[0]]['loki']['dir'] = str(lokidir.resolve())
-    local_loki_bundlefile.write_text(yaml.dump(bundle))
-
-    yield local_loki_bundlefile
-
-    if local_loki_bundlefile.exists():
-        local_loki_bundlefile.unlink()
-    if target.is_symlink():
-        target.unlink()
-    if not target.exists() and backup.exists():
-        shutil.move(backup, target)
+    """Call setup utilities for injecting ourselves into the CLOUDSC bundle"""
+    lokidir, target, backup = local_loki_setup(here)
+    yield lokidir
+    local_loki_cleanup(target, backup)
 
 
 @pytest.fixture(scope='module', name='bundle_create')
 def fixture_bundle_create(here, local_loki_bundle):
+    """Inject ourselves into the CLOUDSC bundle"""
+    env = os.environ.copy()
+    env['CLOUDSC_BUNDLE_LOKI_DIR'] = local_loki_bundle
+
     # Run ecbundle to fetch dependencies
     execute(
-        ['./cloudsc-bundle', 'create', '--bundle', str(local_loki_bundle)],
-        cwd=here,
-        silent=False
+        ['./cloudsc-bundle', 'create'], cwd=here, silent=False, env=env
     )
 
 
@@ -122,15 +98,7 @@ def test_cloudsc(here, frontend):
     failures, warnings = {}, {}
     for binary, *args in binaries:
         # Write a script to source env.sh and launch the binary
-        script = Path(here/f'build/run_{binary}.sh')
-        script.write_text(f"""
-#!/bin/bash
-
-source env.sh >&2
-bin/{binary} {' '.join(args)}
-exit $?
-        """.strip())
-        script.chmod(0o750)
+        script = write_env_launch_script(here, binary, args)
 
         # Run the script and verify error norms
         try:

transformations/tests/test_ecwam.py

@@ -0,0 +1,119 @@
+# (C) Copyright 2018- ECMWF.
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+import os
+import resource
+from subprocess import CalledProcessError
+from pathlib import Path
+import pytest
+
+from conftest import write_env_launch_script, local_loki_setup, local_loki_cleanup
+from loki import execute, HAVE_FP, FP
+
+pytestmark = pytest.mark.skipif('ECWAM_DIR' not in os.environ, reason='ECWAM_DIR not set')
+
+@pytest.fixture(scope='module', name='here')
+def fixture_here():
+    return Path(os.environ['ECWAM_DIR'])
+
+
+@pytest.fixture(scope='module', name='local_loki_bundle')
+def fixture_local_loki_bundle(here):
+    """Call setup utilities for injecting ourselves into the ECWAM bundle"""
+    lokidir, target, backup = local_loki_setup(here)
+    yield lokidir
+    local_loki_cleanup(target, backup)
+
+
+@pytest.fixture(scope='module', name='bundle_create')
+def fixture_bundle_create(here, local_loki_bundle):
+    """Inject ourselves into the ECWAM bundle"""
+    env = os.environ.copy()
+    env['ECWAM_CONCEPT_BUNDLE_LOKI_DIR'] = local_loki_bundle
+
+    # Run ecbundle to fetch dependencies
+    execute(
+        ['./ecwam-bundle', 'create'],
+        cwd=here,
+        silent=False, env=env
+    )
+
+
+@pytest.mark.usefixtures('bundle_create')
+@pytest.mark.skipif(not HAVE_FP, reason="FP needed for ECWAM parsing")
+def test_ecwam(here, frontend=FP):
+    build_cmd = [
+        './ecwam-bundle', 'build', '--clean',
+        '--with-loki', '--loki-frontend=' + str(frontend), '--without-loki-install'
+    ]
+
+    if 'ECWAM_ARCH' in os.environ:
+        build_cmd += [f"--arch={os.environ['ECWAM_ARCH']}"]
+    else:
+        # Build without OpenACC support as this makes problems
+        # with older versions of GNU
+        build_cmd += ['--cmake=ENABLE_ACC=OFF']
+
+    execute(build_cmd, cwd=here, silent=False)
+
+    # Raise stack limit
+    resource.setrlimit(resource.RLIMIT_STACK, (resource.RLIM_INFINITY, resource.RLIM_INFINITY))
+    env = os.environ.copy()
+    env.update({'OMP_STACKSIZE': '2G', 'NVCOMPILER_ACC_CUDA_HEAPSIZE': '2G'})
+
+    # create rundir
+    os.mkdir(here/'build/wamrun_48')
+
+    # Run pre-processing steps
+    preprocs = [
+        ('ecwam-run-preproc', '--run-dir=wamrun_48', '--config=../source/ecwam/tests/etopo1_oper_an_fc_O48.yml'),
+        ('ecwam-run-preset', '--run-dir=wamrun_48')
+    ]
+
+    failures = {}
+    for preproc, *args in preprocs:
+        script = write_env_launch_script(here, preproc, args)
+
+        # Run the script and verify error norms
+        try:
+            execute([str(script)], cwd=here/'build', silent=False, env=env)
+        except CalledProcessError as err:
+            failures[preproc] = err.returncode
+
+    if failures:
+        msg = '\n'.join([f'Non-zero return code {rcode} in {p}' for p, rcode in failures.items()])
+        pytest.fail(msg)
+
+    # Run the produced binaries
+    binaries = [
+        ('ecwam-run-model', '--run-dir=wamrun_48', '--variant=loki-idem'),
+        ('ecwam-run-model', '--run-dir=wamrun_48', '--variant=loki-idem-stack'),
+        ('ecwam-run-model', '--run-dir=wamrun_48', '--variant=loki-scc'),
+        ('ecwam-run-model', '--run-dir=wamrun_48', '--variant=loki-scc-stack')
+    ]
+
+    failures = {}
+    for binary, *args in binaries:
+        # Write a script to source env.sh and launch the binary
+        script = write_env_launch_script(here, binary, args)
+
+        # Run the script and verify error norms
+        try:
+            execute([str(script)], cwd=here/'build', silent=False, env=env)
+            with open(here/"build/wamrun_48/logs/model/stdout.log") as reader:
+                lines = list(reader)
+
+            if 'Validation FAILED' in lines[-1]:
+                failures[binary] = 'Validation failed'
+            elif not 'Validation PASSED' in lines[-1]:
+                failures[binary] = 'Validation check never run'
+        except CalledProcessError as err:
+            failures[binary] = f'Failed with error code: {err.returncode}'
+
+    if failures:
+        msg = '\n'.join([f'{binary}: {stat}' for binary, stat in failures.items()])
+        pytest.fail(msg)