Merge pull request #110 from ecmwf-ifs/102-install-and-doc

Migrate to pyproject.toml package installation

CMakeLists.txt

@@ -153,6 +153,8 @@ if( NOT HAVE_NO_INSTALL )
         # Less verbose output
         list ( APPEND PIP_OPTIONS "-q" )
     endif()
+    execute_process( COMMAND ${Python3_EXECUTABLE} -m pip install --upgrade ${PIP_OPTIONS} pip )
+
     if( HAVE_EDITABLE )
         # Use checked-out source instead of installing into venv
         list( APPEND PIP_OPTIONS "-e" )
@@ -166,6 +168,7 @@ if( NOT HAVE_NO_INSTALL )
 
     # We install Loki at configure time (for now), since bulk-transformation planning
     # requires configure time execution to allow injection with CMake targets.
+
     message( STATUS "Install Loki in virtual environment" )
     execute_process( COMMAND ${Python3_EXECUTABLE} -m pip install ${PIP_OPTIONS} ${CMAKE_CURRENT_SOURCE_DIR}${_INSTALL_OPTIONS} )
     message( STATUS "Install Loki in virtual environment - done" )

INSTALL.md

@@ -206,8 +206,12 @@ git clone https://github.com/ecmwf-ifs/loki
 ```bash
 python3 -m venv loki_env
 source loki_env/bin/activate
+pip install --upgrade pip
 ```
 
+Note that we need to make sure that we use a recent pip version (21.3 or newer)
+that has support for editable installs using `pyproject.toml`.
+
 ### 3.  Install Loki and Python dependencies
 
 ```bash
@@ -218,7 +222,7 @@ pip install -e ./lint_rules
 popd
 ```
 
-### 4.  Install CLAW with OMNI compiler
+### 4.  Install CLAW with OMNI compiler -- optional
 
 ```bash
 git clone --recursive --single-branch --branch=mlange-dev https://github.com/mlange05/claw-compiler.git claw-compiler
@@ -233,7 +237,7 @@ popd
 popd
 ```
 
-### 5.  Install OpenFortranParser (OFP)
+### 5.  Install OpenFortranParser (OFP) -- optional
 
 ```bash
 pip install -e git+https://github.com/mlange05/open-fortran-parser-xml@mlange05-dev#egg=open-fortran-parser
@@ -252,6 +256,8 @@ popd
 
 ```bash
 pushd loki
-py.test
+py.test transformations lint_rules .
 popd
 ```
+
+Note that the order is important to avoid clashes with conftest utilities.

install

@@ -13,8 +13,8 @@ set -euo pipefail
 loki_ant_version=1.10.13
 
 hpc2020_java_version=11.0.6
-hpc2020_python_version=3.8.8-01
-hpc2020_cmake_version=3.20.2
+hpc2020_python_version=3.10.10-01
+hpc2020_cmake_version=3.25.2
 
 # Determine base path for loki
 # Either take the root of the current git tree or, if not inside a git repository, then
@@ -284,6 +284,7 @@ else
   pip_opts=[${pip_opts:1}]
 fi
 
+pip install $PIPPROXYOPTIONS --upgrade pip
 pip install $PIPPROXYOPTIONS -e .$pip_opts  # Installs Loki dev copy in editable mode
 pip install $PIPPROXYOPTIONS -e ./transformations
 pip install $PIPPROXYOPTIONS -e ./lint_rules
@@ -507,5 +508,5 @@ echo "Activate the Loki environment with"
 echo
 echo "    source loki-activate"
 echo
-echo "You can test the installation then by calling 'py.test'"
+echo "You can test the Loki installation by running 'py.test transformations lint_rules .'"
 echo

lint_rules/lint_rules/__init__.py

@@ -5,4 +5,10 @@
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 
-__version__ = "0.0.0"
+from importlib.metadata import version, PackageNotFoundError
+
+try:
+    __version__ = version("lint_rules")
+except PackageNotFoundError:
+    # package is not installed
+    pass

lint_rules/pyproject.toml

@@ -1,5 +1,31 @@
 [build-system]
 requires = [
-  "setuptools >= 45",
+  "setuptools >= 61",
+  "setuptools_scm[toml] >= 6.2",
 ]
 build-backend = "setuptools.build_meta"
+
+[project]
+name = "lint_rules"
+authors = [
+  {name = "ECMWF", email = "user_support_section@ecmwf.int"},
+]
+description = "Linter rule implementations for loki-lint"
+requires-python = ">=3.8"
+license = {text = "Apache-2.0"}
+dynamic = ["version"]
+dependencies = ["loki"]
+
+[tool.setuptools]
+license-files = ["../LICENSE", "../AUTHORS.md"]
+packages = ["lint_rules"]
+
+# Enable SCM versioning
+[tool.setuptools_scm]
+root = ".."
+relative_to = "__file__"
+
+[tool.pytest.ini_options]
+testpaths = [
+  "tests"
+]

loki/__init__.py

@@ -5,7 +5,7 @@
 # granted to it by virtue of its status as an intergovernmental organisation
 # nor does it submit to any jurisdiction.
 
-from pkg_resources import get_distribution, DistributionNotFound
+from importlib.metadata import version, PackageNotFoundError
 
 # Import the global configuration map
 from loki.config import *  # noqa
@@ -31,9 +31,10 @@
 from loki.analyse import *  # noqa
 from loki.dimension import *  # noqa
 
+
 try:
-    __version__ = get_distribution(__name__).version
-except DistributionNotFound:
+    __version__ = version("loki")
+except PackageNotFoundError:
     # package is not installed
     pass
 

loki/frontend/regex.py

@@ -881,7 +881,7 @@ def match(self, reader, parser_classes, scope):
             symbols = None
 
         return ir.Import(
-            module, symbols=as_tuple(symbols), rename_list=rename_list,
+            module, symbols=as_tuple(symbols), rename_list=as_tuple(rename_list),
             source=reader.source_from_current_line()
         )
 

pyproject.toml

@@ -1,12 +1,90 @@
 # Make sure we use setuptools and have all required dependencies for that
 [build-system]
 requires = [
-  "setuptools >= 45",
+  "setuptools >= 61",
   "wheel",
-  "setuptools_scm >= 6.2",
-  "numpy"
+  "setuptools_scm[toml] >= 6.2",
+  "numpy<1.24"
 ]
 build-backend = "setuptools.build_meta"
 
+[project]
+name = "loki"
+authors = [
+  {name = "ECMWF", email = "user_support_section@ecmwf.int"},
+]
+description = "Experimental Fortran IR to facilitate source-to-source transformations"
+requires-python = ">=3.8,<3.12"
+license = {text = "Apache-2.0"}
+dynamic = ["version", "readme"]
+dependencies = [
+  "numpy<1.24",  # essential for tests, loop transformations and other dependencies
+  "pymbolic>=2022.1",  # essential for expression tree
+  "PyYAML",  # essential for loki-lint
+  "pcpp",  # essential for preprocessing
+  "more-itertools",  # essential for SCC transformation
+  "click",  # essential for CLI scripts
+  "toml",  # essential for scheduler configuration
+  "networkx",  # essential for scheduler and build utilities
+  "fparser>=0.0.15",  # (almost) essential as frontend
+  "graphviz",  # optional for scheduler callgraph
+  "tqdm",  # optional for build utilities
+  "coloredlogs",  # optional for loki-build utility
+  "junit_xml",  # optional for JunitXML output in loki-lint
+  "codetiming",  # essential for scheduler and sourcefile timings
+  "pydantic<2.0",  # type checking for IR nodes
+]
+
+[project.optional-dependencies]
+tests = [
+  "pytest",
+  "pytest-cov",
+  "coverage2clover",
+  "pylint!=2.11.0,!=2.11.1",
+  "pandas",
+  "f90wrap>=0.2.3",
+  "nbconvert",
+]
+ofp = [
+  "open-fortran-parser @ git+https://github.com/mlange05/open-fortran-parser-xml@mlange05-dev#egg=open-fortran-parser",
+]
+dace = [
+  "dace>=0.11.2",
+]
+docs = [
+  "sphinx",  # to build documentation
+  "recommonmark",  # to allow parsing markdown
+  "sphinx-rtd-theme",  # ReadTheDocs theme
+  "myst-parser",  # Markdown parser for sphinx
+  "nbsphinx",  # Jupyter notebook parser for sphinx
+  "sphinx-design",  # Add panels, cards and dropdowns for sphinx
+]
+examples = [
+  "jupyter",
+  "ipyparams",
+]
+
+[project.scripts]
+"loki-transform.py" = "scripts.loki_transform:cli"
+"loki-lint.py" = "scripts.loki_lint:cli"
+
+[tool.setuptools]
+license-files = ["LICENSE", "AUTHORS.md"]
+packages = ["loki", "scripts"]
+
+[tool.setuptools.dynamic]
+# version = {attr = "loki.__version__"}
+readme = {file = ["README.md", "INSTALL.md"], content-type = "test/markdown"}
+
+# [tool.setuptools.packages.find]
+# where = ["."]
+# include = ["loki", "scripts"]
+# exclude = ["tests", "transformations", "lint_rules"]
+
 # Enable SCM versioning
 [tool.setuptools_scm]
+
+[tool.pytest.ini_options]
+testpaths = [
+  "tests"
+]