Remove automatic recursive to_dict/from_dict again

.github/workflows/atomistic-notebooks-compat.yml

@@ -0,0 +1,44 @@
+# This workflow is used to check the compatibility with the pyiron_atomistics
+
+name: Compatibility with pyiron_atomistics Notebooks
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    types: [labeled, opened, synchronize, reopened]
+
+jobs:
+  build:
+    if: |
+      github.event_name == 'push' ||
+      ( github.event_name == 'pull_request'  && contains(github.event.pull_request.labels.*.name, 'integration' ))
+
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Merge environment
+      shell: bash -l {0}
+      run: |
+        git clone https://github.com/pyiron/pyiron_atomistics ../pyiron_atomistics
+        cp ../pyiron_atomistics/.ci_support/environment.yml environment.yml
+        tail --lines=+4 ../pyiron_atomistics/.ci_support/environment-notebooks.yml >> environment.yml
+        echo -e "channels:\n  - conda-forge\n" > .condarc
+    - name: Setup Mambaforge
+      uses: conda-incubator/setup-miniconda@v3
+      with:
+        python-version: "3.12"
+        miniforge-version: latest
+        condarc-file: .condarc
+        environment-file: environment.yml
+    - name: Tests
+      shell: bash -l {0}
+      timeout-minutes: 30
+      run: |
+        pip install versioneer[toml]==0.29
+        cd ../pyiron_atomistics
+        pip install . --no-deps --no-build-isolation
+        cd ../pyiron_base
+        pip install . --no-deps --no-build-isolation
+        cd ../pyiron_atomistics
+        ./.ci_support/build_notebooks.sh

.github/workflows/atomistics-compat.yml

@@ -30,26 +30,26 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - name: Merge environment
+      shell: bash -l {0}
       run: |
         git clone https://github.com/pyiron/pyiron_atomistics ../pyiron_atomistics
-        grep -v "pyiron_base" ../pyiron_atomistics/.ci_support/environment.yml > ../pyiron_atomistics/environment.yml
-        cp .ci_support/environment.yml environment.yml
-        tail --lines=+4 ../pyiron_atomistics/environment.yml >> environment.yml
         echo -e "channels:\n  - conda-forge\n" > .condarc
     - name: Setup Mambaforge
       uses: conda-incubator/setup-miniconda@v3
       with:
         python-version: ${{ matrix.python-version }}
         miniforge-version: latest
         condarc-file: .condarc
-        environment-file: environment.yml
+        environment-file: ../pyiron_atomistics/.ci_support/environment.yml
     - name: Tests
       shell: bash -l {0}
       timeout-minutes: 30
       run: |
         pip install versioneer[toml]==0.29
-        pip install . --no-deps --no-build-isolation
         cd ../pyiron_atomistics
         pip install . --no-deps --no-build-isolation
+        cd ../pyiron_base
+        pip install . --no-deps --no-build-isolation
+        cd ../pyiron_atomistics
         python .ci_support/pyironconfig.py
         python -m unittest discover tests/

.github/workflows/contrib-compat.yml

@@ -17,19 +17,17 @@ jobs:
     steps:
     - uses: actions/checkout@v4
     - name: Merge environment
+      shell: bash -l {0}
       run: |
         git clone https://github.com/pyiron/pyiron_contrib ../pyiron_contrib
-        grep -v "pyiron_base" ../pyiron_contrib/.ci_support/environment.yml > ../pyiron_contrib/environment.yml
-        cp .ci_support/environment.yml environment.yml
-        tail --lines=+4 ../pyiron_contrib/environment.yml >> environment.yml
         echo -e "channels:\n  - conda-forge\n" > .condarc
     - name: Setup Mambaforge
       uses: conda-incubator/setup-miniconda@v3
       with:
         python-version: '3.11'
         miniforge-version: latest
         condarc-file: .condarc
-        environment-file: environment.yml
+        environment-file: ../pyiron_contrib/.ci_support/environment.yml
     - name: Test
       shell: bash -l {0}
       timeout-minutes: 30

pyiron_base/interfaces/has_dict.py

@@ -61,6 +61,95 @@ def create_from_dict(obj_dict):
     return obj
 
 
+def _join_children_dict(children: dict[str, dict[str, Any]]) -> dict[str, Any]:
+    """
+    Given a nested dictionary, flatten the first level.
+
+    >>> d = {'a': {'a1': 3}, 'b': {'b1': 4, 'b2': {'c': 42}}}
+    >>> _join_children_dict(d)
+    {'a/a1': 3, 'b/b1': 4, 'b/b2': {'c': 42}}
+
+    This is intended as a utility function for nested HasDict objects, that
+    to_dict their children and then want to give a flattened dict for
+    writing to ProjectHDFio.write_dict_to_hdf.
+
+    See also :func:`._split_children_dict`.
+    """
+    return {
+        "/".join((k1, k2)): v2 for k1, v1 in children.items() for k2, v2 in v1.items()
+    }
+
+
+def _split_children_dict(obj_dict: dict[str, Any]) -> dict[str, Any | dict[str, Any]]:
+    """
+    Undoes _join_children_dict.
+
+    Classes that use :func:`._join_children_dict` in their `_to_dict`, must
+    call this function in their `_from_dict`.
+    """
+    subs = defaultdict(dict)
+    plain = {}
+    for k, v in obj_dict.items():
+        if "/" not in k:
+            plain[k] = v
+            continue
+        root, k = k.split("/", maxsplit=1)
+        subs[root][k] = v
+    # using update keeps type stability, i.e. we always return a plain dict
+    plain.update(subs)
+    return plain
+
+
+def _from_dict_children(obj_dict: dict) -> dict:
+    """
+    Recurse through `obj_dict` and restore any objects with :class:`~.HasDict`.
+
+    Args:
+        obj_dict (dict): data previously returned from :meth:`.to_dict`
+    """
+
+    def load(inner_dict):
+        # object is a not a dict, so nothing to do
+        if not isinstance(inner_dict, dict):
+            return inner_dict
+        # if object is a dict but doesn't have type information, recurse through it to load any sub dicts that might
+        if not all(k in inner_dict for k in ("NAME", "TYPE", "OBJECT", "DICT_VERSION")):
+            return {k: load(v) for k, v in inner_dict.items()}
+        # object has type info, so just load it
+        return create_from_dict(inner_dict)
+
+    return {k: load(v) for k, v in obj_dict.items()}
+
+
+def _to_dict_children(obj_dict: dict) -> dict:
+    """
+    Call to_dict on any objects in the values that support it.
+
+    Intended as a helper function for recursives object that want to to_dict
+    their nested objects automatically.  It uses :func:`._join_children_dict`
+    for any dictionaries returned from the children.
+
+    The function only goes through the *first* layer of dictionary values and
+    does *not* recurse through nested dictionaries.
+
+    Args:
+        obj_dict (dict): data previously returned from :meth:`._to_dict`
+
+    Returns:
+        obj_dict (dict): new dictionary with the obj_dict of the children
+    """
+    data_dict = {}
+    child_dict = {}
+    for k, v in obj_dict.items():
+        if isinstance(v, HasDict):
+            child_dict[k] = v.to_dict()
+        elif isinstance(v, HasHDF):
+            child_dict[k] = HasDictfromHDF.to_dict(v)
+        else:
+            data_dict[k] = v
+    return data_dict | _join_children_dict(child_dict)
+
+
 class HasDict(ABC):
     """
     Abstract interface to convert objects to dictionaries for storage.
@@ -114,61 +203,43 @@ def from_dict(self, obj_dict: dict, version: str = None):
             version (str): version tag written together with the data
         """
 
-        def load(inner_dict):
-            if not isinstance(inner_dict, dict):
-                return inner_dict
-            if not all(
-                k in inner_dict for k in ("NAME", "TYPE", "OBJECT", "DICT_VERSION")
-            ):
-                return {k: load(v) for k, v in inner_dict.items()}
-            return create_from_dict(inner_dict)
-
-        obj_dict = self._split_children_dict(obj_dict)
+        obj_dict = _split_children_dict(obj_dict)
         if version is None:
             version = obj_dict.get("DICT_VERSION", None)
-        self._from_dict({k: load(v) for k, v in obj_dict.items()}, version)
+        self._from_dict(obj_dict, version)
 
     @abstractmethod
     def _from_dict(self, obj_dict: dict, version: str = None):
         """
         Populate the object from the serialized object.
 
-        :meth:`.from_dict` will already recurse through `obj_dict` and deserialize any :class:`.HasDict` data it finds,
-        so implementations do not need to deserialize their children explicitly.
+        Implementations must use :func:`._from_dict_children` if they use
+        `._to_dict_children` in their implementation of :meth:`._to_dict`.
 
         Args:
             obj_dict (dict): data previously returned from :meth:`.to_dict`
             version (str): version tag written together with the data
         """
         pass
 
-    def to_dict(self):
+    def to_dict(self) -> dict:
         """
         Reduce the object to a dictionary.
 
         Returns:
             dict: serialized state of this object
         """
         type_dict = self._type_to_dict()
-        data_dict = {}
-        child_dict = {}
-        for k, v in self._to_dict().items():
-            if isinstance(v, HasDict):
-                child_dict[k] = v.to_dict()
-            elif isinstance(v, HasHDF):
-                child_dict[k] = HasDictfromHDF.to_dict(v)
-            else:
-                data_dict[k] = v
-        return data_dict | self._join_children_dict(child_dict) | type_dict
+        return self._to_dict() | type_dict
 
     @abstractmethod
-    def _to_dict(self):
+    def _to_dict(self) -> dict:
         """
         Reduce the object to a dictionary.
 
-        :meth:`.to_dict` will find any objects *in the first level* of the returned dictionary and reduce them to
-        dictionaries as well, so implementations do not need to explicitly serialize their children.
-        It will also append the type information obtained from :meth:`._type_to_dict`.
+        Implementations may use :func:`._to_dict_children`, if they
+        automatically want to call `to_dict` on any objects possible from their
+        returned dictionary.
 
         Returns:
             dict: serialized state of this object
@@ -196,44 +267,6 @@ def _type_to_dict(self):
             type_dict["VERSION"] = self.__version__
         return type_dict
 
-    @staticmethod
-    def _join_children_dict(children: dict[str, dict[str, Any]]) -> dict[str, Any]:
-        """
-        Given a nested dictionary, flatten the first level.
-
-        >>> d = {'a': {'a1': 3}, 'b': {'b1': 4, 'b2': {'c': 42}}}
-        >>> _join_children_dict(d)
-        {'a/a1': 3, 'b/b1': 4, 'b/b2': {'c': 42}}
-
-        This is intended as a utility function for nested HasDict objects, that
-        to_dict their children and then want to give a flattened dict for
-        writing to ProjectHDFio.write_dict_to_hdf
-        """
-        return {
-            "/".join((k1, k2)): v2
-            for k1, v1 in children.items()
-            for k2, v2 in v1.items()
-        }
-
-    @staticmethod
-    def _split_children_dict(
-        obj_dict: dict[str, Any],
-    ) -> dict[str, Any | dict[str, Any]]:
-        """
-        Undoes _join_children_dict.
-        """
-        subs = defaultdict(dict)
-        plain = {}
-        for k, v in obj_dict.items():
-            if "/" not in k:
-                plain[k] = v
-                continue
-            root, k = k.split("/", maxsplit=1)
-            subs[root][k] = v
-        # using update keeps type stability, i.e. we always return a plain dict
-        plain.update(subs)
-        return plain
-
 
 class HasHDFfromDict(HasHDF, HasDict):
     """

pyiron_base/jobs/job/generic.py

@@ -1172,26 +1172,16 @@ def _to_dict(self):
             data_dict["files_to_compress"] = self._files_to_compress
         if len(self._files_to_remove) > 0:
             data_dict["files_to_compress"] = self._files_to_remove
+        data_dict["HDF_VERSION"] = self.__version__
         return data_dict
 
     def _from_dict(self, obj_dict, version=None):
         self._type_from_dict(type_dict=obj_dict)
         if "import_directory" in obj_dict.keys():
             self._import_directory = obj_dict["import_directory"]
-        # Backwards compatibility: Previously server and executable were stored
-        # as plain dicts, but now they are dicts with additional info so that
-        # HasDict can load them automatically.
-        # We need to check whether that was possible with the instance check
-        # below and if not, call from_dict ourselves.
-        if isinstance(server := obj_dict["server"], Server):
-            self._server = server
-        else:
-            self._server.from_dict(server)
+        self._server.from_dict(obj_dict["server"])
         if "executable" in obj_dict.keys() and obj_dict["executable"] is not None:
-            if isinstance(executable := obj_dict["executable"], Executable):
-                self._executable = executable
-            else:
-                self.executable.from_dict(executable)
+            self.executable.from_dict(obj_dict["executable"])
         input_dict = obj_dict["input"]
         if "generic_dict" in input_dict.keys():
             generic_dict = input_dict["generic_dict"]

pyiron_base/storage/datacontainer.py

@@ -13,7 +13,12 @@
 import numpy as np
 import pandas
 
-from pyiron_base.interfaces.has_dict import HasDict, HasDictfromHDF
+from pyiron_base.interfaces.has_dict import (
+    HasDict,
+    HasDictfromHDF,
+    _from_dict_children,
+    _to_dict_children,
+)
 from pyiron_base.interfaces.has_groups import HasGroups
 from pyiron_base.interfaces.has_hdf import HasHDF
 from pyiron_base.interfaces.lockable import Lockable, sentinel
@@ -1034,9 +1039,10 @@ def _to_dict(self):
         order = list(data)
         data["READ_ONLY"] = self.read_only
         data["KEY_ORDER"] = order
-        return data
+        return _to_dict_children(data)
 
     def _from_dict(self, obj_dict, version=None):
+        obj_dict = _from_dict_children(obj_dict)
         if version == "0.2.0":
             order = obj_dict.pop("KEY_ORDER")
         else: