diff --git a/pyiron_base/jobs/job/core.py b/pyiron_base/jobs/job/core.py index bd0bffeef..606f591b3 100644 --- a/pyiron_base/jobs/job/core.py +++ b/pyiron_base/jobs/job/core.py @@ -209,6 +209,8 @@ def __init__(self, project, job_name): self._import_directory = None self._database_property = DatabaseProperties() self._hdf5_content = HDF5Content(project_hdf5=self._hdf5) + self._files_to_remove = list() + self._files_to_compress = list() @property def content(self): @@ -385,6 +387,14 @@ def project_hdf5(self, project): """ self._hdf5 = project.copy() + @property + def files_to_compress(self): + return self._files_to_compress + + @property + def files_to_remove(self): + return self._files_to_remove + def relocate_hdf5(self, h5_path=None): """ Relocate the hdf file. This function is needed when the child job is @@ -1061,14 +1071,22 @@ def _list_ext_childs(self): childs = self.list_childs() return list(set(childs) - set(nodes)) - def compress(self, files_to_compress=None): + def compress(self, files_to_compress=None, files_to_remove=None): """ Compress the output files of a job object. Args: files_to_compress (list): """ - _job_compress(job=self, files_to_compress=files_to_compress) + if files_to_compress is None and len(self._files_to_compress) != 0: + files_to_compress = self._files_to_compress + elif files_to_compress is None: + files_to_compress = self.files.list() + if files_to_remove is None: + files_to_remove = self._files_to_remove + else: + files_to_remove = [] + _job_compress(job=self, files_to_compress=files_to_compress, files_to_remove=files_to_remove) def decompress(self): """ diff --git a/pyiron_base/jobs/job/generic.py b/pyiron_base/jobs/job/generic.py index c74fb1123..4bf6f2d7c 100644 --- a/pyiron_base/jobs/job/generic.py +++ b/pyiron_base/jobs/job/generic.py @@ -1069,6 +1069,10 @@ def to_dict(self): data_dict["import_directory"] = self._import_directory if self._executor_type is not None: data_dict["executor_type"] = self._executor_type + if len(self._files_to_compress) > 0: + data_dict["files_to_compress"] = self._files_to_compress + if len(self._files_to_remove) > 0: + data_dict["files_to_compress"] = self._files_to_remove return data_dict def from_dict(self, job_dict): diff --git a/pyiron_base/jobs/job/util.py b/pyiron_base/jobs/job/util.py index e5629c5c8..b0063e712 100644 --- a/pyiron_base/jobs/job/util.py +++ b/pyiron_base/jobs/job/util.py @@ -275,17 +275,25 @@ def _get_compressed_job_name(working_directory): ) -def _job_compress(job, files_to_compress=None): +def _job_compress(job, files_to_compress=[], files_to_remove=[]): """ Compress the output files of a job object. Args: job (JobCore): job object to compress files_to_compress (list): list of files to compress + files_to_remove (list): list of files to remove """ + + def delete_file_or_folder(fullname): + if os.path.isfile(fullname): + os.remove(fullname) + elif os.path.isdir(fullname): + shutil.rmtree(fullname) + if not _job_is_compressed(job): - if files_to_compress is None: - files_to_compress = job.files.list() + for name in files_to_remove: + delete_file_or_folder(fullname=os.path.join(job.working_directory, name)) cwd = os.getcwd() try: os.chdir(job.working_directory) @@ -294,12 +302,10 @@ def _job_compress(job, files_to_compress=None): if "tar" not in name and not stat.S_ISFIFO(os.stat(name).st_mode): tar.add(name) for name in files_to_compress: - if "tar" not in name: - fullname = os.path.join(job.working_directory, name) - if os.path.isfile(fullname): - os.remove(fullname) - elif os.path.isdir(fullname): - shutil.rmtree(fullname) + if name != _job_compressed_name(job): + delete_file_or_folder( + fullname=os.path.join(job.working_directory, name) + ) finally: os.chdir(cwd) else: