ShellJob: Fix RemoteData inputs shadowing job's own input files

When a `ShellJob` would get a `RemoteData` node as input that was itself
the `remote_folder` of a previously completed `ShellJob`, the contents
of the `RemoteData` would override the input scripts of the new job
itself.

The reason is because AiiDA's engine would first create the job's input
scripts in the working directory and only then copy over the files of
the `RemoteData`. Since that folder contains many of the same files,
such as the submission script, the inputs of the new job would be
overwritten completely.

AiiDA v2.6 introduced a new feature that allows a `CalcJob` plugin to
define the order in which these files are copied over. This is leveraged
to first copy the `RemoteData` node files and copy the input files of
the job itself last.

src/aiida_shell/calculations/shell.py

@@ -7,7 +7,7 @@
 import shlex
 import typing as t
 
-from aiida.common.datastructures import CalcInfo, CodeInfo
+from aiida.common.datastructures import CalcInfo, CodeInfo, FileCopyOperation
 from aiida.common.folders import Folder
 from aiida.engine import CalcJob, CalcJobProcessSpec
 from aiida.orm import Data, Dict, FolderData, List, RemoteData, SinglefileData, to_aiida_type
@@ -308,6 +308,11 @@ def prepare_for_submission(self, folder: Folder) -> CalcInfo:
         calc_info.remote_symlink_list = remote_symlink_list
         calc_info.retrieve_temporary_list = retrieve_list
         calc_info.provenance_exclude_list = [p.name for p in dirpath.iterdir()]
+        calc_info.file_copy_operation_order = [
+            FileCopyOperation.REMOTE,
+            FileCopyOperation.LOCAL,
+            FileCopyOperation.SANDBOX,
+        ]
 
         return calc_info
 

tests/calculations/test_shell.py

@@ -4,6 +4,7 @@
 
 import pytest
 from aiida.common.datastructures import CodeInfo
+from aiida.engine import run_get_node
 from aiida.orm import Data, Float, FolderData, Int, List, RemoteData, SinglefileData, Str
 from aiida_shell.calculations.shell import ShellJob
 from aiida_shell.data import EntryPointData, PickledData
@@ -430,3 +431,30 @@ def test_input_output_filename_overlap(generate_calc_job, generate_code, tmp_pat
                 'nodes': {'folder': folder_data},
             },
         )
+
+
+def test_remote_folder_copying_order(generate_code, aiida_localhost, tmp_path):
+    """Test that files in ``RemoteData`` input nodes do not overwrite files written by the ``ShellJob`` itself."""
+    filename_submit_script = ShellJob.spec().inputs['metadata']['options']['submit_script_filename'].default
+
+    # Create a ``RemoteData`` node containing a file with the default submit script name. The content is ``SENTINEL``
+    # so that it can be easily detected in the final assert of this test.
+    dirpath = tmp_path / 'remote'
+    dirpath.mkdir()
+    (dirpath / filename_submit_script).write_text('SENTINEL')
+
+    inputs = {
+        'code': generate_code(),
+        'arguments': [],
+        'nodes': {
+            'remote': RemoteData(remote_path=str(dirpath.absolute()), computer=aiida_localhost),
+        },
+    }
+    _, node = run_get_node(ShellJob, inputs)
+
+    # Now retrieve the content of the submit script that was written to the working directory. It should not be equal
+    # to the content of the submit script in the ``remote`` input node which would mean it overwrote the submit script
+    # of the shell job itself.
+    filepath_submit_script = dirpath / 'output'
+    node.outputs.remote_folder.getfile(filename_submit_script, str(filepath_submit_script))
+    assert filepath_submit_script.read_text() != 'SENTINEL'