[WIP] Proof of concept for using PyO3 in cargo unit tests.

.azure/test-linux.yml

@@ -45,17 +45,6 @@ jobs:
           path: .stestr
         displayName: "Cache stestr"
 
-      - ${{ if eq(parameters.testRust, true) }}:
-        # We need to avoid linking our crates into full Python extension libraries during Rust-only
-        # testing because Rust/PyO3 can't handle finding a static CPython interpreter.
-        - bash: cargo test --no-default-features
-          env:
-            # On Linux we link against `libpython` dynamically, but it isn't written into the rpath
-            # of the test executable (I'm not 100% sure why ---Jake).  It's easiest just to forcibly
-            # include the correct place in the `dlopen` search path.
-            LD_LIBRARY_PATH: '$(usePython.pythonLocation)/lib:$LD_LIBRARY_PATH'
-          displayName: "Run Rust tests"
-
       - bash: |
           set -e
           python -m pip install --upgrade pip setuptools wheel virtualenv
@@ -107,6 +96,22 @@ jobs:
             sudo apt-get install -y graphviz
           displayName: 'Install optional non-Python dependencies'
 
+      # Note that we explicitly use the virtual env with Qiskit installed to run the Rust
+      # tests since some of them still depend on Qiskit's Python API via PyO3.
+      - ${{ if eq(parameters.testRust, true) }}:
+          # We need to avoid linking our crates into full Python extension libraries during Rust-only
+          # testing because Rust/PyO3 can't handle finding a static CPython interpreter.
+          - bash: |
+              source test-job/bin/activate
+              python tools/report_numpy_state.py
+              cargo test --no-default-features
+            env:
+              # On Linux we link against `libpython` dynamically, but it isn't written into the rpath
+              # of the test executable (I'm not 100% sure why ---Jake).  It's easiest just to forcibly
+              # include the correct place in the `dlopen` search path.
+              LD_LIBRARY_PATH: '$(usePython.pythonLocation)/lib:$LD_LIBRARY_PATH'
+            displayName: "Run Rust tests"
+
       - bash: |
           set -e
           source test-job/bin/activate

CONTRIBUTING.md

@@ -572,10 +572,15 @@ Note: If you have run `test/ipynb/mpl_tester.ipynb` locally it is possible some
 
 ### Testing Rust components
 
-Rust-accelerated functions are generally tested from Python space, but in cases
-where there is Rust-specific internal details to be tested, `#[test]` functions
-can be included inline.  Typically it's most convenient to place these in a
-separate inline module that is only conditionally compiled in, such as
+Many of Qiskit's core data structures and algorithms are implemented in Rust.
+The bulk of this code is exercised heavily by our Python-based unit testing,
+but this coverage really only provides integration-level testing from the
+perspective of Rust.
+
+To provide proper Rust unit testing, we use `cargo test`. Rust tests are
+integrated directly into the Rust file being tested within a `tests` module.
+Functions decorated with `#[test]` within these modules are built and run
+as tests.
 
 ```rust
 #[cfg(test)]
@@ -587,15 +592,57 @@ mod tests {
 }
 ```
 
-To run the Rust-space tests, do
+Rust tests are run separately from the Python tests. To run them, do
 
 ```bash
+python setup.py build_rust --release --inplace
 cargo test --no-default-features
 ```
 
-Our Rust-space components are configured such that setting the
-``-no-default-features`` flag will compile the test runner, but not attempt to
-build a linked CPython extension module, which would cause linker failures.
+The first command builds Qiskit from source (in release mode, but --debug is fine too),
+which ensures that Rust tests that interact with Qiskit's Python code actually
+use the latest Python code from your working directory.
+
+The second command actually invokes the tests via Cargo. The ``-no-default-features``
+flag is used to compile an isolated test runner without building a linked CPython
+extension module (which would otherwise cause linker failures).
+
+#### Calling Python from Rust tests
+By default, our Cargo project configuration allows Rust tests to interact with the
+Python interpreter by calling `Python::with_gil` to obtain a `Python` (`py`) token.
+This is particularly helpful when testing Rust code that (still) requires interaction
+with Python.
+
+To execute code that needs the GIL in your tests, define the `tests` module as
+follows:
+
+```rust
+#[cfg(all(test, not(miri)))] // disable for Miri!
+mod tests {
+    use pyo3::prelude::*;
+
+    #[test]
+    fn my_first_test() {
+        Python::with_gil(|py| {
+            todo!() // do something that needs a `py` token.
+        })
+    }
+}
+```
+
+To ensure that Rust tests are properly importing changes in Qiskit's Python code
+from the working directory, be sure to build and install Qiskit from source prior
+to running `cargo test --no-default-features`. 
+
+> [!IMPORTANT]
+> Note that we explicitly disable compilation of such tests when running with Miri, i.e.
+`#[cfg(not(miri))]`. This is necessary because Miri doesn't support the FFI
+> code used internally by PyO3.
+>
+> If not all of your tests will use the `Python` token, you can disable Miri on a per-test
+basis within the same module by decorating *the specific test* with `#[cfg_attr(miri, ignore)]`
+instead of disabling Miri for the entire module.
+
 
 ### Unsafe code and Miri
 

crates/accelerate/Cargo.toml

@@ -58,3 +58,6 @@ features = ["ndarray"]
 [dependencies.pulp]
 version = "0.18.22"
 features = ["macro"]
+
+[dev-dependencies]
+pyo3 = { workspace = true, features = ["auto-initialize"] }

crates/circuit/Cargo.toml

@@ -39,3 +39,6 @@ features = ["union"]
 
 [features]
 cache_pygates = []
+
+[dev-dependencies]
+pyo3 = { workspace = true, features = ["auto-initialize"] }

crates/circuit/src/dag_circuit.rs

@@ -6800,3 +6800,48 @@ fn add_global_phase(py: Python, phase: &Param, other: &Param) -> PyResult<Param>
 }
 
 type SortKeyType<'a> = (&'a [Qubit], &'a [Clbit]);
+
+#[cfg(all(test, not(miri)))]
+mod test {
+    use crate::dag_circuit::DAGCircuit;
+    use crate::imports::{CLASSICAL_REGISTER, QUANTUM_REGISTER};
+    use crate::operations::StandardGate;
+    use crate::packed_instruction::{PackedInstruction, PackedOperation};
+    use crate::Qubit;
+    use pyo3::prelude::*;
+    use pyo3::Python;
+
+    fn new_dag(py: Python, qubits: u32, clbits: u32) -> DAGCircuit {
+        let qreg = QUANTUM_REGISTER.get_bound(py).call1((qubits,)).unwrap();
+        let creg = CLASSICAL_REGISTER.get_bound(py).call1((clbits,)).unwrap();
+        let mut dag = DAGCircuit::new(py).unwrap();
+        dag.add_qreg(py, &qreg).unwrap();
+        dag.add_creg(py, &creg).unwrap();
+        dag
+    }
+
+    #[test]
+    fn test_push_back() {
+        Python::with_gil(|py| {
+            let mut dag = new_dag(py, 4, 4);
+            let cx = PackedInstruction {
+                op: PackedOperation::from_standard(StandardGate::CXGate),
+                qubits: dag.qargs_interner.insert_owned(vec![Qubit(0), Qubit(1)]),
+                clbits: dag.cargs_interner.get_default(),
+                params: None,
+                extra_attrs: Default::default(),
+                #[cfg(feature = "cache_pygates")]
+                py_op: Default::default(),
+            };
+            let cx_node = dag.push_back(py, cx).unwrap();
+
+            let [q0_in_node, q0_out_node] = dag.qubit_io_map[0];
+            let [q1_in_node, q1_out_node] = dag.qubit_io_map[1];
+
+            assert!(dag.dag.find_edge(q0_in_node, cx_node).is_some());
+            assert!(dag.dag.find_edge(q1_in_node, cx_node).is_some());
+            assert!(dag.dag.find_edge(cx_node, q0_out_node).is_some());
+            assert!(dag.dag.find_edge(cx_node, q1_out_node).is_some());
+        });
+    }
+}