From 0d006b3da78f5f538f66094b6eb718d1f8d743d2 Mon Sep 17 00:00:00 2001
From: Stavros Efthymiou <35475381+stavros11@users.noreply.github.com>
Date: Mon, 5 Apr 2021 15:14:43 +0400
Subject: [PATCH 01/11] Fix argument order

---
 examples/benchmarks/main.py | 103 +++++++++++++++---------------------
 1 file changed, 42 insertions(+), 61 deletions(-)

diff --git a/examples/benchmarks/main.py b/examples/benchmarks/main.py
index 74f4a4ba4f..33f708fdbe 100644
--- a/examples/benchmarks/main.py
+++ b/examples/benchmarks/main.py
@@ -6,25 +6,28 @@
 import argparse
 import os
 import time
-from typing import Dict, List, Optional
 
-_PARAM_NAMES = {"theta", "phi"}
+
 parser = argparse.ArgumentParser()
 parser.add_argument("--nqubits", default="20", type=str)
-parser.add_argument("--backend", default="custom", type=str)
 parser.add_argument("--type", default="qft", type=str)
+parser.add_argument("--backend", default="custom", type=str)
+parser.add_argument("--precision", default="double", type=str)
+
+parser.add_argument("--device", default="/CPU:0", type=str)
+parser.add_argument("--accelerators", default=None, type=str)
+
+parser.add_argument("--nshots", default=None, type=int)
 parser.add_argument("--fuse", action="store_true")
+parser.add_argument("--compile", action="store_true")
 parser.add_argument("--nlayers", default=None, type=int)
 parser.add_argument("--gate-type", default=None, type=str)
-parser.add_argument("--nshots", default=None, type=int)
-parser.add_argument("--device", default="/CPU:0", type=str)
-parser.add_argument("--accelerators", default=None, type=str)
+
 parser.add_argument("--memory", default=None, type=int)
-parser.add_argument("--directory", default=None, type=str)
-parser.add_argument("--name", default=None, type=str)
-parser.add_argument("--compile", action="store_true")
-parser.add_argument("--precision", default="double", type=str)
+parser.add_argument("--filename", default=None, type=str)
+
 # params
+_PARAM_NAMES = {"theta", "phi"}
 parser.add_argument("--theta", default=None, type=float)
 parser.add_argument("--phi", default=None, type=float)
 args = vars(parser.parse_args())
@@ -53,25 +56,17 @@ def limit_gpu_memory(memory_limit=None):
 limit_gpu_memory(args.pop("memory"))
 
 import qibo
-qibo.set_backend(args.pop("backend"))
-qibo.set_precision(args.pop("precision"))
 import circuits
 import utils
 
 
-def main(nqubits_list: List[int],
-         type: str,
-         device: Optional[str] = "/CPU:0",
-         accelerators: Optional[Dict[str, int]] = None,
-         fuse: bool = False,
-         nlayers: Optional[int] = None,
-         gate_type: Optional[str] = None,
-         params: Dict[str, float] = {},
-         nshots: Optional[int] = None,
-         directory: Optional[str] = None,
-         name: Optional[str] = None,
-         compile: bool = False):
-    """Runs benchmarks for the Quantum Fourier Transform.
+def main(nqubits_list, type,
+         backend="custom", precision="double",
+         device="/CPU:0", accelerators=None,
+         nshots=None, fuse=False, compile=False,
+         nlayers=None, gate_type=None, params={},
+         filename=None):
+    """Runs benchmarks for different circuit types.
 
     If `directory` is specified this saves an `.h5` file that contains the
     following keys:
@@ -81,49 +76,35 @@ def main(nqubits_list: List[int],
             qubits. This is saved only if `compile` is `True`.
 
     Args:
-        nqubits_list: List with the number of qubits to run for.
-        type: Type of Circuit to use.
+        nqubits (int): Number of qubits in the circuit.
+        type (str): Type of Circuit to use.
             See ``benchmark_models.py`` for available types.
-        device: Tensorflow logical device to use for the benchmark.
+        device (str): Tensorflow logical device to use for the benchmark.
             If ``None`` the first available device is used.
-        nlayers: Number of layers for supremacy-like or gate circuits.
+        accelerators (dict): Dictionary that specifies the accelarator devices
+            for multi-GPU setups.
+        nshots (int): Number of measurement shots.
+            If ``None`` no measurements are performed.
+        fuse (bool): If ``True`` gate fusion is used for faster circuit execution.
+        compile: If ``True`` then the Tensorflow graph is compiled using
+            ``circuit.compile()``. Compilation time is logged in this case.
+        nlayers (int): Number of layers for supremacy-like or gate circuits.
             If a different circuit is used ``nlayers`` is ignored.
-        gate_type: Type of gate for gate circuits.
+        gate_type (str): Type of gate for gate circuits.
             If a different circuit is used ``gate_type`` is ignored.
-        params: Gate parameter for gate circuits.
+        params (dict): Gate parameter for gate circuits.
             If a non-parametrized circuit is used then ``params`` is ignored.
-        nshots: Number of measurement shots.
-        directory: Directory to save the log files.
-            If ``None`` then logs are not saved.
-        name: Name of the run to be used when saving logs.
-            This should be specified if a directory in given. Otherwise it
-            is ignored.
-        compile: If ``True`` then the Tensorflow graph is compiled using
-            ``circuit.compile()``. In this case the compile time is also logged.
-
-    Raises:
-        FileExistsError if the file with the `name` specified exists in the
-        given `directory`.
+        filename (str): Name of file to write logs.
+            If ``None`` logs will not be saved.
     """
+    qibo.set_backend(args.pop("backend"))
+    qibo.set_precision(args.pop("precision"))
+
     if device is None:
         device = tf.config.list_logical_devices()[0].name
 
-    if directory is not None:
-        if name is None:
-            raise ValueError("A run name should be given in order to save logs.")
-
-        # Generate log file name
-        log_name = [name]
-        if compile:
-            log_name.append("compiled")
-        log_name = "{}.h5".format("_".join(log_name))
-        # Generate log file path
-        file_path = os.path.join(directory, log_name)
-        if os.path.exists(file_path):
-            raise FileExistsError("File {} already exists in {}."
-                                  "".format(log_name, directory))
-
-        print("Saving logs in {}.".format(file_path))
+    if filename is not None:
+        print("Saving logs in {}.".format(filename))
 
     # Create log dict
     logs = {"nqubits": [], "simulation_time": [], "creation_time": []}
@@ -172,8 +153,8 @@ def main(nqubits_list: List[int],
         logs["nqubits"].append(nqubits)
 
         # Write updated logs in file
-        if directory is not None:
-            utils.update_file(file_path, logs)
+        if filename is not None:
+            utils.update_file(filename, logs)
 
         # Print results during run
         print("Creation time:", logs["creation_time"][-1])

From ea2ba5cd447fc3b299ec98d6c6c62a7ab64f152d Mon Sep 17 00:00:00 2001
From: Stavros Efthymiou <35475381+stavros11@users.noreply.github.com>
Date: Mon, 5 Apr 2021 15:16:42 +0400
Subject: [PATCH 02/11] Remove nqubits_list

---
 examples/benchmarks/main.py | 90 ++++++++++++++++++-------------------
 1 file changed, 44 insertions(+), 46 deletions(-)

diff --git a/examples/benchmarks/main.py b/examples/benchmarks/main.py
index 33f708fdbe..d97155af6c 100644
--- a/examples/benchmarks/main.py
+++ b/examples/benchmarks/main.py
@@ -9,7 +9,7 @@
 
 
 parser = argparse.ArgumentParser()
-parser.add_argument("--nqubits", default="20", type=str)
+parser.add_argument("--nqubits", default=20, type=int)
 parser.add_argument("--type", default="qft", type=str)
 parser.add_argument("--backend", default="custom", type=str)
 parser.add_argument("--precision", default="double", type=str)
@@ -60,7 +60,7 @@ def limit_gpu_memory(memory_limit=None):
 import utils
 
 
-def main(nqubits_list, type,
+def main(nqubits, type,
          backend="custom", precision="double",
          device="/CPU:0", accelerators=None,
          nshots=None, fuse=False, compile=False,
@@ -114,58 +114,56 @@ def main(nqubits_list, type,
     # Set circuit type
     print("Running {} benchmarks.".format(type))
 
-    for nqubits in nqubits_list:
-        kwargs = {"nqubits": nqubits, "circuit_type": type}
-        params = {k: v for k, v in params.items() if v is not None}
-        if params: kwargs["params"] = params
-        if nlayers is not None: kwargs["nlayers"] = nlayers
-        if gate_type is not None: kwargs["gate_type"] = gate_type
-        if accelerators is not None:
-            kwargs["accelerators"] = accelerators
-            kwargs["memory_device"] = device
-
-        start_time = time.time()
-        circuit = circuits.CircuitFactory(**kwargs)
-        if fuse:
-            circuit = circuit.fuse()
-        logs["creation_time"].append(time.time() - start_time)
-
-        try:
-            actual_backend = circuit.queue[0].einsum.__class__.__name__
-        except AttributeError:
-            actual_backend = "Custom"
-
-        print("\nBenchmark parameters:", kwargs)
-        print("Actual backend:", actual_backend)
-        with tf.device(device):
-            if compile:
-                start_time = time.time()
-                circuit.compile()
-                # Try executing here so that compile time is not included
-                # in the simulation time
-                final_state = circuit.execute(nshots=nshots)
-                logs["compile_time"].append(time.time() - start_time)
-
+    kwargs = {"nqubits": nqubits, "circuit_type": type}
+    params = {k: v for k, v in params.items() if v is not None}
+    if params: kwargs["params"] = params
+    if nlayers is not None: kwargs["nlayers"] = nlayers
+    if gate_type is not None: kwargs["gate_type"] = gate_type
+    if accelerators is not None:
+        kwargs["accelerators"] = accelerators
+        kwargs["memory_device"] = device
+
+    start_time = time.time()
+    circuit = circuits.CircuitFactory(**kwargs)
+    if fuse:
+        circuit = circuit.fuse()
+    logs["creation_time"].append(time.time() - start_time)
+
+    try:
+        actual_backend = circuit.queue[0].einsum.__class__.__name__
+    except AttributeError:
+        actual_backend = "Custom"
+
+    print("\nBenchmark parameters:", kwargs)
+    print("Actual backend:", actual_backend)
+    with tf.device(device):
+        if compile:
             start_time = time.time()
+            circuit.compile()
+            # Try executing here so that compile time is not included
+            # in the simulation time
             final_state = circuit.execute(nshots=nshots)
-            logs["simulation_time"].append(time.time() - start_time)
+            logs["compile_time"].append(time.time() - start_time)
 
-        logs["nqubits"].append(nqubits)
+        start_time = time.time()
+        final_state = circuit.execute(nshots=nshots)
+        logs["simulation_time"].append(time.time() - start_time)
 
-        # Write updated logs in file
-        if filename is not None:
-            utils.update_file(filename, logs)
+    logs["nqubits"].append(nqubits)
 
-        # Print results during run
-        print("Creation time:", logs["creation_time"][-1])
-        if compile:
-            print("Compile time:", logs["compile_time"][-1])
-        print("Simulation time:", logs["simulation_time"][-1])
-        print("Final dtype:", final_state.dtype)
+    # Write updated logs in file
+    if filename is not None:
+        utils.update_file(filename, logs)
+
+    # Print results during run
+    print("Creation time:", logs["creation_time"][-1])
+    if compile:
+        print("Compile time:", logs["compile_time"][-1])
+    print("Simulation time:", logs["simulation_time"][-1])
+    print("Final dtype:", final_state.dtype)
 
 
 if __name__ == "__main__":
-    args["nqubits_list"] = utils.parse_nqubits(args.pop("nqubits"))
     args["accelerators"] = utils.parse_accelerators(args.pop("accelerators"))
     args["params"] = {k: args.pop(k) for k in _PARAM_NAMES}
     main(**args)

From 573e34a1a0270e1b9325ac0e1c3fd0c30a1824c2 Mon Sep 17 00:00:00 2001
From: Stavros Efthymiou <35475381+stavros11@users.noreply.github.com>
Date: Mon, 5 Apr 2021 15:39:00 +0400
Subject: [PATCH 03/11] Fix logging

---
 examples/benchmarks/circuits.py |  4 +-
 examples/benchmarks/main.py     | 81 +++++++++++++++++----------------
 2 files changed, 43 insertions(+), 42 deletions(-)

diff --git a/examples/benchmarks/circuits.py b/examples/benchmarks/circuits.py
index 5f8fa8008d..4566bd2c83 100644
--- a/examples/benchmarks/circuits.py
+++ b/examples/benchmarks/circuits.py
@@ -103,11 +103,11 @@ def ToffoliGate(nqubits: int, nlayers: int = 1):
 def CircuitFactory(nqubits: int,
                    circuit_type: str,
                    accelerators: Dict[str, int] = None,
-                   memory_device: str = "/CPU:0",
+                   device: str = "/CPU:0",
                    **kwargs):
     if circuit_type == "qft":
         circuit = models.QFT(nqubits, accelerators=accelerators,
-                             memory_device=memory_device)
+                             memory_device=device)
     else:
         if circuit_type not in _CIRCUITS:
             raise TypeError("Unknown benchmark circuit type {}."
diff --git a/examples/benchmarks/main.py b/examples/benchmarks/main.py
index d97155af6c..baf598f11a 100644
--- a/examples/benchmarks/main.py
+++ b/examples/benchmarks/main.py
@@ -6,6 +6,7 @@
 import argparse
 import os
 import time
+import json
 
 
 parser = argparse.ArgumentParser()
@@ -14,7 +15,7 @@
 parser.add_argument("--backend", default="custom", type=str)
 parser.add_argument("--precision", default="double", type=str)
 
-parser.add_argument("--device", default="/CPU:0", type=str)
+parser.add_argument("--device", default=None, type=str)
 parser.add_argument("--accelerators", default=None, type=str)
 
 parser.add_argument("--nshots", default=None, type=int)
@@ -33,6 +34,7 @@
 args = vars(parser.parse_args())
 
 
+os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
 import tensorflow as tf
 def limit_gpu_memory(memory_limit=None):
     """Limits GPU memory that is available to Tensorflow.
@@ -62,7 +64,7 @@ def limit_gpu_memory(memory_limit=None):
 
 def main(nqubits, type,
          backend="custom", precision="double",
-         device="/CPU:0", accelerators=None,
+         device=None, accelerators=None,
          nshots=None, fuse=False, compile=False,
          nlayers=None, gate_type=None, params={},
          filename=None):
@@ -99,68 +101,67 @@ def main(nqubits, type,
     """
     qibo.set_backend(args.pop("backend"))
     qibo.set_precision(args.pop("precision"))
+    if device is not None:
+        qibo.set_device(device)
 
     if device is None:
         device = tf.config.list_logical_devices()[0].name
 
     if filename is not None:
-        print("Saving logs in {}.".format(filename))
+        if os.path.isfile(filename):
+            with open(filename, "r") as file:
+                logs = json.load(file)
+            print("Extending existing logs from {}.".format(filename))
+        else:
+            print("Creating new logs in {}.".format(filename))
+            logs = []
+    else:
+        logs = []
 
     # Create log dict
-    logs = {"nqubits": [], "simulation_time": [], "creation_time": []}
-    if compile:
-        logs["compile_time"] = []
-
-    # Set circuit type
-    print("Running {} benchmarks.".format(type))
+    logs.append({
+        "nqubits": nqubits, "circuit_type": type,
+        "backend": qibo.get_backend(), "precision": qibo.get_precision(),
+        "device": qibo.get_device(), "accelerators": accelerators,
+        "nshots": nshots, "fuse": fuse, "compile": compile
+        })
 
-    kwargs = {"nqubits": nqubits, "circuit_type": type}
     params = {k: v for k, v in params.items() if v is not None}
+    kwargs = {"nqubits": nqubits, "circuit_type": type}
     if params: kwargs["params"] = params
     if nlayers is not None: kwargs["nlayers"] = nlayers
     if gate_type is not None: kwargs["gate_type"] = gate_type
     if accelerators is not None:
         kwargs["accelerators"] = accelerators
-        kwargs["memory_device"] = device
+        kwargs["device"] = device
+    logs[-1].update(kwargs)
 
     start_time = time.time()
     circuit = circuits.CircuitFactory(**kwargs)
     if fuse:
         circuit = circuit.fuse()
-    logs["creation_time"].append(time.time() - start_time)
-
-    try:
-        actual_backend = circuit.queue[0].einsum.__class__.__name__
-    except AttributeError:
-        actual_backend = "Custom"
-
-    print("\nBenchmark parameters:", kwargs)
-    print("Actual backend:", actual_backend)
-    with tf.device(device):
-        if compile:
-            start_time = time.time()
-            circuit.compile()
-            # Try executing here so that compile time is not included
-            # in the simulation time
-            final_state = circuit.execute(nshots=nshots)
-            logs["compile_time"].append(time.time() - start_time)
+    logs[-1]["creation_time"] = time.time() - start_time
 
+    if compile:
         start_time = time.time()
-        final_state = circuit.execute(nshots=nshots)
-        logs["simulation_time"].append(time.time() - start_time)
+        circuit.compile()
+        # Try executing here so that compile time is not included
+        # in the simulation time
+        final_state = circuit(nshots=nshots)
+        logs[-1]["compile_time"] = time.time() - start_time
 
-    logs["nqubits"].append(nqubits)
+    start_time = time.time()
+    final_state = circuit(nshots=nshots)
+    logs[-1]["simulation_time"] = time.time() - start_time
+    logs[-1]["dtype"] = str(final_state.dtype)
 
-    # Write updated logs in file
-    if filename is not None:
-        utils.update_file(filename, logs)
+    print()
+    for k, v in logs[-1].items():
+        print("{}: {}".format(k, v))
 
-    # Print results during run
-    print("Creation time:", logs["creation_time"][-1])
-    if compile:
-        print("Compile time:", logs["compile_time"][-1])
-    print("Simulation time:", logs["simulation_time"][-1])
-    print("Final dtype:", final_state.dtype)
+    if filename is not None:
+        with open(filename, "w") as file:
+            json.dump(logs, file)
 
 
 if __name__ == "__main__":

From f4608c0875e2eba1064015c4b8a865c7c8029dbb Mon Sep 17 00:00:00 2001
From: Stavros Efthymiou <35475381+stavros11@users.noreply.github.com>
Date: Mon, 5 Apr 2021 15:56:03 +0400
Subject: [PATCH 04/11] Fix measurements

---
 examples/benchmarks/circuits.py |  1 -
 examples/benchmarks/main.py     | 29 ++++++++++++++---------------
 2 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/examples/benchmarks/circuits.py b/examples/benchmarks/circuits.py
index 4566bd2c83..273a007efc 100644
--- a/examples/benchmarks/circuits.py
+++ b/examples/benchmarks/circuits.py
@@ -18,7 +18,6 @@ def SupremacyLikeCircuit(nqubits: int, nlayers: int):
     for i in range(nqubits):
         gate = getattr(gates, one_qubit_gates[int(np.random.randint(0, len(one_qubit_gates)))])
         yield gate(i, np.pi / 2.0)
-        yield gates.M(i)
 
 
 def PrepareGHZ(nqubits: int):
diff --git a/examples/benchmarks/main.py b/examples/benchmarks/main.py
index baf598f11a..8ea05ab979 100644
--- a/examples/benchmarks/main.py
+++ b/examples/benchmarks/main.py
@@ -7,6 +7,7 @@
 import os
 import time
 import json
+os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" # disable Tensorflow warnings
 
 
 parser = argparse.ArgumentParser()
@@ -34,14 +35,13 @@
 args = vars(parser.parse_args())
 
 
-os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
-import tensorflow as tf
 def limit_gpu_memory(memory_limit=None):
     """Limits GPU memory that is available to Tensorflow.
 
     Args:
         memory_limit: Memory limit in MBs.
     """
+    import tensorflow as tf
     if memory_limit is None:
         print("\nNo GPU memory limiter used.\n")
         return
@@ -70,13 +70,6 @@ def main(nqubits, type,
          filename=None):
     """Runs benchmarks for different circuit types.
 
-    If `directory` is specified this saves an `.h5` file that contains the
-    following keys:
-        * nqubits: List with the number of qubits that were simulated.
-        * simulation_time: List with simulation times for each number of qubits.
-        * compile_time (optional): List with compile times for each number of
-            qubits. This is saved only if `compile` is `True`.
-
     Args:
         nqubits (int): Number of qubits in the circuit.
         type (str): Type of Circuit to use.
@@ -86,6 +79,7 @@ def main(nqubits, type,
         accelerators (dict): Dictionary that specifies the accelarator devices
             for multi-GPU setups.
         nshots (int): Number of measurement shots.
+            Logs the time required to sample frequencies (no samples).
             If ``None`` no measurements are performed.
         fuse (bool): If ``True`` gate fusion is used for faster circuit execution.
         compile: If ``True`` then the Tensorflow graph is compiled using
@@ -104,9 +98,6 @@ def main(nqubits, type,
     if device is not None:
         qibo.set_device(device)
 
-    if device is None:
-        device = tf.config.list_logical_devices()[0].name
-
     if filename is not None:
         if os.path.isfile(filename):
             with open(filename, "r") as file:
@@ -138,6 +129,9 @@ def main(nqubits, type,
 
     start_time = time.time()
     circuit = circuits.CircuitFactory(**kwargs)
+    if nshots is not None:
+        # add measurement gates
+        circuit.add(qibo.gates.M(*range(nqubits)))
     if fuse:
         circuit = circuit.fuse()
     logs[-1]["creation_time"] = time.time() - start_time
@@ -147,13 +141,18 @@ def main(nqubits, type,
         circuit.compile()
         # Try executing here so that compile time is not included
         # in the simulation time
-        final_state = circuit(nshots=nshots)
+        result = circuit(nshots=nshots)
         logs[-1]["compile_time"] = time.time() - start_time
 
     start_time = time.time()
-    final_state = circuit(nshots=nshots)
+    result = circuit(nshots=nshots)
     logs[-1]["simulation_time"] = time.time() - start_time
-    logs[-1]["dtype"] = str(final_state.dtype)
+    logs[-1]["dtype"] = str(result.dtype)
+
+    if nshots is not None:
+        start_time = time.time()
+        freqs = result.frequencies()
+        logs[-1]["measurement_time"] = time.time() - start_time
 
     print()
     for k, v in logs[-1].items():

From b8a7d8d1ce0b952cbc713c37c8c47e64ad3cb62e Mon Sep 17 00:00:00 2001
From: Stavros Efthymiou <35475381+stavros11@users.noreply.github.com>
Date: Mon, 5 Apr 2021 16:13:14 +0400
Subject: [PATCH 05/11] Update test

---
 examples/test_examples.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/test_examples.py b/examples/test_examples.py
index 8af59df3b5..4b50a13025 100644
--- a/examples/test_examples.py
+++ b/examples/test_examples.py
@@ -134,9 +134,9 @@ def test_unary(data, bins, M=10, shots=1000):
     run_script(args)
 
 
-@pytest.mark.parametrize("nqubits_list", [[3, 4]])
+@pytest.mark.parametrize("nqubits", [3, 6])
 @pytest.mark.parametrize("type", ["qft", "variational"])
-def test_benchmarks(nqubits_list, type):
+def test_benchmarks(nqubits, type):
     args = locals()
     path = os.path.join(base_dir, "benchmarks")
     sys.path[-1] = path

From 6642fd630e4ec6e0f5fc4ac65aaa35baaba17742 Mon Sep 17 00:00:00 2001
From: Stavros Efthymiou <35475381+stavros11@users.noreply.github.com>
Date: Mon, 5 Apr 2021 16:31:39 +0400
Subject: [PATCH 06/11] Update docs

---
 doc/source/benchmarks.rst | 67 ++++++++++++++++++---------------------
 1 file changed, 30 insertions(+), 37 deletions(-)

diff --git a/doc/source/benchmarks.rst b/doc/source/benchmarks.rst
index f4d9e6ded4..3d9a079632 100644
--- a/doc/source/benchmarks.rst
+++ b/doc/source/benchmarks.rst
@@ -79,55 +79,48 @@ The main benchmark script is ``main.py``. This can be
 executed as ``python main.py (OPTIONS)`` where ``(OPTIONS)`` can be any of the
 following options:
 
-* ``--nqubits``: Number of qubits in the circuit. Can be a single integer or
-  an interval defined with a dash (``-``) as ``a-b``.
-  Example: ``--nqubits 5-10`` will run the benchmark for all ``nqubits``
-  from 5 to 10 inclusive.
-
-* ``--backend``: Qibo backend to use for the calculation.
-  Available backends are ``"custom"``, ``"matmuleinsum"`` and ``"defaulteinsum"``.
-  ``"custom"`` is the default backend.
+* ``--nqubits`` (``int``): Number of qubits in the circuit.
 
-* ``--type``: Type of benchmark circuit.
+* ``--type`` (``str``): Type of benchmark circuit.
   Available circuit types are shown in the next section. Some circuit types
-  support additional options which are analyzed bellow.
+  support additional options which are described below.
 
-* ``--nshots``: Number of measurement shots.
-  If not given no measurements will be performed and the benchmark will
-  terminate once the final state vector is found.
+* ``--backend`` (``str``): Qibo backend to use for the calculation.
+  Available backends are ``"custom"``, ``"matmuleinsum"``, ``"defaulteinsum"``,
+  ``"numpy_defaulteinsum"`` and ``"numpy_matmuleinsum"``.
+  ``"custom"`` is the default backend.
 
-* ``--device``: Tensorflow device to use for the benchmarks.
+* ``--precision`` (``str``): Complex number precision to use for the benchmark.
+    Available options are ``'single'`` and ``'double'``.
+
+* ``--device`` (``str``): Tensorflow device to use for the benchmarks.
   Example: ``--device /GPU:0`` or ``--device /CPU:0``.
 
-* ``--accelerators``: Devices to use for distributed execution of the circuit.
+* ``--accelerators`` (``str``): Devices to use for distributed execution of the circuit.
   Example: ``--accelerators 1/GPU:0,1/GPU:1`` will distribute the execution
-  on two GPUs, if these are available and compatible to Tensorflow.
-
-* ``--compile``: If used, the circuit will be compiled using ``tf.function``.
-  Note: custom operators do not support compilation.
+  on two GPUs.
 
-* ``--precision``: Complex number precision to use for the benchmark.
-  Available options are ``'single'`` and ``'double'``.
+* ``--memory`` (``int``): Limits GPU memory used for execution. If no limiter is used,
+  Tensorflow uses all available by default.
 
-When a benchmark is executed, the total simulation time will be printed in the
-terminal once the simulation finishes. Optionally execution times can be saved
-in a ``.h5`` file. This can be enabled by passing the following additional flags:
-
-* ``--directory``: Directory where the ``.h5`` will be saved.
-
-* ``--name``: Name of the ``.h5`` file.
+* ``--nshots`` (``int``): Number of measurement shots.
+  This will benchmark the sampling of frequencies, not individual shot samples.
+  If not given no measurements will be performed and the benchmark will
+  terminate once the final state vector is found.
 
-If the file exists in the given directory an error will be raised. The saved file
-contains two arrays with the following keys:
+* ``--compile`` (``bool``): If used, the circuit will be compiled using ``tf.function``.
+  Note that custom operators do not support compilation.
+  Default is ``False``.
 
-  1. ``nqubits``: List with the number of qubits.
-  2. ``creation_time``: List with the time required to create the circuit for
-     each number of qubits.
-  3. ``simulation_time``: List with the total execution time for each number of
-     qubits.
+* ``--fuse`` (``bool``): Circuit gates will be fused for faster execution of some circuit
+  types. Default is ``False``.
 
-If ``--compile`` option is used, then the measured simulation time is the second
-call, while the execution time of the first call is saved as ``compile_time``.
+When a benchmark is executed, the total simulation time will be printed in the
+terminal once the simulation finishes. Optionally execution times can be saved
+by passing the ``--filename`` (``str``) flag. All benchmarks details are logged
+in a Python dictionary and saved in a text file using ``json.dump``. The logs
+include circuit creation and simulation times. If the given ``filename`` already
+exists it will be updated, otherwise it will be created.
 
 
 Available circuit types

From 892be184f9ab8034bef125a4726bb57a6f9004da Mon Sep 17 00:00:00 2001
From: Stavros Efthymiou <35475381+stavros11@users.noreply.github.com>
Date: Mon, 5 Apr 2021 16:39:26 +0400
Subject: [PATCH 07/11] Fix tests

---
 examples/benchmarks/circuits.py |  2 +-
 examples/test_examples.py       | 11 ++++++-----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/examples/benchmarks/circuits.py b/examples/benchmarks/circuits.py
index 273a007efc..f276f2de5f 100644
--- a/examples/benchmarks/circuits.py
+++ b/examples/benchmarks/circuits.py
@@ -112,6 +112,6 @@ def CircuitFactory(nqubits: int,
             raise TypeError("Unknown benchmark circuit type {}."
                             "".format(circuit_type))
         circuit = models.Circuit(nqubits, accelerators=accelerators,
-                                 memory_device=memory_device)
+                                 memory_device=device)
         circuit.add(_CIRCUITS[circuit_type](nqubits, **kwargs))
     return circuit
diff --git a/examples/test_examples.py b/examples/test_examples.py
index 4b50a13025..f6906f22be 100644
--- a/examples/test_examples.py
+++ b/examples/test_examples.py
@@ -137,7 +137,6 @@ def test_unary(data, bins, M=10, shots=1000):
 @pytest.mark.parametrize("nqubits", [3, 6])
 @pytest.mark.parametrize("type", ["qft", "variational"])
 def test_benchmarks(nqubits, type):
-    args = locals()
     path = os.path.join(base_dir, "benchmarks")
     sys.path[-1] = path
     os.chdir(path)
@@ -145,11 +144,13 @@ def test_benchmarks(nqubits, type):
     start = code.find("def main")
     end = code.find("\nif __name__ ==")
     header = ("import argparse\nimport os\nimport time"
-              "\nfrom typing import Dict, List, Optional"
-              "\nimport tensorflow as tf"
               "\nimport qibo\nimport circuits\nimport utils\n\n")
-    import qibo
-    qibo.set_backend("custom")
+    args = {"nqubits": nqubits, "type": type,
+            "backend": "custom", "precision": "double",
+            "device": None, "accelerators": None,
+            "nshots": None, "fuse": False, "compile": False,
+            "nlayers": None, "gate_type": None, "params": {},
+            "filename": None}
     code = header + code[start: end] + "\n\nmain(**args)"
     with timeout(max_time):
         exec(code, {"args": args})

From 7d77d17a94621dfca562a7a8d09b44829a4325a3 Mon Sep 17 00:00:00 2001
From: Stavros Efthymiou <35475381+stavros11@users.noreply.github.com>
Date: Mon, 5 Apr 2021 16:48:06 +0400
Subject: [PATCH 08/11] Remove utils

---
 examples/benchmarks/evolution.py | 61 ++++++++++++++++++++++--
 examples/benchmarks/main.py      | 33 ++++++++++++-
 examples/benchmarks/utils.py     | 82 --------------------------------
 3 files changed, 89 insertions(+), 87 deletions(-)
 delete mode 100644 examples/benchmarks/utils.py

diff --git a/examples/benchmarks/evolution.py b/examples/benchmarks/evolution.py
index a3fe209f0c..eb448a2a40 100644
--- a/examples/benchmarks/evolution.py
+++ b/examples/benchmarks/evolution.py
@@ -1,7 +1,6 @@
 """Adiabatic evolution for the Ising Hamiltonian using linear scaling."""
 import argparse
 import time
-import utils
 from qibo import callbacks, hamiltonians, models
 
 parser = argparse.ArgumentParser()
@@ -12,6 +11,62 @@
 parser.add_argument("--accelerators", default=None, type=str)
 
 
+def parse_nqubits(nqubits_str):
+    """Transforms a string that specifies number of qubits to list.
+
+    Supported string formats are the following:
+        * 'a-b' with a and b integers.
+            Then the returned list is range(a, b + 1).
+        * 'a,b,c,d,...' with a, b, c, d, ... integers.
+            Then the returned list is [a, b, c, d]
+    """
+    # TODO: Support usage of both `-` and `,` in the same string.
+    if "-" in nqubits_str:
+        if "," in nqubits_str:
+            raise ValueError("String that specifies qubits cannot contain "
+                             "both , and -.")
+
+        nqubits_split = nqubits_str.split("-")
+        if len(nqubits_split) != 2:
+            raise ValueError("Invalid string that specifies nqubits "
+                             "{}.".format(nqubits_str))
+
+        n_start, n_end = nqubits_split
+        return list(range(int(n_start), int(n_end) + 1))
+
+    return [int(x) for x in nqubits_str.split(",")]
+
+
+def parse_accelerators(accelerators):
+    """Transforms string that specifies accelerators to dictionary.
+
+    The string that is parsed has the following format:
+        n1device1,n2device2,n3device3,...
+    and is transformed to the dictionary:
+        {'device1': n1, 'device2': n2, 'device3': n3, ...}
+
+    Example:
+        2/GPU:0,2/GPU:1 --> {'/GPU:0': 2, '/GPU:1': 2}
+    """
+    if accelerators is None:
+        return None
+
+    def read_digit(x):
+        i = 0
+        while x[i].isdigit():
+            i += 1
+        return x[i:], int(x[:i])
+
+    acc_dict = {}
+    for entry in accelerators.split(","):
+        device, n = read_digit(entry)
+        if device in acc_dict:
+            acc_dict[device] += n
+        else:
+            acc_dict[device] = n
+    return acc_dict
+
+
 def main(nqubits_list, dt, solver, trotter=False, accelerators=None):
     """Performs adiabatic evolution with critical TFIM as the "hard" Hamiltonian."""
     if accelerators is not None:
@@ -45,6 +100,6 @@ def main(nqubits_list, dt, solver, trotter=False, accelerators=None):
 
 if __name__ == "__main__":
     args = vars(parser.parse_args())
-    args["nqubits_list"] = utils.parse_nqubits(args.pop("nqubits"))
-    args["accelerators"] = utils.parse_accelerators(args.pop("accelerators"))
+    args["nqubits_list"] = parse_nqubits(args.pop("nqubits"))
+    args["accelerators"] = parse_accelerators(args.pop("accelerators"))
     main(**args)
diff --git a/examples/benchmarks/main.py b/examples/benchmarks/main.py
index 8ea05ab979..9fe8b6a124 100644
--- a/examples/benchmarks/main.py
+++ b/examples/benchmarks/main.py
@@ -59,7 +59,36 @@ def limit_gpu_memory(memory_limit=None):
 
 import qibo
 import circuits
-import utils
+
+
+def parse_accelerators(accelerators):
+    """Transforms string that specifies accelerators to dictionary.
+
+    The string that is parsed has the following format:
+        n1device1,n2device2,n3device3,...
+    and is transformed to the dictionary:
+        {'device1': n1, 'device2': n2, 'device3': n3, ...}
+
+    Example:
+        2/GPU:0,2/GPU:1 --> {'/GPU:0': 2, '/GPU:1': 2}
+    """
+    if accelerators is None:
+        return None
+
+    def read_digit(x):
+        i = 0
+        while x[i].isdigit():
+            i += 1
+        return x[i:], int(x[:i])
+
+    acc_dict = {}
+    for entry in accelerators.split(","):
+        device, n = read_digit(entry)
+        if device in acc_dict:
+            acc_dict[device] += n
+        else:
+            acc_dict[device] = n
+    return acc_dict
 
 
 def main(nqubits, type,
@@ -164,6 +193,6 @@ def main(nqubits, type,
 
 
 if __name__ == "__main__":
-    args["accelerators"] = utils.parse_accelerators(args.pop("accelerators"))
+    args["accelerators"] = parse_accelerators(args.pop("accelerators"))
     args["params"] = {k: args.pop(k) for k in _PARAM_NAMES}
     main(**args)
diff --git a/examples/benchmarks/utils.py b/examples/benchmarks/utils.py
deleted file mode 100644
index 677b23ac82..0000000000
--- a/examples/benchmarks/utils.py
+++ /dev/null
@@ -1,82 +0,0 @@
-"""
-Various utilities used by the benchmark scripts.
-"""
-import h5py
-import numpy as np
-from typing import Dict, List, Optional
-
-
-def update_file(file_path: str, logs: Dict[str, List]):
-    """Updates (or creates) an `.h5` file with the current logs.
-
-    Args:
-        file_path: Full path of the file to be created/updated.
-            This should end in `.h5`.
-        logs: Dictionary that contains the data to be written in the file.
-    """
-    file = h5py.File(file_path, "w")
-    for k, v in logs.items():
-        file[k] = np.array(v)
-    file.close()
-
-
-def random_state(nqubits: int) -> np.ndarray:
-    """Generates a random state."""
-    x = np.random.random(2**nqubits) + 1j * np.random.random(2**nqubits)
-    return x / np.sqrt((np.abs(x)**2).sum())
-
-
-def parse_nqubits(nqubits_str: str) -> List[int]:
-    """Transforms a string that specifies number of qubits to list.
-
-    Supported string formats are the following:
-        * 'a-b' with a and b integers.
-            Then the returned list is range(a, b + 1).
-        * 'a,b,c,d,...' with a, b, c, d, ... integers.
-            Then the returned list is [a, b, c, d]
-    """
-    # TODO: Support usage of both `-` and `,` in the same string.
-    if "-" in nqubits_str:
-        if "," in nqubits_str:
-            raise ValueError("String that specifies qubits cannot contain "
-                             "both , and -.")
-
-        nqubits_split = nqubits_str.split("-")
-        if len(nqubits_split) != 2:
-            raise ValueError("Invalid string that specifies nqubits "
-                             "{}.".format(nqubits_str))
-
-        n_start, n_end = nqubits_split
-        return list(range(int(n_start), int(n_end) + 1))
-
-    return [int(x) for x in nqubits_str.split(",")]
-
-
-def parse_accelerators(accelerators: Optional[str]) -> Dict[str, int]:
-    """Transforms string that specifies accelerators to the dictionary.
-
-    The string that is parsed has the following format:
-        n1device1,n2device2,n3device3,...
-    and is transformed to the dictionary:
-        {'device1': n1, 'device2': n2, 'device3': n3, ...}
-
-    Example:
-        2/GPU:0,2/GPU:1 --> {'/GPU:0': 2, '/GPU:1': 2}
-    """
-    if accelerators is None:
-        return None
-
-    def read_digit(x):
-        i = 0
-        while x[i].isdigit():
-            i += 1
-        return x[i:], int(x[:i])
-
-    acc_dict = {}
-    for entry in accelerators.split(","):
-        device, n = read_digit(entry)
-        if device in acc_dict:
-            acc_dict[device] += n
-        else:
-            acc_dict[device] = n
-    return acc_dict

From 218856545bbe8192b0909e9c5271b5f040d6a191 Mon Sep 17 00:00:00 2001
From: Stavros Efthymiou <35475381+stavros11@users.noreply.github.com>
Date: Mon, 5 Apr 2021 18:04:32 +0400
Subject: [PATCH 09/11] Fix accelerators documentation

---
 doc/source/benchmarks.rst   | 3 ++-
 examples/benchmarks/main.py | 4 ++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/doc/source/benchmarks.rst b/doc/source/benchmarks.rst
index 3d9a079632..5270e7e179 100644
--- a/doc/source/benchmarks.rst
+++ b/doc/source/benchmarks.rst
@@ -98,7 +98,8 @@ following options:
 
 * ``--accelerators`` (``str``): Devices to use for distributed execution of the circuit.
   Example: ``--accelerators 1/GPU:0,1/GPU:1`` will distribute the execution
-  on two GPUs.
+  on two GPUs. The coefficient of each device denotes the number of times to
+  reuse this device.
 
 * ``--memory`` (``int``): Limits GPU memory used for execution. If no limiter is used,
   Tensorflow uses all available by default.
diff --git a/examples/benchmarks/main.py b/examples/benchmarks/main.py
index 9fe8b6a124..1daf9312e2 100644
--- a/examples/benchmarks/main.py
+++ b/examples/benchmarks/main.py
@@ -122,8 +122,8 @@ def main(nqubits, type,
         filename (str): Name of file to write logs.
             If ``None`` logs will not be saved.
     """
-    qibo.set_backend(args.pop("backend"))
-    qibo.set_precision(args.pop("precision"))
+    qibo.set_backend(backend)
+    qibo.set_precision(precision)
     if device is not None:
         qibo.set_device(device)
 

From 9efd46757ae24b05d6b57949389b2958e9b328b3 Mon Sep 17 00:00:00 2001
From: Stavros Efthymiou <35475381+stavros11@users.noreply.github.com>
Date: Mon, 5 Apr 2021 18:09:54 +0400
Subject: [PATCH 10/11] Example bash script

---
 examples/benchmarks/example.sh | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100755 examples/benchmarks/example.sh

diff --git a/examples/benchmarks/example.sh b/examples/benchmarks/example.sh
new file mode 100755
index 0000000000..beb3cc2320
--- /dev/null
+++ b/examples/benchmarks/example.sh
@@ -0,0 +1,32 @@
+# example bash file for executing `main.py` benchmarks
+# device configurations and `nqubits` are based on DGX machine with
+# 4x V100 32GB GPUs
+FILENAME="dgx.dat" # log file name
+export CUDA_VISIBLE_DEVICES="0"
+for NQUBITS in 20 21 22 23 24 25 26 27 28 29 30
+do
+  python main.py --type qft --nqubits $NQUBITS --filename $FILENAME
+  python main.py --type qft --nqubits $NQUBITS --filename $FILENAME --precision "single"
+  python main.py --type variational --nqubits $NQUBITS --filename $FILENAME
+  python main.py --type variational --nqubits $NQUBITS --filename $FILENAME --precision "single"
+done
+python main.py --type qft --nqubits 31 --filename $FILENAME --precision "single"
+python main.py --type variational --nqubits 31 --filename $FILENAME --precision "single"
+export CUDA_VISIBLE_DEVICES=""
+for NQUBITS in 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
+do
+  python main.py --type qft --nqubits $NQUBITS --filename $FILENAME
+  python main.py --type variational --nqubits $NQUBITS --filename $FILENAME
+done
+
+export CUDA_VISIBLE_DEVICES="0,1"
+ACCEL="1/GPU:0,1/GPU:1"
+python main.py --type qft --nqubits 31 --filename $FILENAME --device "/CPU:0" --accelerators $ACCEL
+python main.py --type qft --nqubits 32 --filename $FILENAME --device "/CPU:0" --accelerators $ACCEL --precision "single"
+export CUDA_VISIBLE_DEVICES="0,1,2,3"
+ACCEL="1/GPU:0,1/GPU:1,1/GPU:2,1/GPU:3"
+python main.py --type qft --nqubits 32 --filename $FILENAME --device "/CPU:0" --accelerators $ACCEL
+python main.py --type qft --nqubits 33 --filename $FILENAME --device "/CPU:0" --accelerators $ACCEL --precision "single"
+ACCEL="2/GPU:0,2/GPU:1,2/GPU:2,2/GPU:3"
+python main.py --type qft --nqubits 33 --filename $FILENAME --device "/CPU:0" --accelerators $ACCEL
+python main.py --type qft --nqubits 34 --filename $FILENAME --device "/CPU:0" --accelerators $ACCEL --precision "single"

From 5eb964ab7ce1b16bdc46289e1b782329767d1377 Mon Sep 17 00:00:00 2001
From: Stavros Efthymiou <35475381+stavros11@users.noreply.github.com>
Date: Mon, 5 Apr 2021 18:38:02 +0400
Subject: [PATCH 11/11] Remove utils from example tests

---
 examples/test_examples.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/test_examples.py b/examples/test_examples.py
index f6906f22be..d0bfcfa163 100644
--- a/examples/test_examples.py
+++ b/examples/test_examples.py
@@ -144,7 +144,7 @@ def test_benchmarks(nqubits, type):
     start = code.find("def main")
     end = code.find("\nif __name__ ==")
     header = ("import argparse\nimport os\nimport time"
-              "\nimport qibo\nimport circuits\nimport utils\n\n")
+              "\nimport qibo\nimport circuits\n\n")
     args = {"nqubits": nqubits, "type": type,
             "backend": "custom", "precision": "double",
             "device": None, "accelerators": None,