Update main benchmark script

doc/source/benchmarks.rst

@@ -79,55 +79,49 @@ The main benchmark script is ``main.py``. This can be
 executed as ``python main.py (OPTIONS)`` where ``(OPTIONS)`` can be any of the
 following options:
 
-* ``--nqubits``: Number of qubits in the circuit. Can be a single integer or
-  an interval defined with a dash (``-``) as ``a-b``.
-  Example: ``--nqubits 5-10`` will run the benchmark for all ``nqubits``
-  from 5 to 10 inclusive.
-
-* ``--backend``: Qibo backend to use for the calculation.
-  Available backends are ``"custom"``, ``"matmuleinsum"`` and ``"defaulteinsum"``.
-  ``"custom"`` is the default backend.
+* ``--nqubits`` (``int``): Number of qubits in the circuit.
 
-* ``--type``: Type of benchmark circuit.
+* ``--type`` (``str``): Type of benchmark circuit.
   Available circuit types are shown in the next section. Some circuit types
-  support additional options which are analyzed bellow.
+  support additional options which are described below.
 
-* ``--nshots``: Number of measurement shots.
-  If not given no measurements will be performed and the benchmark will
-  terminate once the final state vector is found.
+* ``--backend`` (``str``): Qibo backend to use for the calculation.
+  Available backends are ``"custom"``, ``"matmuleinsum"``, ``"defaulteinsum"``,
+  ``"numpy_defaulteinsum"`` and ``"numpy_matmuleinsum"``.
+  ``"custom"`` is the default backend.
 
-* ``--device``: Tensorflow device to use for the benchmarks.
+* ``--precision`` (``str``): Complex number precision to use for the benchmark.
+    Available options are ``'single'`` and ``'double'``.
+
+* ``--device`` (``str``): Tensorflow device to use for the benchmarks.
   Example: ``--device /GPU:0`` or ``--device /CPU:0``.
 
-* ``--accelerators``: Devices to use for distributed execution of the circuit.
+* ``--accelerators`` (``str``): Devices to use for distributed execution of the circuit.
   Example: ``--accelerators 1/GPU:0,1/GPU:1`` will distribute the execution
-  on two GPUs, if these are available and compatible to Tensorflow.
-
-* ``--compile``: If used, the circuit will be compiled using ``tf.function``.
-  Note: custom operators do not support compilation.
+  on two GPUs. The coefficient of each device denotes the number of times to
+  reuse this device.
 
-* ``--precision``: Complex number precision to use for the benchmark.
-  Available options are ``'single'`` and ``'double'``.
+* ``--memory`` (``int``): Limits GPU memory used for execution. If no limiter is used,
+  Tensorflow uses all available by default.
 
-When a benchmark is executed, the total simulation time will be printed in the
-terminal once the simulation finishes. Optionally execution times can be saved
-in a ``.h5`` file. This can be enabled by passing the following additional flags:
-
-* ``--directory``: Directory where the ``.h5`` will be saved.
-
-* ``--name``: Name of the ``.h5`` file.
+* ``--nshots`` (``int``): Number of measurement shots.
+  This will benchmark the sampling of frequencies, not individual shot samples.
+  If not given no measurements will be performed and the benchmark will
+  terminate once the final state vector is found.
 
-If the file exists in the given directory an error will be raised. The saved file
-contains two arrays with the following keys:
+* ``--compile`` (``bool``): If used, the circuit will be compiled using ``tf.function``.
+  Note that custom operators do not support compilation.
+  Default is ``False``.
 
-  1. ``nqubits``: List with the number of qubits.
-  2. ``creation_time``: List with the time required to create the circuit for
-     each number of qubits.
-  3. ``simulation_time``: List with the total execution time for each number of
-     qubits.
+* ``--fuse`` (``bool``): Circuit gates will be fused for faster execution of some circuit
+  types. Default is ``False``.
 
-If ``--compile`` option is used, then the measured simulation time is the second
-call, while the execution time of the first call is saved as ``compile_time``.
+When a benchmark is executed, the total simulation time will be printed in the
+terminal once the simulation finishes. Optionally execution times can be saved
+by passing the ``--filename`` (``str``) flag. All benchmarks details are logged
+in a Python dictionary and saved in a text file using ``json.dump``. The logs
+include circuit creation and simulation times. If the given ``filename`` already
+exists it will be updated, otherwise it will be created.
 
 
 Available circuit types

examples/benchmarks/circuits.py

@@ -18,7 +18,6 @@ def SupremacyLikeCircuit(nqubits: int, nlayers: int):
     for i in range(nqubits):
         gate = getattr(gates, one_qubit_gates[int(np.random.randint(0, len(one_qubit_gates)))])
         yield gate(i, np.pi / 2.0)
-        yield gates.M(i)
 
 
 def PrepareGHZ(nqubits: int):
@@ -103,16 +102,16 @@ def ToffoliGate(nqubits: int, nlayers: int = 1):
 def CircuitFactory(nqubits: int,
                    circuit_type: str,
                    accelerators: Dict[str, int] = None,
-                   memory_device: str = "/CPU:0",
+                   device: str = "/CPU:0",
                    **kwargs):
     if circuit_type == "qft":
         circuit = models.QFT(nqubits, accelerators=accelerators,
-                             memory_device=memory_device)
+                             memory_device=device)
     else:
         if circuit_type not in _CIRCUITS:
             raise TypeError("Unknown benchmark circuit type {}."
                             "".format(circuit_type))
         circuit = models.Circuit(nqubits, accelerators=accelerators,
-                                 memory_device=memory_device)
+                                 memory_device=device)
         circuit.add(_CIRCUITS[circuit_type](nqubits, **kwargs))
     return circuit

examples/benchmarks/evolution.py

@@ -1,7 +1,6 @@
 """Adiabatic evolution for the Ising Hamiltonian using linear scaling."""
 import argparse
 import time
-import utils
 from qibo import callbacks, hamiltonians, models
 
 parser = argparse.ArgumentParser()
@@ -12,6 +11,62 @@
 parser.add_argument("--accelerators", default=None, type=str)
 
 
+def parse_nqubits(nqubits_str):
+    """Transforms a string that specifies number of qubits to list.
+
+    Supported string formats are the following:
+        * 'a-b' with a and b integers.
+            Then the returned list is range(a, b + 1).
+        * 'a,b,c,d,...' with a, b, c, d, ... integers.
+            Then the returned list is [a, b, c, d]
+    """
+    # TODO: Support usage of both `-` and `,` in the same string.
+    if "-" in nqubits_str:
+        if "," in nqubits_str:
+            raise ValueError("String that specifies qubits cannot contain "
+                             "both , and -.")
+
+        nqubits_split = nqubits_str.split("-")
+        if len(nqubits_split) != 2:
+            raise ValueError("Invalid string that specifies nqubits "
+                             "{}.".format(nqubits_str))
+
+        n_start, n_end = nqubits_split
+        return list(range(int(n_start), int(n_end) + 1))
+
+    return [int(x) for x in nqubits_str.split(",")]
+
+
+def parse_accelerators(accelerators):
+    """Transforms string that specifies accelerators to dictionary.
+
+    The string that is parsed has the following format:
+        n1device1,n2device2,n3device3,...
+    and is transformed to the dictionary:
+        {'device1': n1, 'device2': n2, 'device3': n3, ...}
+
+    Example:
+        2/GPU:0,2/GPU:1 --> {'/GPU:0': 2, '/GPU:1': 2}
+    """
+    if accelerators is None:
+        return None
+
+    def read_digit(x):
+        i = 0
+        while x[i].isdigit():
+            i += 1
+        return x[i:], int(x[:i])
+
+    acc_dict = {}
+    for entry in accelerators.split(","):
+        device, n = read_digit(entry)
+        if device in acc_dict:
+            acc_dict[device] += n
+        else:
+            acc_dict[device] = n
+    return acc_dict
+
+
 def main(nqubits_list, dt, solver, trotter=False, accelerators=None):
     """Performs adiabatic evolution with critical TFIM as the "hard" Hamiltonian."""
     if accelerators is not None:
@@ -45,6 +100,6 @@ def main(nqubits_list, dt, solver, trotter=False, accelerators=None):
 
 if __name__ == "__main__":
     args = vars(parser.parse_args())
-    args["nqubits_list"] = utils.parse_nqubits(args.pop("nqubits"))
-    args["accelerators"] = utils.parse_accelerators(args.pop("accelerators"))
+    args["nqubits_list"] = parse_nqubits(args.pop("nqubits"))
+    args["accelerators"] = parse_accelerators(args.pop("accelerators"))
     main(**args)

examples/benchmarks/example.sh

@@ -0,0 +1,32 @@
+# example bash file for executing `main.py` benchmarks
+# device configurations and `nqubits` are based on DGX machine with
+# 4x V100 32GB GPUs
+FILENAME="dgx.dat" # log file name
+export CUDA_VISIBLE_DEVICES="0"
+for NQUBITS in 20 21 22 23 24 25 26 27 28 29 30
+do
+  python main.py --type qft --nqubits $NQUBITS --filename $FILENAME
+  python main.py --type qft --nqubits $NQUBITS --filename $FILENAME --precision "single"
+  python main.py --type variational --nqubits $NQUBITS --filename $FILENAME
+  python main.py --type variational --nqubits $NQUBITS --filename $FILENAME --precision "single"
+done
+python main.py --type qft --nqubits 31 --filename $FILENAME --precision "single"
+python main.py --type variational --nqubits 31 --filename $FILENAME --precision "single"
+export CUDA_VISIBLE_DEVICES=""
+for NQUBITS in 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34
+do
+  python main.py --type qft --nqubits $NQUBITS --filename $FILENAME
+  python main.py --type variational --nqubits $NQUBITS --filename $FILENAME
+done
+
+export CUDA_VISIBLE_DEVICES="0,1"
+ACCEL="1/GPU:0,1/GPU:1"
+python main.py --type qft --nqubits 31 --filename $FILENAME --device "/CPU:0" --accelerators $ACCEL
+python main.py --type qft --nqubits 32 --filename $FILENAME --device "/CPU:0" --accelerators $ACCEL --precision "single"
+export CUDA_VISIBLE_DEVICES="0,1,2,3"
+ACCEL="1/GPU:0,1/GPU:1,1/GPU:2,1/GPU:3"
+python main.py --type qft --nqubits 32 --filename $FILENAME --device "/CPU:0" --accelerators $ACCEL
+python main.py --type qft --nqubits 33 --filename $FILENAME --device "/CPU:0" --accelerators $ACCEL --precision "single"
+ACCEL="2/GPU:0,2/GPU:1,2/GPU:2,2/GPU:3"
+python main.py --type qft --nqubits 33 --filename $FILENAME --device "/CPU:0" --accelerators $ACCEL
+python main.py --type qft --nqubits 34 --filename $FILENAME --device "/CPU:0" --accelerators $ACCEL --precision "single"