Use pixel masks from XML file for Loki

docs/examples/beam-center-finder.ipynb

@@ -51,7 +51,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "params = {isis.Filename[SampleRun]: 'SANS2D00063114.nxs'}\n",
+    "params = {Filename[SampleRun]: 'SANS2D00063114.nxs'}\n",
     "\n",
     "# Build the pipeline\n",
     "providers = list(\n",
@@ -246,7 +246,7 @@
     "params[WavelengthBins] = sc.linspace(\n",
     "    'wavelength', start=2.0, stop=16.0, num=141, unit='angstrom'\n",
     ")\n",
-    "params[isis.Filename[EmptyBeamRun]] = 'SANS2D00063091.nxs'\n",
+    "params[Filename[EmptyBeamRun]] = 'SANS2D00063091.nxs'\n",
     "\n",
     "params[NeXusMonitorName[Incident]] = 'monitor2'\n",
     "params[NeXusMonitorName[Transmission]] = 'monitor4'\n",
@@ -570,7 +570,7 @@
     "\n",
     "*Note*\n",
     "\n",
-    "The result obtained just above (`x=0.09170493, y=-0.08166288`) is slightly different from the one obtained using the pipeline (`x=0.09266183, y=-0.08203386`).\n",
+    "The result obtained just above is slightly different from the one obtained earlier [using the pipeline](#Method-2:-computing-I(Q)-inside-4-quadrants).\n",
     "\n",
     "This is because in our example, we used `x=0, y=0` as our initial guess,\n",
     "while the pipeline uses an additional optimization where it first computes a better initial guess using method 1 (center-of-mass).\n",

docs/examples/loki-direct-beam.ipynb

@@ -44,6 +44,7 @@
     "import scippneutron as scn\n",
     "import plopp as pp\n",
     "import esssans as sans\n",
+    "from esssans import isis\n",
     "from esssans.types import *\n",
     "from esssans.direct_beam import direct_beam"
    ]
@@ -67,12 +68,12 @@
    "source": [
     "params = sans.loki.default_parameters.copy()\n",
     "\n",
-    "# List of files\n",
-    "params[FileList[SampleRun]] = ['60339-2022-02-28_2215.nxs']\n",
-    "params[FileList[BackgroundRun]] = ['60393-2022-02-28_2215.nxs']\n",
-    "params[FileList[TransmissionRun[SampleRun]]] = ['60394-2022-02-28_2215.nxs']\n",
-    "params[FileList[TransmissionRun[BackgroundRun]]] = ['60392-2022-02-28_2215.nxs']\n",
-    "params[FileList[EmptyBeamRun]] = ['60392-2022-02-28_2215.nxs']\n",
+    "# File names\n",
+    "params[Filename[SampleRun]] = '60339-2022-02-28_2215.nxs'\n",
+    "params[Filename[BackgroundRun]] = '60393-2022-02-28_2215.nxs'\n",
+    "params[Filename[TransmissionRun[SampleRun]]] = '60394-2022-02-28_2215.nxs'\n",
+    "params[Filename[TransmissionRun[BackgroundRun]]] = '60392-2022-02-28_2215.nxs'\n",
+    "params[Filename[EmptyBeamRun]] = '60392-2022-02-28_2215.nxs'\n",
     "\n",
     "# Wavelength binning parameters\n",
     "wavelength_min = sc.scalar(1.0, unit='angstrom')\n",
@@ -87,8 +88,8 @@
     "    'wavelength', wavelength_min, wavelength_max, n_wavelength_bands + 1\n",
     ")\n",
     "\n",
-    "params[BeamStopPosition] = sc.vector([-0.026, -0.022, 0.0], unit='m')\n",
-    "params[BeamStopRadius] = sc.scalar(0.042, unit='m')\n",
+    "# Masking\n",
+    "masks = ['mask_new_July2022.xml']\n",
     "\n",
     "params[CorrectForGravity] = True\n",
     "params[UncertaintyBroadcastMode] = UncertaintyBroadcastMode.upper_bound\n",
@@ -117,9 +118,16 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "providers = sans.providers + sans.loki.providers\n",
+    "providers = (\n",
+    "    sans.providers + sans.loki.providers + (sans.isis.io.read_xml_detector_masking,)\n",
+    ")\n",
+    "\n",
+    "pipeline = sciline.Pipeline(providers, params=params)\n",
+    "pipeline.set_param_series(PixelMaskFilename, masks)\n",
     "\n",
-    "pipeline = sciline.Pipeline(providers, params=params)"
+    "# Add providers that fetch data from online resource\n",
+    "for provider in sans.loki.data.providers:\n",
+    "    pipeline.insert(provider)"
    ]
   },
   {
@@ -254,7 +262,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "pp.plot({'reference': Iq_theory, 'data': iofq_full}, norm='log')"
+    "pp.plot(\n",
+    "    {'reference': Iq_theory, 'data': iofq_full},\n",
+    "    color={'reference': 'darkgrey', 'data': 'C0'},\n",
+    "    norm='log',\n",
+    ")"
    ]
   },
   {
@@ -301,11 +313,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "results = direct_beam(pipeline=pipeline, I0=I0, niter=6)\n",
+    "results_layers = direct_beam(pipeline=pipeline, I0=I0, niter=6)\n",
     "# Unpack the final result\n",
-    "iofq_full = results[-1]['iofq_full']\n",
-    "iofq_bands = results[-1]['iofq_bands']\n",
-    "direct_beam_function = results[-1]['direct_beam']"
+    "iofq_full_layers = results_layers[-1]['iofq_full']\n",
+    "iofq_bands_layers = results_layers[-1]['iofq_bands']\n",
+    "direct_beam_function_layers = results_layers[-1]['direct_beam']"
    ]
   },
   {
@@ -326,8 +338,8 @@
     "plots = [\n",
     "    pp.plot(\n",
     "        {\n",
-    "            **sc.collapse(iofq_bands['layer', i], keep='Q'),\n",
-    "            **{'full': iofq_full['layer', i]},\n",
+    "            **sc.collapse(iofq_bands_layers['layer', i], keep='Q'),\n",
+    "            **{'full': iofq_full_layers['layer', i]},\n",
     "        },\n",
     "        norm='log',\n",
     "        color={'full': 'k'},\n",
@@ -355,7 +367,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "pp.plot(sc.collapse(direct_beam_function, keep='wavelength'))"
+    "pp.plot(sc.collapse(direct_beam_function_layers, keep='wavelength'))"
    ]
   },
   {
@@ -373,7 +385,142 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "pp.plot({**{'reference': Iq_theory}, **sc.collapse(iofq_full, keep='Q')}, norm='log')"
+    "layers = sc.collapse(iofq_full_layers, keep='Q')\n",
+    "pp.plot(\n",
+    "    {**{'reference': Iq_theory}, **layers},\n",
+    "    color={\n",
+    "        **{'reference': 'darkgrey'},\n",
+    "        **{key: f'C{i}' for i, key in enumerate(layers)},\n",
+    "    },\n",
+    "    norm='log',\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "7cbff973-fbb7-4187-82e8-8e4fa82efcb8",
+   "metadata": {},
+   "source": [
+    "## Combining multiple runs to boost signal\n",
+    "\n",
+    "It is common practise to combine the events from multiple runs to improve the statistics on the computed $I(Q)$,\n",
+    "and thus obtain a more robust direct beam function.\n",
+    "\n",
+    "To achieve this, we need to replace the `SampleRun` and `BackgroundRun` file names with parameter series.\n",
+    "We then need to supply additional providers which will merge the events from the runs appropriately\n",
+    "(note that these providers will merge both the detector and the monitor events).\n",
+    "\n",
+    "We first define a list of file names for the sample and background runs (two files for each):"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bf5f23b0-e4b6-48d8-b98a-99ec5e56be4e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "del params[Filename[SampleRun]]\n",
+    "del params[Filename[BackgroundRun]]\n",
+    "\n",
+    "sample_runs = ['60250-2022-02-28_2215.nxs', '60339-2022-02-28_2215.nxs']\n",
+    "background_runs = ['60248-2022-02-28_2215.nxs', '60393-2022-02-28_2215.nxs']"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "c3e2ff3d-2b3d-4ba7-b4ca-f0a5617d22dc",
+   "metadata": {},
+   "source": [
+    "We now construct a new pipeline, inserting parameter series and merging providers:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "2a39bd50-5d63-4183-9afe-6d990548bc11",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Same as original pipeline, but without sample and background run file names\n",
+    "pipeline = sciline.Pipeline(providers, params=params)\n",
+    "for provider in sans.loki.data.providers:\n",
+    "    pipeline.insert(provider)\n",
+    "pipeline.set_param_series(PixelMaskFilename, masks)\n",
+    "\n",
+    "# Set parameter series for file names\n",
+    "pipeline.set_param_series(Filename[SampleRun], sample_runs)\n",
+    "pipeline.set_param_series(Filename[BackgroundRun], background_runs)\n",
+    "# Add event merging providers\n",
+    "pipeline.insert(sans.loki.io.merge_sample_runs)\n",
+    "pipeline.insert(sans.loki.io.merge_background_runs)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "6a50ef4f-a15c-4ae4-a8e7-666483a93da0",
+   "metadata": {},
+   "source": [
+    "If we now visualize the initial section of the pipeline,\n",
+    "we can see that the series of runs are merged into a single `LoadedFileContents`:"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "4ab8426a-d7b9-4ffe-aa9b-e029b5888ae0",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pipeline.visualize(\n",
+    "    LoadedFileContents[SampleRun], compact=True, graph_attr={'rankdir': 'LR'}\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "1c674237-c85e-4e1f-b136-1d9f53494572",
+   "metadata": {},
+   "source": [
+    "We run the direct beam iterations again and compare with our original results."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0c132101-9dfc-4896-bb68-56a0878bfcef",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "results = direct_beam(pipeline=pipeline, I0=I0, niter=6)\n",
+    "# Unpack the final result\n",
+    "iofq_full_new = results[-1]['iofq_full']\n",
+    "iofq_bands_new = results[-1]['iofq_bands']\n",
+    "direct_beam_function_new = results[-1]['direct_beam']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c731cf0c-4451-445f-a8bf-46a952ab171f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pp.plot({'one run': direct_beam_function, 'two runs': direct_beam_function_new})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "50f78446-b7e4-406a-a5fd-5548e18dedc6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "pp.plot(\n",
+    "    {'reference': Iq_theory, 'one run': iofq_full, 'two runs': iofq_full_new},\n",
+    "    color={'reference': 'darkgrey', 'one run': 'C0', 'two runs': 'C1'},\n",
+    "    norm='log',\n",
+    ")"
    ]
   }
  ],
@@ -392,8 +539,7 @@
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.10.12"
+   "pygments_lexer": "ipython3"
   }
  },
  "nbformat": 4,

docs/examples/sans2d.ipynb

@@ -50,9 +50,9 @@
     "params = {}\n",
     "\n",
     "params[DirectBeamFilename] = 'DIRECT_SANS2D_REAR_34327_4m_8mm_16Feb16.dat'\n",
-    "params[sans.isis.Filename[SampleRun]] = 'SANS2D00063114.nxs'\n",
-    "params[sans.isis.Filename[BackgroundRun]] = 'SANS2D00063159.nxs'\n",
-    "params[sans.isis.Filename[EmptyBeamRun]] = 'SANS2D00063091.nxs'\n",
+    "params[Filename[SampleRun]] = 'SANS2D00063114.nxs'\n",
+    "params[Filename[BackgroundRun]] = 'SANS2D00063159.nxs'\n",
+    "params[Filename[EmptyBeamRun]] = 'SANS2D00063091.nxs'\n",
     "params[OutFilename] = 'reduced.nxs'\n",
     "\n",
     "params[NeXusMonitorName[Incident]] = 'monitor2'\n",
@@ -271,7 +271,11 @@
     "\n",
     "display(sc.plot({str(key): results[key] for key in monitors}, norm='log'))\n",
     "\n",
-    "display(plot_flat_detector_xy(results[MaskedData[SampleRun]].hist(), norm='log'))\n",
+    "display(\n",
+    "    plot_flat_detector_xy(\n",
+    "        results[MaskedData[SampleRun]]['spectrum', :61440].hist(), norm='log'\n",
+    "    )\n",
+    ")\n",
     "\n",
     "wavelength = pipeline.compute(WavelengthBins)\n",
     "display(\n",

docs/examples/zoom.ipynb

@@ -67,8 +67,8 @@
     "params = {\n",
     "    sans.types.DirectBeamFilename: 'Direct_Zoom_4m_8mm_100522.txt',\n",
     "    sans.isis.CalibrationFilename: '192tubeCalibration_11-02-2019_r5_10lines.nxs',\n",
-    "    sans.isis.Filename[sans.types.SampleRun]: 'ZOOM00034786.nxs',\n",
-    "    sans.isis.Filename[sans.types.EmptyBeamRun]: 'ZOOM00034787.nxs',\n",
+    "    Filename[sans.types.SampleRun]: 'ZOOM00034786.nxs',\n",
+    "    Filename[sans.types.EmptyBeamRun]: 'ZOOM00034787.nxs',\n",
     "    sans.isis.SampleOffset: sc.vector([0.0, 0.0, 0.11], unit='m'),\n",
     "    sans.isis.DetectorBankOffset: sc.vector([0.0, 0.0, 0.5], unit='m'),\n",
     "}\n",
@@ -81,7 +81,7 @@
     "    'tube_1120_bottom.xml',\n",
     "    'tubes_beg_18_2.xml',\n",
     "]\n",
-    "masks = sciline.ParamTable(str, {sans.isis.PixelMaskFilename: masks}, index=masks)"
+    "masks = sciline.ParamTable(str, {PixelMaskFilename: masks}, index=masks)"
    ]
   },
   {

requirements/base.in

@@ -9,7 +9,7 @@ dask
 graphviz
 plopp
 pythreejs
-sciline>=23.9.1
+sciline>=24.2.0
 scipp>=23.8.0
 scippneutron>=23.9.0
 scippnexus>=23.12.1

requirements/base.txt

@@ -1,9 +1,8 @@
-# SHA1:d9131f52f58d87889ced7f09fded3a470b1c0a71
 #
-# This file is autogenerated by pip-compile-multi
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.10
+# by the following command:
 #
-#    pip-compile-multi
+#    pip-compile base.in
 #
 asttokens==2.4.1
     # via stack-data
@@ -118,7 +117,7 @@ pyyaml==6.0.1
     # via dask
 requests==2.31.0
     # via pooch
-sciline==24.1.1
+sciline==24.2.1
     # via -r base.in
 scipp==24.2.0
     # via

src/esssans/isis/__init__.py

@@ -4,25 +4,20 @@
 
 from . import components, data, general, io, masking, sans2d
 from .components import DetectorBankOffset, MonitorOffset, SampleOffset
-from .io import CalibrationFilename, DataFolder, Filename, PixelMaskFilename
-from .masking import PixelMask
+from .io import CalibrationFilename
 from .visualization import plot_flat_detector_xy
 
 providers = components.providers + general.providers + io.providers
 
 
 __all__ = [
     'CalibrationFilename',
-    'DataFolder',
     'DetectorBankOffset',
-    'Filename',
     'apply_component_user_offsets_to_raw_data',
     'data',
     'io',
     'masking',
     'MonitorOffset',
-    'PixelMask',
-    'PixelMaskFilename',
     'providers',
     'SampleOffset',
     'plot_flat_detector_xy',