From be46f3b992e574525d63b08501f2e0942d794908 Mon Sep 17 00:00:00 2001 From: David Stirling Date: Tue, 25 Jul 2023 10:43:17 +0100 Subject: [PATCH 01/32] Initial OMERO reader --- active_plugins/omeroreader.py | 804 ++++++++++++++++++++++++++++++++++ 1 file changed, 804 insertions(+) create mode 100644 active_plugins/omeroreader.py diff --git a/active_plugins/omeroreader.py b/active_plugins/omeroreader.py new file mode 100644 index 0000000..314731d --- /dev/null +++ b/active_plugins/omeroreader.py @@ -0,0 +1,804 @@ +""" +An image reader which connects to OMERO to load data + +# Installation - +This depends on platform. At the most basic level you'll need the `omero-py` package. For headless run and +more convenient server login you'll also want the `omero_user_token` package. + +Both should be possible to pip install on Windows. On MacOS, you'll probably have trouble with the zeroc-ice dependency. +omero-py uses an older version and so needs specific wheels. Fortunately we've built some for you. +Macos - https://github.com/glencoesoftware/zeroc-ice-py-macos-x86_64/releases/latest +Linux (Generic) - https://github.com/glencoesoftware/zeroc-ice-py-linux-x86_64/releases/latest +Ubuntu 22.04 - https://github.com/glencoesoftware/zeroc-ice-py-ubuntu2204-x86_64/releases/latest + +Download the .whl file from whichever is most appropriate and run `pip install `. + +From there pip install omero-py should do the rest. + +# Usage - +Like the old functionality from <=CP4, connection to OMERO is triggered through a login dialog within the GUI which +should appear automatically when needed. Enter your credentials and hit 'connect'. Once connected you should be able to +load OMERO data into the workspace. + +We've also made a "Connect to OMERO" menu option available in the new Plugins menu, in case you ever need to forcibly +open that window again (e.g. changing server). + +To get OMERO data into a pipeline, you can construct a file list in the special URL format `omero:iid=`. +E.g. "omero:iid=4345" + +Alternatively, direct URLs pointing to an image can be provided. +e.g. https://omero.mywebsite.com/webclient/?show=image-1234 +These can be obtained in OMERO-web by selecting an image and pressing the link button in the top right corner +of the right side panel. + +To get these into the CellProfiler GUI, there are a few options. Previously this was primarily achieved by using +*File->Import->File List* to load a text file containing one image per line. A LoadData CSV can also be used. +As of CP5 it is also now possible to copy and paste text (e.g. URLs) directly into the file list in the Images module. + +# Working with data - +Unlike previous iterations of this integration, the CP5 plugin has full support for channel and plane indexing. +The previous reader misbehaved in that it would only load the first channel from OMERO if greyscale is requested. +In this version all channels will be returned, so you must declare a colour image in NamesAndTypes when loading one. + +On the plus side, you can now use the 'Extract metadata' option in the Images module to split the C, Z and T axes +into individual planes. Remember to disable the "Filter to images only" option in the Images module, since URLs do +not pass this filter. + +Lastly, with regards to connections, you can only connect to a single server at a time. Opening the connect dialog and +dialling in will replace any existing connection which you had active. This iteration of the plugin will keep +server connections from timing out while CellProfiler is running, though you may need to reconnect if the PC +goes to sleep. +""" +import os +import collections +import atexit + +from struct import unpack + +from cellprofiler_core.preferences import get_headless + +import numpy + +from cellprofiler_core.constants.image import MD_SIZE_S, MD_SIZE_C, MD_SIZE_Z, MD_SIZE_T, \ + MD_SIZE_Y, MD_SIZE_X, MD_SERIES_NAME +from cellprofiler_core.preferences import get_omero_server, get_omero_port, get_omero_user, set_omero_server,\ + set_omero_port, set_omero_user, config_read_typed, config_write_typed +from cellprofiler_core.constants.image import PASSTHROUGH_SCHEMES +from cellprofiler_core.reader import Reader + +import omero +import logging +import re + +import importlib.util + +TOKEN_MODULE = importlib.util.find_spec("omero_user_token") +TOKENS_AVAILABLE = TOKEN_MODULE is not None +if TOKENS_AVAILABLE: + # Only load and enable user tokens if dependency is installed + omero_user_token = importlib.util.module_from_spec(TOKEN_MODULE) + TOKEN_MODULE.loader.exec_module(omero_user_token) + + +REGEX_INDEX_FROM_FILE_NAME = re.compile(r'\?show=image-(\d+)') + +PASSTHROUGH_SCHEMES.append('OMERO') + +SCALE_ONE_TYPE = ["float", "double"] +LOGGER = logging.getLogger(__name__) + + +PIXEL_TYPES = { + "int8": ['b', numpy.int8, (-128, 127)], + "uint8": ['B', numpy.uint8, (0, 255)], + "int16": ['h', numpy.int16, (-32768, 32767)], + "uint16": ['H', numpy.uint16, (0, 65535)], + "int32": ['i', numpy.int32, (-2147483648, 2147483647)], + "uint32": ['I', numpy.uint32, (0, 4294967295)], + "float": ['f', numpy.float32, (0, 1)], + "double": ['d', numpy.float64, (0, 1)] +} + + +class OMEROReader(Reader): + """ + Reads images from OMERO. + """ + reader_name = "OMERO Reader" + variable_revision_number = 1 + supported_filetypes = {} + supported_schemes = {'omero', 'http', 'https'} + + def __init__(self, image_file): + self.login = CREDENTIALS + self.image_id = None + self.omero_image = None + self.pixels = None + self.width = None + self.height = None + self.context = {'omero.group': '-1'} + super().__init__(image_file) + + def __del__(self): + self.close() + + def confirm_connection(self): + if self.login.client is None: + if get_headless(): + raise ValueError("No OMERO connection established") + else: + login(None) + if self.login.client is None: + raise ValueError("Connection failed") + + def init_reader(self): + # Check if session object already exists + self.confirm_connection() + if self.omero_image is not None: + return True + if self.file.scheme == "omero": + self.image_id = int(self.file.url[10:]) + else: + matches = REGEX_INDEX_FROM_FILE_NAME.findall(self.file.url) + if not matches: + raise ValueError("URL may not be from OMERO") + self.image_id = int(matches[0]) + + LOGGER.debug("Initializing OmeroReader for Image id: %s" % self.image_id) + # Get image object from the server + try: + self.omero_image = self.login.container_service.getImages( + "Image", [self.image_id], None, self.context)[0] + except: + message = "Image Id: %s not found on the server." % self.image_id + LOGGER.error(message, exc_info=True) + raise Exception(message) + self.pixels = self.omero_image.getPrimaryPixels() + self.width = self.pixels.getSizeX().val + self.height = self.pixels.getSizeY().val + return True + + def read(self, + series=None, + index=None, + c=None, + z=None, + t=None, + rescale=True, + xywh=None, + wants_max_intensity=False, + channel_names=None, + ): + """Read a single plane from the image file. + :param c: read from this channel. `None` = read color image if multichannel + or interleaved RGB. + :param z: z-stack index + :param t: time index + :param series: series for ``.flex`` and similar multi-stack formats + :param index: if `None`, fall back to ``zct``, otherwise load the indexed frame + :param rescale: `True` to rescale the intensity scale to 0 and 1; `False` to + return the raw values native to the file. + :param xywh: a (x, y, w, h) tuple + :param wants_max_intensity: if `False`, only return the image; if `True`, + return a tuple of image and max intensity + :param channel_names: provide the channel names for the OME metadata + """ + self.init_reader() + + debug_message = \ + "Reading C: %s, Z: %s, T: %s, series: %s, index: %s, " \ + "channel names: %s, rescale: %s, wants_max_intensity: %s, " \ + "XYWH: %s" % (c, z, t, series, index, channel_names, rescale, + wants_max_intensity, xywh) + if c is None and index is not None: + c = index + LOGGER.info(debug_message) + message = None + if (t or 0) >= self.pixels.getSizeT().val: + message = "T index %s exceeds sizeT %s" % \ + (t, self.pixels.getSizeT().val) + LOGGER.error(message) + if (c or 0) >= self.pixels.getSizeC().val: + message = "C index %s exceeds sizeC %s" % \ + (c, self.pixels.getSizeC().val) + LOGGER.error(message) + if (z or 0) >= self.pixels.getSizeZ().val: + message = "Z index %s exceeds sizeZ %s" % \ + (z, self.pixels.getSizeZ().val) + LOGGER.error(message) + if message is not None: + raise Exception("Couldn't retrieve a plane from OMERO image.") + tile = None + if xywh is not None: + assert isinstance(xywh, tuple) and len(xywh) == 4, \ + "Invalid XYWH tuple" + tile = xywh + numpy_image = self.read_planes(z, c, t, tile) + pixel_type = self.pixels.getPixelsType().value.val + min_value = PIXEL_TYPES[pixel_type][2][0] + max_value = PIXEL_TYPES[pixel_type][2][1] + LOGGER.debug("Pixel range [%s, %s]" % (min_value, max_value)) + if rescale or pixel_type == 'double': + LOGGER.info("Rescaling image using [%s, %s]" % (min_value, max_value)) + # Note: The result here differs from: + # https://github.com/emilroz/python-bioformats/blob/a60b5c5a5ae018510dd8aa32d53c35083956ae74/bioformats/formatreader.py#L903 + # Reason: the unsigned types are being properly taken into account + # and converted to [0, 1] using their full scale. + # Further note: float64 should be used for the numpy array in case + # image is stored as 'double', we're keeping it float32 to stay + # consistent with the CellProfiler reader (the double type is also + # converted to single precision) + numpy_image = \ + (numpy_image.astype(numpy.float32) + float(min_value)) / \ + (float(max_value) - float(min_value)) + if wants_max_intensity: + return numpy_image, max_value + return numpy_image + + def read_volume(self, + series=None, + c=None, + z=None, + t=None, + rescale=True, + xywh=None, + wants_max_intensity=False, + channel_names=None, + ): + self.init_reader() + debug_message = \ + "Reading C: %s, Z: %s, T: %s, series: %s, index: %s, " \ + "channel names: %s, rescale: %s, wants_max_intensity: %s, " \ + "XYWH: %s" % (c, z, t, series, index, channel_names, rescale, + wants_max_intensity, xywh) + if c is None and index is not None: + c = index + LOGGER.info(debug_message) + message = None + if (t or 0) >= self.pixels.getSizeT().val: + message = "T index %s exceeds sizeT %s" % \ + (t, self.pixels.getSizeT().val) + LOGGER.error(message) + if (c or 0) >= self.pixels.getSizeC().val: + message = "C index %s exceeds sizeC %s" % \ + (c, self.pixels.getSizeC().val) + LOGGER.error(message) + if (z or 0) >= self.pixels.getSizeZ().val: + message = "Z index %s exceeds sizeZ %s" % \ + (z, self.pixels.getSizeZ().val) + LOGGER.error(message) + if message is not None: + raise Exception("Couldn't retrieve a plane from OMERO image.") + tile = None + if xywh is not None: + assert isinstance(xywh, tuple) and len(xywh) == 4, \ + "Invalid XYWH tuple" + tile = xywh + numpy_image = self.read_planes_volumetric(z, c, t, tile) + pixel_type = self.pixels.getPixelsType().value.val + min_value = PIXEL_TYPES[pixel_type][2][0] + max_value = PIXEL_TYPES[pixel_type][2][1] + LOGGER.debug("Pixel range [%s, %s]" % (min_value, max_value)) + if rescale or pixel_type == 'double': + LOGGER.info("Rescaling image using [%s, %s]" % (min_value, max_value)) + # Note: The result here differs from: + # https://github.com/emilroz/python-bioformats/blob/a60b5c5a5ae018510dd8aa32d53c35083956ae74/bioformats/formatreader.py#L903 + # Reason: the unsigned types are being properly taken into account + # and converted to [0, 1] using their full scale. + # Further note: float64 should be used for the numpy array in case + # image is stored as 'double', we're keeping it float32 to stay + # consitent with the CellProfiler reader (the double type is also + # converted to single precision) + numpy_image = \ + (numpy_image.astype(numpy.float32) + float(min_value)) / \ + (float(max_value) - float(min_value)) + if wants_max_intensity: + return numpy_image, max_value + return numpy_image + + def read_planes(self, z=0, c=None, t=0, tile=None): + ''' + Creates RawPixelsStore and reads planes from the OMERO server. + ''' + channels = [] + if c is None: + channels = range(self.pixels.getSizeC().val) + else: + channels.append(c) + pixel_type = self.pixels.getPixelsType().value.val + numpy_type = PIXEL_TYPES[pixel_type][1] + raw_pixels_store = self.login.session.createRawPixelsStore() + try: + raw_pixels_store.setPixelsId( + self.pixels.getId().val, True, self.context) + LOGGER.debug("Reading pixels Id: %s" % self.pixels.getId().val) + LOGGER.debug("Reading channels %s" % channels) + planes = [] + for channel in channels: + if tile is None: + sizeX = self.width + sizeY = self.height + raw_plane = raw_pixels_store.getPlane( + z, channel, t, self.context) + else: + x, y, sizeX, sizeY = tile + raw_plane = raw_pixels_store.getTile( + z, channel, t, x, y, sizeX, sizeY) + convert_type = '>%d%s' % ( + (sizeY * sizeX), PIXEL_TYPES[pixel_type][0]) + converted_plane = unpack(convert_type, raw_plane) + plane = numpy.array(converted_plane, numpy_type) + plane.resize(sizeY, sizeX) + planes.append(plane) + if c is None: + return numpy.dstack(planes) + else: + return planes[0] + except Exception: + LOGGER.error("Failed to get plane from OMERO", exc_info=True) + finally: + raw_pixels_store.close() + + def read_planes_volumetric(self, z=None, c=None, t=None, tile=None): + ''' + Creates RawPixelsStore and reads planes from the OMERO server. + ''' + if t is not None and z is not None: + raise ValueError(f"Specified parameters {z=}, {t=} would not produce a 3D image") + if z is None: + size_z = self.pixels.getSizeZ().val + else: + size_z = 1 + if t is None: + size_t = self.pixels.getSizeT().val + else: + size_t = 1 + pixel_type = self.pixels.getPixelsType().value.val + numpy_type = PIXEL_TYPES[pixel_type][1] + raw_pixels_store = self.login.session.createRawPixelsStore() + if size_z > 1: + # We assume z is the desired 3D dimension if present and not specified. + t_range = [t or 0] + z_range = range(size_z) + elif size_t > 1: + t_range = range(size_t) + z_range = [z or 0] + else: + # Weird, but perhaps user's 3D image only had 1 plane in this acquisition. + t_range = [t or 0] + z_range = [z or 0] + planes = [] + try: + raw_pixels_store.setPixelsId( + self.pixels.getId().val, True, self.context) + LOGGER.debug("Reading pixels Id: %s" % self.pixels.getId().val) + + for z_index in z_range: + for t_index in t_range: + if tile is None: + size_x = self.width + size_y = self.height + raw_plane = raw_pixels_store.getPlane( + z_index, c, t_index, self.context) + else: + x, y, size_x, size_y = tile + raw_plane = raw_pixels_store.getTile( + z_index, c, t_index, x, y, size_x, size_y) + convert_type = '>%d%s' % ( + (size_y * size_x), PIXEL_TYPES[pixel_type][0]) + converted_plane = unpack(convert_type, raw_plane) + plane = numpy.array(converted_plane, numpy_type) + plane.resize(size_y, size_x) + planes.append(plane) + return numpy.dstack(planes) + except Exception: + LOGGER.error("Failed to get plane from OMERO", exc_info=True) + finally: + raw_pixels_store.close() + + @classmethod + def supports_url(cls): + return True + + @classmethod + def supports_format(cls, image_file, allow_open=False, volume=False): + """This function needs to evaluate whether a given ImageFile object + can be read by this reader class. + + Return value should be an integer representing suitability: + -1 - 'I can't read this at all' + 1 - 'I am the one true reader for this format, don't even bother checking any others' + 2 - 'I am well-suited to this format' + 3 - 'I can read this format, but I might not be the best', + 4 - 'I can give it a go, if you must' + + The allow_open parameter dictates whether the reader is permitted to read the file when + making this decision. If False the decision should be made using file extension only. + Any opened files should be closed before returning. + + The volume parameter specifies whether the reader will need to return a 3D array. + .""" + if image_file.scheme not in cls.supported_schemes: + return -1 + if image_file.scheme == "omero": + return 1 + elif "?show=image" in image_file.url.lower(): + return 2 + return -1 + + def close(self): + # If your reader opens a file, this needs to release any active lock, + pass + + def get_series_metadata(self): + """ + OMERO image IDs only ever refer to a single series + + Should return a dictionary with the following keys: + Key names are in cellprofiler_core.constants.image + MD_SIZE_S - int reflecting the number of series + MD_SIZE_X - list of X dimension sizes, one element per series. + MD_SIZE_Y - list of Y dimension sizes, one element per series. + MD_SIZE_Z - list of Z dimension sizes, one element per series. + MD_SIZE_C - list of C dimension sizes, one element per series. + MD_SIZE_T - list of T dimension sizes, one element per series. + MD_SERIES_NAME - list of series names, one element per series. + """ + self.init_reader() + LOGGER.info(f"Extracting metadata for image {self.image_id}") + meta_dict = collections.defaultdict(list) + meta_dict[MD_SIZE_S] = 1 + meta_dict[MD_SIZE_X].append(self.width) + meta_dict[MD_SIZE_Y].append(self.height) + meta_dict[MD_SIZE_C].append(self.pixels.getSizeC().val) + meta_dict[MD_SIZE_Z].append(self.pixels.getSizeZ().val) + meta_dict[MD_SIZE_T].append(self.pixels.getSizeT().val) + meta_dict[MD_SERIES_NAME].append(self.omero_image.getName().val) + return meta_dict + + @staticmethod + def get_settings(): + return [ + ('allow_token', + "Allow OMERO user tokens", + """ + If enabled, this reader will attempt to use OMERO user tokens to + establish a server connection. + """, + bool, + True), + ('show_server', + "Display 'server connected' popup", + """ + If enabled, a popup will be shown when a server is automatically connected to using a user token. + """, + bool, + True) + ] + + +def get_display_server(): + return config_read_typed(f"Reader.{OMEROReader.reader_name}.show_server", bool) + + +def set_display_server(val): + config_write_typed(key, value, key_type=bool) + + +class LoginHelper: + def __init__(self): + self.server = get_omero_server() + self.port = get_omero_port() + self.username = get_omero_user() + self.passwd = "" + self.session_key = None + self.session = None + self.client = None + self.container_service = None + self.tokens = {} + atexit.register(self.shutdown) + + def get_tokens(self, path=None): + self.tokens.clear() + # Future versions of omero_user_token may support multiple tokens, so we code with that in mind. + if not TOKENS_AVAILABLE: + return + # User tokens sadly default to the home directory. This would override that location. + py_home = os.environ['HOME'] + if path is not None: + os.environ['HOME'] = path + try: + LOGGER.info("Requesting token info") + token = omero_user_token.getter() + server, port = token[token.find('@') + 1:].split(':') + port = int(port) + LOGGER.info("Connection to {}:{}".format(server, port)) + session_key = token[:token.find('@')] + self.tokens[server] = (server, port, session_key) + except Exception: + LOGGER.error("Failed to get user token", exc_info=True) + if path is not None: + os.environ['HOME'] = py_home + + def try_token(self, address): + if address not in self.tokens: + LOGGER.error(f"Token {address} not found") + return False + else: + server, port, session_key = self.tokens[address] + return self.login(server=server, port=port, session_key=session_key) + + def login(self, server=None, port=None, user=None, passwd=None, session_key=None): + self.client = omero.client(host=server, port=port) + if session_key is not None: + try: + self.session = self.client.joinSession(session_key) + self.client.enableKeepAlive(60) + self.session.detachOnDestroy() + self.server = server + self.port = port + self.session_key = session_key + except Exception as e: + print(f"Failed to join session, token may have expired: {e}") + self.client = None + self.session = None + return False + elif self.username is not None: + try: + self.session = self.client.createSession( + username=user, password=passwd) + self.client.enableKeepAlive(60) + self.session.detachOnDestroy() + self.server = server + self.port = port + self.username = user + self.passwd = passwd + except Exception as e: + print(f"Failed to create session: {e}") + self.client = None + self.session = None + return False + else: + self.client = None + self.session = None + raise Exception( + "Not enough details to create a server connection.") + self.container_service = self.session.getContainerService() + return True + + + def shutdown(self): + if self.client is not None: + try: + client.closeSession() + except Exception as e: + print("Failed to close OMERO session - ", e) + + +CREDENTIALS = LoginHelper() + +if not get_headless(): + # We can only construct wx widgets if we're not in headless mode + import wx + import cellprofiler.gui.plugins_menu + + + def show_login_dlg(token=True): + app = wx.GetApp() + frame = app.GetTopWindow() + with OmeroLoginDlg(frame, title="Log into Omero", token=token) as dlg: + dlg.ShowModal() + + def login(e): + CREDENTIALS.get_tokens() + if CREDENTIALS.tokens: + connected = CREDENTIALS.try_token(list(CREDENTIALS.tokens.keys())[0]) + if get_headless(): + if connected: + print("Connected to ", CREDENTIALS.server) + else: + print("Failed to connect, was user token invalid?") + return connected + elif connected: + from cellprofiler.gui.errordialog import show_warning + cellprofiler.gui.errordialog.show_warning("Connected to OMERO", + f"A token was found and used to " + f"connect to the OMERO server at {CREDENTIALS.server}", + get_display_server, + set_display_server) + return + show_login_dlg() + + def login_no_token(e): + show_login_dlg() + + # def browse(e): + # if CREDENTIALS.client is None: + # show_login_dlg() + # browse_images() + + + cellprofiler.gui.plugins_menu.PLUGIN_MENU_ENTRIES.extend([ + (login, wx.NewId(), "Connect to OMERO", "Establish an OMERO connection"), + (login_no_token, wx.NewId(), "Connect to OMERO (no token)", "Establish an OMERO connection," + " but without using user tokens"), + # (browse, wx.NewID(), "Browse OMERO for images", "Browse an OMERO server and add images to the pipeline") + ]) + + class OmeroLoginDlg(wx.Dialog): + + def __init__(self, *args, token=True, **kwargs): + super(self.__class__, self).__init__(*args, **kwargs) + self.credentials = CREDENTIALS + self.token = token + self.SetSizer(wx.BoxSizer(wx.VERTICAL)) + sizer = wx.BoxSizer(wx.VERTICAL) + self.Sizer.Add(sizer, 1, wx.EXPAND | wx.ALL, 6) + sub_sizer = wx.BoxSizer(wx.HORIZONTAL) + sizer.Add(sub_sizer, 0, wx.EXPAND) + + max_width = 0 + max_height = 0 + for label in ("Server:", "Port:", "Username:", "Password:"): + w, h, _, _ = self.GetFullTextExtent(label) + max_width = max(w, max_width) + max_height = max(h, max_height) + + # Add extra padding + lsize = wx.Size(max_width + 5, max_height) + sub_sizer.Add( + wx.StaticText(self, label="Server:", size=lsize), + 0, wx.ALIGN_CENTER_VERTICAL) + self.omero_server_ctrl = wx.TextCtrl(self, value=self.credentials.server) + sub_sizer.Add(self.omero_server_ctrl, 1, wx.EXPAND) + + sizer.AddSpacer(5) + sub_sizer = wx.BoxSizer(wx.HORIZONTAL) + sizer.Add(sub_sizer, 0, wx.EXPAND) + sub_sizer.Add( + wx.StaticText(self, label="Port:", size=lsize), + 0, wx.ALIGN_CENTER_VERTICAL) + self.omero_port_ctrl = wx.lib.intctrl.IntCtrl(self, value=self.credentials.port) + sub_sizer.Add(self.omero_port_ctrl, 1, wx.EXPAND) + + sizer.AddSpacer(5) + sub_sizer = wx.BoxSizer(wx.HORIZONTAL) + sizer.Add(sub_sizer, 0, wx.EXPAND) + sub_sizer.Add( + wx.StaticText(self, label="User:", size=lsize), + 0, wx.ALIGN_CENTER_VERTICAL) + self.omero_user_ctrl = wx.TextCtrl(self, value=self.credentials.username) + sub_sizer.Add(self.omero_user_ctrl, 1, wx.EXPAND) + + sizer.AddSpacer(5) + sub_sizer = wx.BoxSizer(wx.HORIZONTAL) + sizer.Add(sub_sizer, 0, wx.EXPAND) + sub_sizer.Add( + wx.StaticText(self, label="Password:", size=lsize), + 0, wx.ALIGN_CENTER_VERTICAL) + self.omero_password_ctrl = wx.TextCtrl(self, value="", style=wx.TE_PASSWORD|wx.TE_PROCESS_ENTER) + self.omero_password_ctrl.Bind(wx.EVT_TEXT_ENTER, self.on_connect_pressed) + sub_sizer.Add(self.omero_password_ctrl, 1, wx.EXPAND) + + sizer.AddSpacer(5) + sub_sizer = wx.BoxSizer(wx.HORIZONTAL) + sizer.Add(sub_sizer, 0, wx.EXPAND) + connect_button = wx.Button(self, label="Connect") + connect_button.Bind(wx.EVT_BUTTON, self.on_connect_pressed) + sub_sizer.Add(connect_button, 0, wx.EXPAND) + sub_sizer.AddSpacer(5) + + self.message_ctrl = wx.StaticText(self, label="Not connected") + sub_sizer.Add(self.message_ctrl, 1, wx.EXPAND) + + self.token_button = wx.Button(self, label="Set Token") + self.token_button.Bind(wx.EVT_BUTTON, self.on_set_pressed) + self.token_button.Disable() + self.token_button.SetToolTip("Use these credentials to set a long-lasting token for automatic login") + sub_sizer.Add(self.token_button, 0, wx.EXPAND) + sub_sizer.AddSpacer(5) + + + button_sizer = wx.StdDialogButtonSizer() + self.Sizer.Add(button_sizer, 0, wx.EXPAND) + + cancel_button = wx.Button(self, wx.ID_CANCEL) + button_sizer.AddButton(cancel_button) + cancel_button.Bind(wx.EVT_BUTTON, self.on_cancel) + + self.ok_button = wx.Button(self, wx.ID_OK) + button_sizer.AddButton(self.ok_button) + self.ok_button.Bind(wx.EVT_BUTTON, self.on_ok) + self.ok_button.Enable(False) + button_sizer.Realize() + + self.omero_password_ctrl.Bind(wx.EVT_TEXT, self.mark_dirty) + self.omero_port_ctrl.Bind(wx.EVT_TEXT, self.mark_dirty) + self.omero_server_ctrl.Bind(wx.EVT_TEXT, self.mark_dirty) + self.omero_user_ctrl.Bind(wx.EVT_TEXT, self.mark_dirty) + self.Layout() + + def mark_dirty(self, event): + if self.ok_button.IsEnabled(): + self.ok_button.Enable(False) + self.message_ctrl.Label = "Please connect with your new credentials" + self.message_ctrl.ForegroundColour = "black" + + def on_connect_pressed(self, event): + self.connect() + + def on_set_pressed(self, event): + if self.credentials.client is None or not TOKENS_AVAILABLE: + return + token_path = omero_user_token.assert_and_get_token_path() + if os.path.exists(token_path): + dlg2 = wx.MessageDialog(self, + "Existing omero_user_token will be overwritten. Proceed?", + "Overwrite existing token?", + wx.YES_NO | wx.CANCEL | wx.ICON_WARNING) + result = dlg2.ShowModal() + if result != wx.ID_YES: + print("Cancelled") + return + token = omero_user_token.setter( + self.cretentials.server, + self.credentials.port, + self.credentials.username, + self.credentials.passwd, + -1) + if token: + print("Done") + self.message_ctrl.Label = "Connected. Token Set!" + self.message_ctrl.ForegroundColour = "forest green" + else: + print("Failed") + self.message_ctrl.Label = "Failed to set token." + self.message_ctrl.ForegroundColour = "red" + self.message_ctrl.Refresh() + + def connect(self): + try: + server = self.omero_server_ctrl.GetValue() + port = self.omero_port_ctrl.GetValue() + user = self.omero_user_ctrl.GetValue() + passwd = self.omero_password_ctrl.GetValue() + except: + self.message_ctrl.Label = ( + "The port number must be an integer between 0 and 65535 (try 4064)" + ) + self.message_ctrl.ForegroundColour = "red" + self.message_ctrl.Refresh() + return False + self.message_ctrl.ForegroundColour = "black" + self.message_ctrl.Label = "Connecting..." + self.message_ctrl.Refresh() + # Allow UI to update before connecting + wx.Yield() + success = self.credentials.login(server, port, user, passwd) + if success: + self.message_ctrl.Label = "Connected" + self.message_ctrl.ForegroundColour = "forest green" + self.token_button.Enable() + self.message_ctrl.Refresh() + set_omero_server(server) + set_omero_port(port) + set_omero_user(user) + self.ok_button.Enable(True) + return True + else: + self.message_ctrl.Label = "Failed to log onto server" + self.message_ctrl.ForegroundColour = "red" + self.message_ctrl.Refresh() + self.token_button.Disable() + return False + + def on_cancel(self, event): + self.EndModal(wx.CANCEL) + + def on_ok(self, event): + self.EndModal(wx.OK) + +# TODO: Connection manager +# TODO: user token support +# TODO: headless mode +# TODO: Handle multichannel images when requesting grey From 31e4bb50d8200b42fb583a5310e791abd2373e47 Mon Sep 17 00:00:00 2001 From: David Stirling Date: Thu, 27 Jul 2023 16:14:40 +0100 Subject: [PATCH 02/32] Add OMERO browser --- active_plugins/omeroreader.py | 399 ++++++++++++++++++++++++++++++++-- 1 file changed, 377 insertions(+), 22 deletions(-) diff --git a/active_plugins/omeroreader.py b/active_plugins/omeroreader.py index 314731d..01dc32f 100644 --- a/active_plugins/omeroreader.py +++ b/active_plugins/omeroreader.py @@ -49,9 +49,14 @@ server connections from timing out while CellProfiler is running, though you may need to reconnect if the PC goes to sleep. """ +import base64 +import functools +import io import os import collections import atexit +from io import BytesIO +import requests from struct import unpack @@ -481,8 +486,8 @@ def get_display_server(): return config_read_typed(f"Reader.{OMEROReader.reader_name}.show_server", bool) -def set_display_server(val): - config_write_typed(key, value, key_type=bool) +def set_display_server(value): + config_write_typed(f"Reader.{OMEROReader.reader_name}.show_server", value, key_type=bool) class LoginHelper: @@ -496,6 +501,7 @@ def __init__(self): self.client = None self.container_service = None self.tokens = {} + self.browser_window = None atexit.register(self.shutdown) def get_tokens(self, path=None): @@ -509,7 +515,7 @@ def get_tokens(self, path=None): os.environ['HOME'] = path try: LOGGER.info("Requesting token info") - token = omero_user_token.getter() + token = omero_user_token.get_token() server, port = token[token.find('@') + 1:].split(':') port = int(port) LOGGER.info("Connection to {}:{}".format(server, port)) @@ -539,7 +545,7 @@ def login(self, server=None, port=None, user=None, passwd=None, session_key=None self.port = port self.session_key = session_key except Exception as e: - print(f"Failed to join session, token may have expired: {e}") + LOGGER.error(f"Failed to join session, token may have expired: {e}") self.client = None self.session = None return False @@ -549,12 +555,13 @@ def login(self, server=None, port=None, user=None, passwd=None, session_key=None username=user, password=passwd) self.client.enableKeepAlive(60) self.session.detachOnDestroy() + self.session_key = self.client.getSessionId() self.server = server self.port = port self.username = user self.passwd = passwd except Exception as e: - print(f"Failed to create session: {e}") + LOGGER.error(f"Failed to create session: {e}") self.client = None self.session = None return False @@ -572,7 +579,7 @@ def shutdown(self): try: client.closeSession() except Exception as e: - print("Failed to close OMERO session - ", e) + LOGGER.error("Failed to close OMERO session - ", e) CREDENTIALS = LoginHelper() @@ -589,15 +596,15 @@ def show_login_dlg(token=True): with OmeroLoginDlg(frame, title="Log into Omero", token=token) as dlg: dlg.ShowModal() - def login(e): + def login(e=None): CREDENTIALS.get_tokens() if CREDENTIALS.tokens: connected = CREDENTIALS.try_token(list(CREDENTIALS.tokens.keys())[0]) if get_headless(): if connected: - print("Connected to ", CREDENTIALS.server) + LOGGER.info("Connected to ", CREDENTIALS.server) else: - print("Failed to connect, was user token invalid?") + LOGGER.warning("Failed to connect, was user token invalid?") return connected elif connected: from cellprofiler.gui.errordialog import show_warning @@ -612,17 +619,24 @@ def login(e): def login_no_token(e): show_login_dlg() - # def browse(e): - # if CREDENTIALS.client is None: - # show_login_dlg() - # browse_images() + + def browse(e): + if CREDENTIALS.client is None: + login() + app = wx.GetApp() + frame = app.GetTopWindow() + if CREDENTIALS.browser_window is None: + CREDENTIALS.browser_window = OmeroBrowseDlg(frame, title=f"Browse OMERO: {CREDENTIALS.server}") + CREDENTIALS.browser_window.Show() + else: + CREDENTIALS.browser_window.Raise() cellprofiler.gui.plugins_menu.PLUGIN_MENU_ENTRIES.extend([ (login, wx.NewId(), "Connect to OMERO", "Establish an OMERO connection"), (login_no_token, wx.NewId(), "Connect to OMERO (no token)", "Establish an OMERO connection," " but without using user tokens"), - # (browse, wx.NewID(), "Browse OMERO for images", "Browse an OMERO server and add images to the pipeline") + (browse, wx.NewId(), "Browse OMERO for images", "Browse an OMERO server and add images to the pipeline") ]) class OmeroLoginDlg(wx.Dialog): @@ -725,6 +739,9 @@ def mark_dirty(self, event): self.message_ctrl.ForegroundColour = "black" def on_connect_pressed(self, event): + if self.credentials.client is not None and self.credentials.server == self.omero_server_ctrl.GetValue(): + # Already connected, accept another 'Connect' command as an ok to close + self.EndModal(wx.OK) self.connect() def on_set_pressed(self, event): @@ -738,20 +755,20 @@ def on_set_pressed(self, event): wx.YES_NO | wx.CANCEL | wx.ICON_WARNING) result = dlg2.ShowModal() if result != wx.ID_YES: - print("Cancelled") + LOGGER.debug("Cancelled") return token = omero_user_token.setter( - self.cretentials.server, + self.credentials.server, self.credentials.port, self.credentials.username, self.credentials.passwd, -1) if token: - print("Done") + LOGGER.info("Set OMERO user token") self.message_ctrl.Label = "Connected. Token Set!" self.message_ctrl.ForegroundColour = "forest green" else: - print("Failed") + LOGGER.error("Failed to set OMERO user token") self.message_ctrl.Label = "Failed to set token." self.message_ctrl.ForegroundColour = "red" self.message_ctrl.Refresh() @@ -798,7 +815,345 @@ def on_cancel(self, event): def on_ok(self, event): self.EndModal(wx.OK) -# TODO: Connection manager -# TODO: user token support -# TODO: headless mode -# TODO: Handle multichannel images when requesting grey + + class OmeroBrowseDlg(wx.Frame): + def __init__(self, *args, **kwargs): + super(self.__class__, self).__init__(*args, + style=wx.RESIZE_BORDER | wx.CAPTION | wx.CLOSE_BOX, + size=(800, 600), + **kwargs) + self.credentials = CREDENTIALS + self.admin_service = self.credentials.session.getAdminService() + self.url_loader = self.Parent.pipeline.add_urls + + self.Bind(wx.EVT_CLOSE, self.close_browser) + + ec = self.admin_service.getEventContext() + self.groups = [self.admin_service.getGroup(v) for v in ec.memberOfGroups] + self.group_names = [group.name.val for group in self.groups] + self.current_group = self.groups[0].id.getValue() + self.levels = {'projects': 'datasets', + 'datasets': 'images', + 'screens': 'plates', + 'orphaned': 'images' + } + + splitter = wx.SplitterWindow(self, -1, style=wx.SP_BORDER) + self.browse_controls = wx.Panel(splitter, -1) + b = wx.BoxSizer(wx.VERTICAL) + + self.groups_box = wx.Choice(self.browse_controls, choices=self.group_names) + self.groups_box.Bind(wx.EVT_CHOICE, self.switch_group) + + b.Add(self.groups_box, 0, wx.EXPAND) + self.container = self.credentials.session.getContainerService() + + self.tree = wx.TreeCtrl(self.browse_controls, style=wx.TR_HAS_BUTTONS | wx.TR_HIDE_ROOT) + image_list = wx.ImageList(16, 12) + image_data = { + 'projects': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAANCAYAAACgu+4kAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAA5NpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuMy1jMDExIDY2LjE0NTY2MSwgMjAxMi8wMi8wNi0xNDo1NjoyNyAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wTU09Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9tbS8iIHhtbG5zOnN0UmVmPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvc1R5cGUvUmVzb3VyY2VSZWYjIiB4bWxuczp4bXA9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC8iIHhtcE1NOk9yaWdpbmFsRG9jdW1lbnRJRD0ieG1wLmRpZDpDNDk5ODU0N0U5MjA2ODExODhDNkJBNzRDM0U2QkE2NyIgeG1wTU06RG9jdW1lbnRJRD0ieG1wLmRpZDpFNjZDNDcxNzc5NEUxMUUxOTY2OEJEQjhGOUExQ0Y3RCIgeG1wTU06SW5zdGFuY2VJRD0ieG1wLmlpZDpFNjZDNDcxNjc5NEUxMUUxOTY2OEJEQjhGOUExQ0Y3RCIgeG1wOkNyZWF0b3JUb29sPSJBZG9iZSBQaG90b3Nob3AgQ1M2ICgxMy4wIDIwMTIwMzA1Lm0uNDE1IDIwMTIvMDMvMDU6MjE6MDA6MDApICAoTWFjaW50b3NoKSI+IDx4bXBNTTpEZXJpdmVkRnJvbSBzdFJlZjppbnN0YW5jZUlEPSJ4bXAuaWlkOkM1NzAxRDEzMjkyMTY4MTE4OEM2QkE3NEMzRTZCQTY3IiBzdFJlZjpkb2N1bWVudElEPSJ4bXAuZGlkOkM0OTk4NTQ3RTkyMDY4MTE4OEM2QkE3NEMzRTZCQTY3Ii8+IDwvcmRmOkRlc2NyaXB0aW9uPiA8L3JkZjpSREY+IDwveDp4bXBtZXRhPiA8P3hwYWNrZXQgZW5kPSJyIj8+vd9MhwAAALhJREFUeNpi/P//P0NEXsd/BhxgxaQKRgY8gAXGMNbVwpA8e/kaAyHAGJhWe5iNnctGVkoaQ/Lxs6cMv35+w6f/CMvfP39svDytscrqaijgtX3t5u02IAMYXrx5z0AOAOll+fPnF8Ov37/IMgCkl+XP799Af5JpAFAv0IBfDD9//STTAKALfgMJcl0A0gvxwi8KvPAXFIhkGvAXHIjAqPjx4zuZsQCMxn9//my9eOaEFQN54ChAgAEAzRBnWnEZWFQAAAAASUVORK5CYII=', + 'datasets': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAANCAYAAACgu+4kAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAA5NpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuMy1jMDExIDY2LjE0NTY2MSwgMjAxMi8wMi8wNi0xNDo1NjoyNyAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wTU09Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9tbS8iIHhtbG5zOnN0UmVmPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvc1R5cGUvUmVzb3VyY2VSZWYjIiB4bWxuczp4bXA9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC8iIHhtcE1NOk9yaWdpbmFsRG9jdW1lbnRJRD0ieG1wLmRpZDpDNDk5ODU0N0U5MjA2ODExODhDNkJBNzRDM0U2QkE2NyIgeG1wTU06RG9jdW1lbnRJRD0ieG1wLmRpZDpFNjZGMDA2ODc5NEUxMUUxOTY2OEJEQjhGOUExQ0Y3RCIgeG1wTU06SW5zdGFuY2VJRD0ieG1wLmlpZDpFNjZDNDcxQTc5NEUxMUUxOTY2OEJEQjhGOUExQ0Y3RCIgeG1wOkNyZWF0b3JUb29sPSJBZG9iZSBQaG90b3Nob3AgQ1M2ICgxMy4wIDIwMTIwMzA1Lm0uNDE1IDIwMTIvMDMvMDU6MjE6MDA6MDApICAoTWFjaW50b3NoKSI+IDx4bXBNTTpEZXJpdmVkRnJvbSBzdFJlZjppbnN0YW5jZUlEPSJ4bXAuaWlkOkM1NzAxRDEzMjkyMTY4MTE4OEM2QkE3NEMzRTZCQTY3IiBzdFJlZjpkb2N1bWVudElEPSJ4bXAuZGlkOkM0OTk4NTQ3RTkyMDY4MTE4OEM2QkE3NEMzRTZCQTY3Ii8+IDwvcmRmOkRlc2NyaXB0aW9uPiA8L3JkZjpSREY+IDwveDp4bXBtZXRhPiA8P3hwYWNrZXQgZW5kPSJyIj8+l9tKdwAAAKZJREFUeNpi/P//P0NUm+d/BhxgWdV2RgY8gAXGMNMwwpA8deMcAyHAEljsdJhTmJ3h1YfnWBUA5f/j0X+E5d+//zYhrv5YZU108du+cNlKG6AB/xjefn7FQA4A6WX59/cfw5/ff8gz4C/IAKApf/78pswFv3/9Jt8Ff8FeIM+AvzAv/KbUC/8oC8T/DN9//iLTBf8ZWP7/+7f18MELVgzkgaMAAQYAgLlmT8qQW/sAAAAASUVORK5CYII=', + 'screens': 'iVBORw0KGgoAAAANSUhEUgAAABEAAAANCAYAAABPeYUaAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAA5NpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuMy1jMDExIDY2LjE0NTY2MSwgMjAxMi8wMi8wNi0xNDo1NjoyNyAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wTU09Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9tbS8iIHhtbG5zOnN0UmVmPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvc1R5cGUvUmVzb3VyY2VSZWYjIiB4bWxuczp4bXA9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC8iIHhtcE1NOk9yaWdpbmFsRG9jdW1lbnRJRD0ieG1wLmRpZDpDNDk5ODU0N0U5MjA2ODExODhDNkJBNzRDM0U2QkE2NyIgeG1wTU06RG9jdW1lbnRJRD0ieG1wLmRpZDo5NkU0QUUzNjc4NEExMUUxOTY2OEJEQjhGOUExQ0Y3RCIgeG1wTU06SW5zdGFuY2VJRD0ieG1wLmlpZDo5NkU0QUUzNTc4NEExMUUxOTY2OEJEQjhGOUExQ0Y3RCIgeG1wOkNyZWF0b3JUb29sPSJBZG9iZSBQaG90b3Nob3AgQ1M2ICgxMy4wIDIwMTIwMzA1Lm0uNDE1IDIwMTIvMDMvMDU6MjE6MDA6MDApICAoTWFjaW50b3NoKSI+IDx4bXBNTTpEZXJpdmVkRnJvbSBzdFJlZjppbnN0YW5jZUlEPSJ4bXAuaWlkOkM1NzAxRDEzMjkyMTY4MTE4OEM2QkE3NEMzRTZCQTY3IiBzdFJlZjpkb2N1bWVudElEPSJ4bXAuZGlkOkM0OTk4NTQ3RTkyMDY4MTE4OEM2QkE3NEMzRTZCQTY3Ii8+IDwvcmRmOkRlc2NyaXB0aW9uPiA8L3JkZjpSREY+IDwveDp4bXBtZXRhPiA8P3hwYWNrZXQgZW5kPSJyIj8+wkwZRwAAAQxJREFUeNpi/P//PwMjIyODqbHhfwYc4PTZ84y45ED6WZAFPFxdwQYig+27djEQAowgk6wtLcGu4GBnY0C38vvPX3gNOHr8OCPcJaHh4QwsLCwYiv78+YNV87+/fxnWrlkDZsN1PXr0iGHPnj1wRS4uLnj5jg4OcDYTjPH7928w3WxmhcJ3zmtC4Zc2mUH4SC6EG/LrF8TvtaeOofD3TqpD4XfXnULho3gHJOjt7Q2XePHiBV7+s6dPsRuydetWuISuri5evra2Nm7vVKUZoPDTratR+EZV1RjewTCkbdYFFP7Mo60o/HNtrSgBjeEdMzMzuMRToJ/x8Z88eYJpyKcPH8AYGRDiwwBAgAEAvXKdXsBF6t8AAAAASUVORK5CYII=' + } + self.image_codes = {} + for name, dat in image_data.items(): + decodedImgData = base64.b64decode(dat) + bio = BytesIO(decodedImgData) + img = wx.Image(bio) + self.image_codes[name] = image_list.Add(img.ConvertToBitmap()) + self.tree.AssignImageList(image_list) + + self.tree.Bind(wx.EVT_TREE_ITEM_EXPANDING, self.fetch_children) + self.tree.Bind(wx.EVT_TREE_SEL_CHANGING, self.select_tree) + self.tree.Bind(wx.EVT_TREE_SEL_CHANGED, self.update_thumbnails) + self.tree.Bind(wx.EVT_TREE_BEGIN_DRAG, self.process_drag) + + data = self.fetch_containers() + + self.populate_tree(data) + b.Add(self.tree, 1, wx.EXPAND) + + self.browse_controls.SetSizer(b) + + self.image_controls = wx.Panel(splitter, -1) + + vert_sizer = wx.BoxSizer(wx.VERTICAL) + + self.tile_panel = wx.ScrolledWindow(self.image_controls, 1) + self.tile_panel.url_loader = self.url_loader + + self.tiler_sizer = wx.WrapSizer(wx.HORIZONTAL) + self.tile_panel.SetSizer(self.tiler_sizer) + self.tile_panel.SetScrollbars(0, 20, 0, 20) + + self.update_thumbnails() + vert_sizer.Add(self.tile_panel, wx.EXPAND) + + add_button = wx.Button(self.image_controls, wx.NewId(), "Add to file list") + add_button.Bind(wx.EVT_BUTTON, self.add_selected_to_pipeline) + vert_sizer.Add(add_button, 0, wx.ALIGN_RIGHT | wx.ALL, 5) + self.image_controls.SetSizer(vert_sizer) + splitter.SplitVertically(self.browse_controls, self.image_controls, 200) + + self.Layout() + + def close_browser(self, event): + self.credentials.browser_window = None + event.Skip() + + def add_selected_to_pipeline(self, e=None): + displayed = self.tile_panel.GetChildren() + all_urls = [] + selected_urls = [] + for item in displayed: + if isinstance(item, ImagePanel): + all_urls.append(item.url) + if item.selected: + selected_urls.append(item.url) + if selected_urls: + self.url_loader(selected_urls) + else: + self.url_loader(all_urls) + + def process_drag(self, event): + # We have our own custom handler here + data = wx.FileDataObject() + for file_url in self.fetch_file_list_from_tree(event): + data.AddFile(file_url) + drop_src = wx.DropSource(self) + drop_src.SetData(data) + drop_src.DoDragDrop(wx.Drag_CopyOnly) + + def fetch_file_list_from_tree(self, event): + files = [] + + def recurse_for_images(tree_id): + if not self.tree.IsExpanded(tree_id): + self.tree.Expand(tree_id) + data = self.tree.GetItemData(tree_id) + item_type = data['type'] + if item_type == 'images': + files.append(f"https://{self.credentials.server}/webclient/?show=image-{data['id']}") + elif 'images' in data: + for omero_id, _ in data['images'].items(): + files.append(f"https://{self.credentials.server}/webclient/?show=image-{omero_id}") + else: + child_id, cookie = self.tree.GetFirstChild(tree_id) + while child_id.IsOk(): + recurse_for_images(child_id) + child_id, cookie = self.tree.GetNextChild(tree_id, cookie) + + recurse_for_images(event.GetItem()) + return files + + def select_tree(self, event): + target_id = event.GetItem() + self.tree.Expand(target_id) + + def next_level(self, level): + return self.levels.get(level, None) + + def switch_group(self, e=None): + new_group = self.groups_box.GetCurrentSelection() + self.current_group = self.groups[new_group].id.getValue() + data = self.fetch_containers() + self.populate_tree(data) + + def fetch_children(self, event): + target_id = event.GetItem() + data = self.tree.GetItemData(target_id) + self.tree.DeleteChildren(target_id) + subject_type = data['type'] + target_type = self.levels.get(subject_type, None) + if target_type is None: + # We're at the bottom level already + return + subject = data['id'] + if subject == -1: + sub_str = "orphaned=true&" + else: + sub_str = f"id={subject}&" + url = f"https://{self.credentials.server}/webclient/api/{target_type}/?{sub_str}experimenter_id=-1&page=0&group={self.current_group}&bsession={self.credentials.session_key}" + result = requests.get(url, timeout=5) + result.raise_for_status() + result = result.json() + if 'images' in result: + image_map = {entry['id']: entry['name'] for entry in result['images']} + data['images'] = image_map + self.tree.SetItemData(target_id, data) + self.populate_tree(result, target_id) + + def fetch_containers(self): + url = f"https://{self.credentials.server}/webclient/api/containers/?experimenter_id=-1&page=0&group={self.current_group}&bsession={self.credentials.session_key}" + data = requests.get(url, timeout=5) + data.raise_for_status() + return data.json() + + def fetch_thumbnails(self, id_list): + if not id_list: + return {} + id_list = [str(x) for x in id_list] + chunk_size = 100 + buffer = {x: "" for x in id_list} + for i in range(0, len(id_list), chunk_size): + ids_to_get = '&id='.join(id_list[i:i + chunk_size]) + url = f"https://{self.credentials.server}/webclient/get_thumbnails/128/?&bsession={self.credentials.session_key}&id={ids_to_get}" + LOGGER.debug(f"Fetching {url}") + data = requests.get(url, timeout=5) + if data.status_code != 200: + LOGGER.warning(f"Server error: {data.status_code} - {data.reason}") + else: + buffer.update(data.json()) + return buffer + + @functools.lru_cache(maxsize=20) + def fetch_large_thumbnail(self, id): + url = f"https://{self.credentials.server}/webgateway/render_thumbnail/{id}/450/450/?bsession={self.credentials.session_key}" + LOGGER.debug(f"Fetching {url}") + data = requests.get(url) + if data.status_code != 200: + LOGGER.warning("Server error:", data.status_code, data.reason) + return False + elif not data.content: + return False + io_bytes = io.BytesIO(data.content) + return wx.Image(io_bytes) + + def update_thumbnails(self, event=None): + self.tiler_sizer.Clear(delete_windows=True) + if not event: + return + target_id = event.GetItem() + item_data = self.tree.GetItemData(target_id) + if item_data.get('type', None) == 'images': + image_id = item_data['id'] + img_name = item_data['name'] + thumb_img = self.fetch_large_thumbnail(image_id) + if not thumb_img or not thumb_img.IsOk(): + thumb_img = self.get_error_thumbnail(450) + else: + thumb_img = thumb_img.ConvertToBitmap() + tile = ImagePanel(thumb_img, self.tile_panel, image_id, img_name, self.credentials.server, size=450) + tile.selected = True + self.tiler_sizer.Add(tile, 0, wx.ALL, 5) + else: + image_targets = item_data.get('images', {}) + if not image_targets: + return + + id_list = list(image_targets.keys()) + data = self.fetch_thumbnails(id_list) + for image_id, image_data in data.items(): + img_name = image_targets[int(image_id)] + start_data = image_data.find('/9') + if start_data == -1: + img = self.get_error_thumbnail(128) + else: + decoded = base64.b64decode(image_data[start_data:]) + bio = BytesIO(decoded) + img = wx.Image(bio) + if not img.IsOk(): + img = self.get_error_thumbnail(128) + else: + img = img.ConvertToBitmap() + tile = ImagePanel(img, self.tile_panel, image_id, img_name, self.credentials.server) + self.tiler_sizer.Add(tile, 0, wx.ALL, 5) + self.tiler_sizer.Layout() + self.image_controls.Layout() + self.image_controls.Refresh() + + @functools.lru_cache(maxsize=10) + def get_error_thumbnail(self, size): + # Create an image with an error icon + artist = wx.ArtProvider() + size //= 2 + return artist.GetBitmap(wx.ART_WARNING, size=(size, size)) + + def populate_tree(self, data, parent=None): + if parent is None: + self.tree.DeleteAllItems() + parent = self.tree.AddRoot("Server") + for item_type, items in data.items(): + image = self.image_codes.get(item_type, None) + if not isinstance(items, list): + items = [items] + for entry in items: + entry['type'] = item_type + new_id = self.tree.AppendItem(parent, f"{entry['name']}", data=entry) + if image is not None: + self.tree.SetItemImage(new_id, image, wx.TreeItemIcon_Normal) + if entry.get('childCount', 0) > 0: + self.tree.SetItemHasChildren(new_id) + + + class ImagePanel(wx.Panel): + ''' + ImagePanels are wxPanels that display a wxBitmap and store multiple + image channels which can be recombined to mix different bitmaps. + ''' + + def __init__(self, thumbnail, parent, omero_id, name, server, size=128): + """ + thumbnail -- wx Bitmap + parent -- parent window to the wx.Panel + + """ + self.parent = parent + self.bitmap = thumbnail + self.selected = False + self.omero_id = omero_id + self.url = f"https://{server}/webclient/?show=image-{omero_id}" + self.name = name + if len(name) > 17: + self.shortname = name[:14] + '...' + else: + self.shortname = name + self.size_x = size + self.size_y = size + 30 + wx.Panel.__init__(self, parent, wx.NewId(), size=(self.size_x, self.size_y)) + self.Bind(wx.EVT_PAINT, self.OnPaint) + self.Bind(wx.EVT_LEFT_DOWN, self.select) + self.Bind(wx.EVT_RIGHT_DOWN, self.right_click) + self.SetClientSize((self.size_x, self.size_y)) + + def select(self, e): + self.selected = not self.selected + self.Refresh() + e.StopPropagation() + e.Skip() + + def right_click(self, event): + print("Got right click") + popupmenu = wx.Menu() + add_file_item = popupmenu.Append(-1, "Add to file list") + self.Bind(wx.EVT_MENU, self.add_to_pipeline, add_file_item) + add_file_item = popupmenu.Append(-1, "Show in OMERO.web") + self.Bind(wx.EVT_MENU, self.open_in_browser, add_file_item) + # Show menu + self.PopupMenu(popupmenu, event.GetPosition()) + + def add_to_pipeline(self, e): + self.parent.url_loader([self.url]) + + def open_in_browser(self, e): + wx.LaunchDefaultBrowser(self.url) + + def OnPaint(self, evt): + dc = wx.PaintDC(self) + dc.Clear() + dc.DrawBitmap(self.bitmap, (self.size_x - self.bitmap.Width) // 2, + ((self.size_x - self.bitmap.Height) // 2) + 20) + rect = wx.Rect(0, 0, self.size_x, self.size_x + 20) + dc.DrawLabel(self.shortname, rect, alignment=wx.ALIGN_CENTER_HORIZONTAL | wx.ALIGN_TOP) + dc.SetPen(wx.Pen("GREY", style=wx.PENSTYLE_SOLID)) + dc.SetBrush(wx.Brush("BLACK", wx.TRANSPARENT)) + dc.DrawRectangle(rect) + # Outline the whole image + if self.selected: + dc.SetPen(wx.Pen("BLUE", 3)) + dc.SetBrush(wx.Brush("BLACK", style=wx.TRANSPARENT)) + dc.DrawRectangle(0, 0, self.size_x, self.size_y) + return dc + +# Todo: Make better drag selection +# Todo: Split GUI into a helper module From 193dcd0f49df3953df0fef914dd97427b9fe1f67 Mon Sep 17 00:00:00 2001 From: David Stirling Date: Fri, 28 Jul 2023 11:25:58 +0100 Subject: [PATCH 03/32] General improvements --- active_plugins/omeroreader.py | 207 ++++++++++++++++------------------ 1 file changed, 97 insertions(+), 110 deletions(-) diff --git a/active_plugins/omeroreader.py b/active_plugins/omeroreader.py index 01dc32f..4fa1410 100644 --- a/active_plugins/omeroreader.py +++ b/active_plugins/omeroreader.py @@ -57,13 +57,13 @@ import atexit from io import BytesIO import requests +import urllib.parse from struct import unpack -from cellprofiler_core.preferences import get_headless - import numpy +from cellprofiler_core.preferences import get_headless from cellprofiler_core.constants.image import MD_SIZE_S, MD_SIZE_C, MD_SIZE_Z, MD_SIZE_T, \ MD_SIZE_Y, MD_SIZE_X, MD_SERIES_NAME from cellprofiler_core.preferences import get_omero_server, get_omero_port, get_omero_user, set_omero_server,\ @@ -87,12 +87,11 @@ REGEX_INDEX_FROM_FILE_NAME = re.compile(r'\?show=image-(\d+)') -PASSTHROUGH_SCHEMES.append('OMERO') +# Inject omero as a URI scheme which CellProfiler should accept as an image entry. +PASSTHROUGH_SCHEMES.append('omero') -SCALE_ONE_TYPE = ["float", "double"] LOGGER = logging.getLogger(__name__) - PIXEL_TYPES = { "int8": ['b', numpy.int8, (-128, 127)], "uint8": ['B', numpy.uint8, (0, 255)], @@ -107,7 +106,7 @@ class OMEROReader(Reader): """ - Reads images from OMERO. + Reads images from an OMERO server. """ reader_name = "OMERO Reader" variable_revision_number = 1 @@ -117,6 +116,7 @@ class OMEROReader(Reader): def __init__(self, image_file): self.login = CREDENTIALS self.image_id = None + self.server = None self.omero_image = None self.pixels = None self.width = None @@ -128,26 +128,35 @@ def __del__(self): self.close() def confirm_connection(self): + # Verify that we're able to connect to a server if self.login.client is None: if get_headless(): - raise ValueError("No OMERO connection established") + connected = login(server=self.server) + if connected: + return True + else: + raise ValueError("No OMERO connection established") else: - login(None) + login(server=self.server) if self.login.client is None: raise ValueError("Connection failed") def init_reader(self): - # Check if session object already exists - self.confirm_connection() + # Setup the reader if self.omero_image is not None: + # We're already connected and have fetched the image pointer return True if self.file.scheme == "omero": self.image_id = int(self.file.url[10:]) else: matches = REGEX_INDEX_FROM_FILE_NAME.findall(self.file.url) if not matches: - raise ValueError("URL may not be from OMERO") + raise ValueError("URL may not be from OMERO?") self.image_id = int(matches[0]) + self.server = urllib.parse.urlparse(self.file.url).hostname + + # Check if session object already exists + self.confirm_connection() LOGGER.debug("Initializing OmeroReader for Image id: %s" % self.image_id) # Get image object from the server @@ -173,6 +182,7 @@ def read(self, xywh=None, wants_max_intensity=False, channel_names=None, + volumetric=False, ): """Read a single plane from the image file. :param c: read from this channel. `None` = read color image if multichannel @@ -187,9 +197,9 @@ def read(self, :param wants_max_intensity: if `False`, only return the image; if `True`, return a tuple of image and max intensity :param channel_names: provide the channel names for the OME metadata + :param volumetric: Whether we're reading in 3D """ self.init_reader() - debug_message = \ "Reading C: %s, Z: %s, T: %s, series: %s, index: %s, " \ "channel names: %s, rescale: %s, wants_max_intensity: %s, " \ @@ -197,7 +207,7 @@ def read(self, wants_max_intensity, xywh) if c is None and index is not None: c = index - LOGGER.info(debug_message) + LOGGER.debug(debug_message) message = None if (t or 0) >= self.pixels.getSizeT().val: message = "T index %s exceeds sizeT %s" % \ @@ -218,7 +228,10 @@ def read(self, assert isinstance(xywh, tuple) and len(xywh) == 4, \ "Invalid XYWH tuple" tile = xywh - numpy_image = self.read_planes(z, c, t, tile) + if not volumetric: + numpy_image = self.read_planes(z, c, t, tile) + else: + numpy_image = self.read_planes_volumetric(z, c, t, tile) pixel_type = self.pixels.getPixelsType().value.val min_value = PIXEL_TYPES[pixel_type][2][0] max_value = PIXEL_TYPES[pixel_type][2][1] @@ -250,56 +263,18 @@ def read_volume(self, wants_max_intensity=False, channel_names=None, ): - self.init_reader() - debug_message = \ - "Reading C: %s, Z: %s, T: %s, series: %s, index: %s, " \ - "channel names: %s, rescale: %s, wants_max_intensity: %s, " \ - "XYWH: %s" % (c, z, t, series, index, channel_names, rescale, - wants_max_intensity, xywh) - if c is None and index is not None: - c = index - LOGGER.info(debug_message) - message = None - if (t or 0) >= self.pixels.getSizeT().val: - message = "T index %s exceeds sizeT %s" % \ - (t, self.pixels.getSizeT().val) - LOGGER.error(message) - if (c or 0) >= self.pixels.getSizeC().val: - message = "C index %s exceeds sizeC %s" % \ - (c, self.pixels.getSizeC().val) - LOGGER.error(message) - if (z or 0) >= self.pixels.getSizeZ().val: - message = "Z index %s exceeds sizeZ %s" % \ - (z, self.pixels.getSizeZ().val) - LOGGER.error(message) - if message is not None: - raise Exception("Couldn't retrieve a plane from OMERO image.") - tile = None - if xywh is not None: - assert isinstance(xywh, tuple) and len(xywh) == 4, \ - "Invalid XYWH tuple" - tile = xywh - numpy_image = self.read_planes_volumetric(z, c, t, tile) - pixel_type = self.pixels.getPixelsType().value.val - min_value = PIXEL_TYPES[pixel_type][2][0] - max_value = PIXEL_TYPES[pixel_type][2][1] - LOGGER.debug("Pixel range [%s, %s]" % (min_value, max_value)) - if rescale or pixel_type == 'double': - LOGGER.info("Rescaling image using [%s, %s]" % (min_value, max_value)) - # Note: The result here differs from: - # https://github.com/emilroz/python-bioformats/blob/a60b5c5a5ae018510dd8aa32d53c35083956ae74/bioformats/formatreader.py#L903 - # Reason: the unsigned types are being properly taken into account - # and converted to [0, 1] using their full scale. - # Further note: float64 should be used for the numpy array in case - # image is stored as 'double', we're keeping it float32 to stay - # consitent with the CellProfiler reader (the double type is also - # converted to single precision) - numpy_image = \ - (numpy_image.astype(numpy.float32) + float(min_value)) / \ - (float(max_value) - float(min_value)) - if wants_max_intensity: - return numpy_image, max_value - return numpy_image + # Forward 3D calls to the standard reader function + return self.read( + series=series, + c=c, + z=z, + t=t, + rescale=rescale, + xywh=xywh, + wants_max_intensity=wants_max_intensity, + channel_names=channel_names, + volumetric=True + ) def read_planes(self, z=0, c=None, t=0, tile=None): ''' @@ -307,7 +282,13 @@ def read_planes(self, z=0, c=None, t=0, tile=None): ''' channels = [] if c is None: - channels = range(self.pixels.getSizeC().val) + channel_count = self.pixels.getSizeC().val + if channel_count == 1: + # This is obviously greyscale, treat it as such. + channels.append(0) + c = 0 + else: + channels = range(channel_count) else: channels.append(c) pixel_type = self.pixels.getPixelsType().value.val @@ -403,51 +384,29 @@ def read_planes_volumetric(self, z=None, c=None, t=None, tile=None): @classmethod def supports_url(cls): + # We read OMERO URLs directly without caching a download. return True @classmethod def supports_format(cls, image_file, allow_open=False, volume=False): - """This function needs to evaluate whether a given ImageFile object - can be read by this reader class. - - Return value should be an integer representing suitability: - -1 - 'I can't read this at all' - 1 - 'I am the one true reader for this format, don't even bother checking any others' - 2 - 'I am well-suited to this format' - 3 - 'I can read this format, but I might not be the best', - 4 - 'I can give it a go, if you must' - - The allow_open parameter dictates whether the reader is permitted to read the file when - making this decision. If False the decision should be made using file extension only. - Any opened files should be closed before returning. - - The volume parameter specifies whether the reader will need to return a 3D array. - .""" if image_file.scheme not in cls.supported_schemes: + # I can't read this return -1 if image_file.scheme == "omero": + # Yes please return 1 elif "?show=image" in image_file.url.lower(): + # Looks enough like an OMERO URL that I'll have a go. return 2 return -1 def close(self): - # If your reader opens a file, this needs to release any active lock, + # We don't activate any file locks. pass def get_series_metadata(self): """ OMERO image IDs only ever refer to a single series - - Should return a dictionary with the following keys: - Key names are in cellprofiler_core.constants.image - MD_SIZE_S - int reflecting the number of series - MD_SIZE_X - list of X dimension sizes, one element per series. - MD_SIZE_Y - list of Y dimension sizes, one element per series. - MD_SIZE_Z - list of Z dimension sizes, one element per series. - MD_SIZE_C - list of C dimension sizes, one element per series. - MD_SIZE_T - list of T dimension sizes, one element per series. - MD_SERIES_NAME - list of series names, one element per series. """ self.init_reader() LOGGER.info(f"Extracting metadata for image {self.image_id}") @@ -463,6 +422,7 @@ def get_series_metadata(self): @staticmethod def get_settings(): + # Define settings available in the reader return [ ('allow_token', "Allow OMERO user tokens", @@ -509,6 +469,10 @@ def get_tokens(self, path=None): # Future versions of omero_user_token may support multiple tokens, so we code with that in mind. if not TOKENS_AVAILABLE: return + # Check the reader setting which disables tokens. + tokens_enabled = config_read_typed(f"Reader.{OMEROReader.reader_name}.allow_token", bool) + if tokens_enabled is not None and not tokens_enabled: + return # User tokens sadly default to the home directory. This would override that location. py_home = os.environ['HOME'] if path is not None: @@ -590,16 +554,19 @@ def shutdown(self): import cellprofiler.gui.plugins_menu - def show_login_dlg(token=True): + def show_login_dlg(token=True, server=None): app = wx.GetApp() frame = app.GetTopWindow() - with OmeroLoginDlg(frame, title="Log into Omero", token=token) as dlg: + with OmeroLoginDlg(frame, title="Log into Omero", token=token, server=server) as dlg: dlg.ShowModal() - def login(e=None): + def login(e=None, server=None): CREDENTIALS.get_tokens() if CREDENTIALS.tokens: - connected = CREDENTIALS.try_token(list(CREDENTIALS.tokens.keys())[0]) + if server is None: + # URL didn't specify which server we want. Just try whichever token is available + server = list(CREDENTIALS.tokens.keys())[0] + connected = CREDENTIALS.try_token(server) if get_headless(): if connected: LOGGER.info("Connected to ", CREDENTIALS.server) @@ -608,13 +575,12 @@ def login(e=None): return connected elif connected: from cellprofiler.gui.errordialog import show_warning - cellprofiler.gui.errordialog.show_warning("Connected to OMERO", - f"A token was found and used to " - f"connect to the OMERO server at {CREDENTIALS.server}", - get_display_server, - set_display_server) + show_warning("Connected to OMERO", + f"A token was found and used to connect to the OMERO server at {CREDENTIALS.server}", + get_display_server, + set_display_server) return - show_login_dlg() + show_login_dlg(server=server) def login_no_token(e): show_login_dlg() @@ -641,11 +607,13 @@ def browse(e): class OmeroLoginDlg(wx.Dialog): - def __init__(self, *args, token=True, **kwargs): + def __init__(self, *args, token=True, server=None, **kwargs): super(self.__class__, self).__init__(*args, **kwargs) self.credentials = CREDENTIALS self.token = token self.SetSizer(wx.BoxSizer(wx.VERTICAL)) + if server is None: + server = self.credentials.server sizer = wx.BoxSizer(wx.VERTICAL) self.Sizer.Add(sizer, 1, wx.EXPAND | wx.ALL, 6) sub_sizer = wx.BoxSizer(wx.HORIZONTAL) @@ -663,7 +631,7 @@ def __init__(self, *args, token=True, **kwargs): sub_sizer.Add( wx.StaticText(self, label="Server:", size=lsize), 0, wx.ALIGN_CENTER_VERTICAL) - self.omero_server_ctrl = wx.TextCtrl(self, value=self.credentials.server) + self.omero_server_ctrl = wx.TextCtrl(self, value=server) sub_sizer.Add(self.omero_server_ctrl, 1, wx.EXPAND) sizer.AddSpacer(5) @@ -692,6 +660,8 @@ def __init__(self, *args, token=True, **kwargs): 0, wx.ALIGN_CENTER_VERTICAL) self.omero_password_ctrl = wx.TextCtrl(self, value="", style=wx.TE_PASSWORD|wx.TE_PROCESS_ENTER) self.omero_password_ctrl.Bind(wx.EVT_TEXT_ENTER, self.on_connect_pressed) + if self.credentials.username is not None: + self.omero_password_ctrl.SetFocus() sub_sizer.Add(self.omero_password_ctrl, 1, wx.EXPAND) sizer.AddSpacer(5) @@ -974,7 +944,12 @@ def fetch_children(self, event): else: sub_str = f"id={subject}&" url = f"https://{self.credentials.server}/webclient/api/{target_type}/?{sub_str}experimenter_id=-1&page=0&group={self.current_group}&bsession={self.credentials.session_key}" - result = requests.get(url, timeout=5) + + try: + result = requests.get(url, timeout=5) + except requests.exceptions.ConnectTimeout: + LOGGER.error("Server request timed out") + return result.raise_for_status() result = result.json() if 'images' in result: @@ -985,7 +960,11 @@ def fetch_children(self, event): def fetch_containers(self): url = f"https://{self.credentials.server}/webclient/api/containers/?experimenter_id=-1&page=0&group={self.current_group}&bsession={self.credentials.session_key}" - data = requests.get(url, timeout=5) + try: + data = requests.get(url, timeout=5) + except requests.exceptions.ConnectTimeout: + LOGGER.error("Server request timed out") + return {} data.raise_for_status() return data.json() @@ -999,7 +978,10 @@ def fetch_thumbnails(self, id_list): ids_to_get = '&id='.join(id_list[i:i + chunk_size]) url = f"https://{self.credentials.server}/webclient/get_thumbnails/128/?&bsession={self.credentials.session_key}&id={ids_to_get}" LOGGER.debug(f"Fetching {url}") - data = requests.get(url, timeout=5) + try: + data = requests.get(url, timeout=5) + except requests.exceptions.ConnectTimeout: + continue if data.status_code != 200: LOGGER.warning(f"Server error: {data.status_code} - {data.reason}") else: @@ -1008,9 +990,14 @@ def fetch_thumbnails(self, id_list): @functools.lru_cache(maxsize=20) def fetch_large_thumbnail(self, id): + # Get a large thumbnail for single image display mode. We cache the last 20. url = f"https://{self.credentials.server}/webgateway/render_thumbnail/{id}/450/450/?bsession={self.credentials.session_key}" LOGGER.debug(f"Fetching {url}") - data = requests.get(url) + try: + data = requests.get(url, timeout=5) + except requests.exceptions.ConnectTimeout: + LOGGER.error("Thumbnail request timed out") + return False if data.status_code != 200: LOGGER.warning("Server error:", data.status_code, data.reason) return False @@ -1064,7 +1051,7 @@ def update_thumbnails(self, event=None): @functools.lru_cache(maxsize=10) def get_error_thumbnail(self, size): - # Create an image with an error icon + # Draw an image with an error icon. Cache the result since we may need the error icon repeatedly. artist = wx.ArtProvider() size //= 2 return artist.GetBitmap(wx.ART_WARNING, size=(size, size)) @@ -1123,7 +1110,6 @@ def select(self, e): e.Skip() def right_click(self, event): - print("Got right click") popupmenu = wx.Menu() add_file_item = popupmenu.Append(-1, "Add to file list") self.Bind(wx.EVT_MENU, self.add_to_pipeline, add_file_item) @@ -1157,3 +1143,4 @@ def OnPaint(self, evt): # Todo: Make better drag selection # Todo: Split GUI into a helper module +# Todo: User filter From 35d66ebd493603b36bd2da1ca43eb681b75ffe85 Mon Sep 17 00:00:00 2001 From: David Stirling Date: Sat, 29 Jul 2023 14:51:54 +0100 Subject: [PATCH 04/32] Add user filter --- active_plugins/omeroreader.py | 36 +++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/active_plugins/omeroreader.py b/active_plugins/omeroreader.py index 4fa1410..deef211 100644 --- a/active_plugins/omeroreader.py +++ b/active_plugins/omeroreader.py @@ -799,9 +799,15 @@ def __init__(self, *args, **kwargs): self.Bind(wx.EVT_CLOSE, self.close_browser) ec = self.admin_service.getEventContext() - self.groups = [self.admin_service.getGroup(v) for v in ec.memberOfGroups] + # Exclude sys groups + self.groups = [self.admin_service.getGroup(v) for v in ec.memberOfGroups if v > 1] self.group_names = [group.name.val for group in self.groups] self.current_group = self.groups[0].id.getValue() + self.users_in_group = {'All Members': -1} + self.users_in_group.update({ + x.omeName.val: x.id.val for x in self.groups[0].linkedExperimenterList() + }) + self.current_user = -1 self.levels = {'projects': 'datasets', 'datasets': 'images', 'screens': 'plates', @@ -815,7 +821,11 @@ def __init__(self, *args, **kwargs): self.groups_box = wx.Choice(self.browse_controls, choices=self.group_names) self.groups_box.Bind(wx.EVT_CHOICE, self.switch_group) + self.members_box = wx.Choice(self.browse_controls, choices=list(self.users_in_group.keys())) + self.members_box.Bind(wx.EVT_CHOICE, self.switch_member) + b.Add(self.groups_box, 0, wx.EXPAND) + b.Add(self.members_box, 0, wx.EXPAND) self.container = self.credentials.session.getContainerService() self.tree = wx.TreeCtrl(self.browse_controls, style=wx.TR_HAS_BUTTONS | wx.TR_HIDE_ROOT) @@ -926,9 +936,27 @@ def next_level(self, level): def switch_group(self, e=None): new_group = self.groups_box.GetCurrentSelection() self.current_group = self.groups[new_group].id.getValue() + self.current_user = -1 + self.refresh_group_members() + data = self.fetch_containers() + self.populate_tree(data) + + def switch_member(self, e=None): + new_member = self.members_box.GetStringSelection() + self.current_user = self.users_in_group.get(new_member, -1) data = self.fetch_containers() self.populate_tree(data) + def refresh_group_members(self): + self.users_in_group = {'All Members': -1} + group = self.groups[self.groups_box.GetCurrentSelection()] + self.users_in_group.update({ + x.omeName.val: x.id.val for x in group.linkedExperimenterList() + }) + self.members_box.Clear() + self.members_box.AppendItems(list(self.users_in_group.keys())) + + def fetch_children(self, event): target_id = event.GetItem() data = self.tree.GetItemData(target_id) @@ -959,7 +987,7 @@ def fetch_children(self, event): self.populate_tree(result, target_id) def fetch_containers(self): - url = f"https://{self.credentials.server}/webclient/api/containers/?experimenter_id=-1&page=0&group={self.current_group}&bsession={self.credentials.session_key}" + url = f"https://{self.credentials.server}/webclient/api/containers/?id={self.current_user}&page=0&group={self.current_group}&bsession={self.credentials.session_key}" try: data = requests.get(url, timeout=5) except requests.exceptions.ConnectTimeout: @@ -972,7 +1000,7 @@ def fetch_thumbnails(self, id_list): if not id_list: return {} id_list = [str(x) for x in id_list] - chunk_size = 100 + chunk_size = 10 buffer = {x: "" for x in id_list} for i in range(0, len(id_list), chunk_size): ids_to_get = '&id='.join(id_list[i:i + chunk_size]) @@ -1143,4 +1171,4 @@ def OnPaint(self, evt): # Todo: Make better drag selection # Todo: Split GUI into a helper module -# Todo: User filter +# Todo: Handle wells/fields From 1b13edc23bccf114dc17d989cf86fd47be5ce7da Mon Sep 17 00:00:00 2001 From: David Stirling Date: Tue, 1 Aug 2023 13:35:17 +0100 Subject: [PATCH 05/32] Add screens support --- active_plugins/omeroreader.py | 60 ++++++++++++++++++++++++++++------- 1 file changed, 48 insertions(+), 12 deletions(-) diff --git a/active_plugins/omeroreader.py b/active_plugins/omeroreader.py index deef211..1a7489d 100644 --- a/active_plugins/omeroreader.py +++ b/active_plugins/omeroreader.py @@ -55,6 +55,7 @@ import os import collections import atexit +import string from io import BytesIO import requests import urllib.parse @@ -790,7 +791,7 @@ class OmeroBrowseDlg(wx.Frame): def __init__(self, *args, **kwargs): super(self.__class__, self).__init__(*args, style=wx.RESIZE_BORDER | wx.CAPTION | wx.CLOSE_BOX, - size=(800, 600), + size=(900, 600), **kwargs) self.credentials = CREDENTIALS self.admin_service = self.credentials.session.getAdminService() @@ -811,6 +812,7 @@ def __init__(self, *args, **kwargs): self.levels = {'projects': 'datasets', 'datasets': 'images', 'screens': 'plates', + 'plates': 'wells', 'orphaned': 'images' } @@ -829,17 +831,19 @@ def __init__(self, *args, **kwargs): self.container = self.credentials.session.getContainerService() self.tree = wx.TreeCtrl(self.browse_controls, style=wx.TR_HAS_BUTTONS | wx.TR_HIDE_ROOT) - image_list = wx.ImageList(16, 12) + image_list = wx.ImageList(16, 13) image_data = { 'projects': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAANCAYAAACgu+4kAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAA5NpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuMy1jMDExIDY2LjE0NTY2MSwgMjAxMi8wMi8wNi0xNDo1NjoyNyAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wTU09Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9tbS8iIHhtbG5zOnN0UmVmPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvc1R5cGUvUmVzb3VyY2VSZWYjIiB4bWxuczp4bXA9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC8iIHhtcE1NOk9yaWdpbmFsRG9jdW1lbnRJRD0ieG1wLmRpZDpDNDk5ODU0N0U5MjA2ODExODhDNkJBNzRDM0U2QkE2NyIgeG1wTU06RG9jdW1lbnRJRD0ieG1wLmRpZDpFNjZDNDcxNzc5NEUxMUUxOTY2OEJEQjhGOUExQ0Y3RCIgeG1wTU06SW5zdGFuY2VJRD0ieG1wLmlpZDpFNjZDNDcxNjc5NEUxMUUxOTY2OEJEQjhGOUExQ0Y3RCIgeG1wOkNyZWF0b3JUb29sPSJBZG9iZSBQaG90b3Nob3AgQ1M2ICgxMy4wIDIwMTIwMzA1Lm0uNDE1IDIwMTIvMDMvMDU6MjE6MDA6MDApICAoTWFjaW50b3NoKSI+IDx4bXBNTTpEZXJpdmVkRnJvbSBzdFJlZjppbnN0YW5jZUlEPSJ4bXAuaWlkOkM1NzAxRDEzMjkyMTY4MTE4OEM2QkE3NEMzRTZCQTY3IiBzdFJlZjpkb2N1bWVudElEPSJ4bXAuZGlkOkM0OTk4NTQ3RTkyMDY4MTE4OEM2QkE3NEMzRTZCQTY3Ii8+IDwvcmRmOkRlc2NyaXB0aW9uPiA8L3JkZjpSREY+IDwveDp4bXBtZXRhPiA8P3hwYWNrZXQgZW5kPSJyIj8+vd9MhwAAALhJREFUeNpi/P//P0NEXsd/BhxgxaQKRgY8gAXGMNbVwpA8e/kaAyHAGJhWe5iNnctGVkoaQ/Lxs6cMv35+w6f/CMvfP39svDytscrqaijgtX3t5u02IAMYXrx5z0AOAOll+fPnF8Ov37/IMgCkl+XP799Af5JpAFAv0IBfDD9//STTAKALfgMJcl0A0gvxwi8KvPAXFIhkGvAXHIjAqPjx4zuZsQCMxn9//my9eOaEFQN54ChAgAEAzRBnWnEZWFQAAAAASUVORK5CYII=', 'datasets': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAANCAYAAACgu+4kAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAA5NpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuMy1jMDExIDY2LjE0NTY2MSwgMjAxMi8wMi8wNi0xNDo1NjoyNyAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wTU09Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9tbS8iIHhtbG5zOnN0UmVmPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvc1R5cGUvUmVzb3VyY2VSZWYjIiB4bWxuczp4bXA9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC8iIHhtcE1NOk9yaWdpbmFsRG9jdW1lbnRJRD0ieG1wLmRpZDpDNDk5ODU0N0U5MjA2ODExODhDNkJBNzRDM0U2QkE2NyIgeG1wTU06RG9jdW1lbnRJRD0ieG1wLmRpZDpFNjZGMDA2ODc5NEUxMUUxOTY2OEJEQjhGOUExQ0Y3RCIgeG1wTU06SW5zdGFuY2VJRD0ieG1wLmlpZDpFNjZDNDcxQTc5NEUxMUUxOTY2OEJEQjhGOUExQ0Y3RCIgeG1wOkNyZWF0b3JUb29sPSJBZG9iZSBQaG90b3Nob3AgQ1M2ICgxMy4wIDIwMTIwMzA1Lm0uNDE1IDIwMTIvMDMvMDU6MjE6MDA6MDApICAoTWFjaW50b3NoKSI+IDx4bXBNTTpEZXJpdmVkRnJvbSBzdFJlZjppbnN0YW5jZUlEPSJ4bXAuaWlkOkM1NzAxRDEzMjkyMTY4MTE4OEM2QkE3NEMzRTZCQTY3IiBzdFJlZjpkb2N1bWVudElEPSJ4bXAuZGlkOkM0OTk4NTQ3RTkyMDY4MTE4OEM2QkE3NEMzRTZCQTY3Ii8+IDwvcmRmOkRlc2NyaXB0aW9uPiA8L3JkZjpSREY+IDwveDp4bXBtZXRhPiA8P3hwYWNrZXQgZW5kPSJyIj8+l9tKdwAAAKZJREFUeNpi/P//P0NUm+d/BhxgWdV2RgY8gAXGMNMwwpA8deMcAyHAEljsdJhTmJ3h1YfnWBUA5f/j0X+E5d+//zYhrv5YZU108du+cNlKG6AB/xjefn7FQA4A6WX59/cfw5/ff8gz4C/IAKApf/78pswFv3/9Jt8Ff8FeIM+AvzAv/KbUC/8oC8T/DN9//iLTBf8ZWP7/+7f18MELVgzkgaMAAQYAgLlmT8qQW/sAAAAASUVORK5CYII=', - 'screens': 'iVBORw0KGgoAAAANSUhEUgAAABEAAAANCAYAAABPeYUaAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAA5NpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuMy1jMDExIDY2LjE0NTY2MSwgMjAxMi8wMi8wNi0xNDo1NjoyNyAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wTU09Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9tbS8iIHhtbG5zOnN0UmVmPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvc1R5cGUvUmVzb3VyY2VSZWYjIiB4bWxuczp4bXA9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC8iIHhtcE1NOk9yaWdpbmFsRG9jdW1lbnRJRD0ieG1wLmRpZDpDNDk5ODU0N0U5MjA2ODExODhDNkJBNzRDM0U2QkE2NyIgeG1wTU06RG9jdW1lbnRJRD0ieG1wLmRpZDo5NkU0QUUzNjc4NEExMUUxOTY2OEJEQjhGOUExQ0Y3RCIgeG1wTU06SW5zdGFuY2VJRD0ieG1wLmlpZDo5NkU0QUUzNTc4NEExMUUxOTY2OEJEQjhGOUExQ0Y3RCIgeG1wOkNyZWF0b3JUb29sPSJBZG9iZSBQaG90b3Nob3AgQ1M2ICgxMy4wIDIwMTIwMzA1Lm0uNDE1IDIwMTIvMDMvMDU6MjE6MDA6MDApICAoTWFjaW50b3NoKSI+IDx4bXBNTTpEZXJpdmVkRnJvbSBzdFJlZjppbnN0YW5jZUlEPSJ4bXAuaWlkOkM1NzAxRDEzMjkyMTY4MTE4OEM2QkE3NEMzRTZCQTY3IiBzdFJlZjpkb2N1bWVudElEPSJ4bXAuZGlkOkM0OTk4NTQ3RTkyMDY4MTE4OEM2QkE3NEMzRTZCQTY3Ii8+IDwvcmRmOkRlc2NyaXB0aW9uPiA8L3JkZjpSREY+IDwveDp4bXBtZXRhPiA8P3hwYWNrZXQgZW5kPSJyIj8+wkwZRwAAAQxJREFUeNpi/P//PwMjIyODqbHhfwYc4PTZ84y45ED6WZAFPFxdwQYig+27djEQAowgk6wtLcGu4GBnY0C38vvPX3gNOHr8OCPcJaHh4QwsLCwYiv78+YNV87+/fxnWrlkDZsN1PXr0iGHPnj1wRS4uLnj5jg4OcDYTjPH7928w3WxmhcJ3zmtC4Zc2mUH4SC6EG/LrF8TvtaeOofD3TqpD4XfXnULho3gHJOjt7Q2XePHiBV7+s6dPsRuydetWuISuri5evra2Nm7vVKUZoPDTratR+EZV1RjewTCkbdYFFP7Mo60o/HNtrSgBjeEdMzMzuMRToJ/x8Z88eYJpyKcPH8AYGRDiwwBAgAEAvXKdXsBF6t8AAAAASUVORK5CYII=' + 'screens': 'iVBORw0KGgoAAAANSUhEUgAAABEAAAANCAYAAABPeYUaAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAA5NpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuMy1jMDExIDY2LjE0NTY2MSwgMjAxMi8wMi8wNi0xNDo1NjoyNyAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wTU09Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9tbS8iIHhtbG5zOnN0UmVmPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvc1R5cGUvUmVzb3VyY2VSZWYjIiB4bWxuczp4bXA9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC8iIHhtcE1NOk9yaWdpbmFsRG9jdW1lbnRJRD0ieG1wLmRpZDpDNDk5ODU0N0U5MjA2ODExODhDNkJBNzRDM0U2QkE2NyIgeG1wTU06RG9jdW1lbnRJRD0ieG1wLmRpZDo5NkU0QUUzNjc4NEExMUUxOTY2OEJEQjhGOUExQ0Y3RCIgeG1wTU06SW5zdGFuY2VJRD0ieG1wLmlpZDo5NkU0QUUzNTc4NEExMUUxOTY2OEJEQjhGOUExQ0Y3RCIgeG1wOkNyZWF0b3JUb29sPSJBZG9iZSBQaG90b3Nob3AgQ1M2ICgxMy4wIDIwMTIwMzA1Lm0uNDE1IDIwMTIvMDMvMDU6MjE6MDA6MDApICAoTWFjaW50b3NoKSI+IDx4bXBNTTpEZXJpdmVkRnJvbSBzdFJlZjppbnN0YW5jZUlEPSJ4bXAuaWlkOkM1NzAxRDEzMjkyMTY4MTE4OEM2QkE3NEMzRTZCQTY3IiBzdFJlZjpkb2N1bWVudElEPSJ4bXAuZGlkOkM0OTk4NTQ3RTkyMDY4MTE4OEM2QkE3NEMzRTZCQTY3Ii8+IDwvcmRmOkRlc2NyaXB0aW9uPiA8L3JkZjpSREY+IDwveDp4bXBtZXRhPiA8P3hwYWNrZXQgZW5kPSJyIj8+wkwZRwAAAQxJREFUeNpi/P//PwMjIyODqbHhfwYc4PTZ84y45ED6WZAFPFxdwQYig+27djEQAowgk6wtLcGu4GBnY0C38vvPX3gNOHr8OCPcJaHh4QwsLCwYiv78+YNV87+/fxnWrlkDZsN1PXr0iGHPnj1wRS4uLnj5jg4OcDYTjPH7928w3WxmhcJ3zmtC4Zc2mUH4SC6EG/LrF8TvtaeOofD3TqpD4XfXnULho3gHJOjt7Q2XePHiBV7+s6dPsRuydetWuISuri5evra2Nm7vVKUZoPDTratR+EZV1RjewTCkbdYFFP7Mo60o/HNtrSgBjeEdMzMzuMRToJ/x8Z88eYJpyKcPH8AYGRDiwwBAgAEAvXKdXsBF6t8AAAAASUVORK5CYII=', + 'plates': 'iVBORw0KGgoAAAANSUhEUgAAAA8AAAANCAYAAAB2HjRBAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAASBJREFUeNpiZAACRVXNhnfv3tX//v2bgQjwAIgDv316d4FR38hM4MHDh++trSwZ+Pn4COr8+OkTw4GDhx4ANSuygARANj59+ozhxNFDKAplFVQYHj+4gyEGBAogguniuVMfkCUf7Z4IxsjgyqOJYIwOWGCM////g+mfj68woIt9/Ikphqr53z8wrZo0mwFdzFoVUwxF8z+giRbWDijOevjwAVYxrDafOHoARaGElBxWMRhgQvgF4px98x+BMbLYPSD/HpoYqrP/QQLi2Y2fDOhi37CIoYU2xMSYTlUGdDEdLGIgwAgiNHUM/r9//55BR1sbxX+PHj1kkJOTR43zq1cZBAUFGa5fuQDWy3D69Jn7IAO4+IQIYpA6kHqQPoAAAwCQE6mYLjTwJwAAAABJRU5ErkJggg==' } self.image_codes = {} for name, dat in image_data.items(): decodedImgData = base64.b64decode(dat) bio = BytesIO(decodedImgData) img = wx.Image(bio) + img = img.Scale(16, 13) self.image_codes[name] = image_list.Add(img.ConvertToBitmap()) self.tree.AssignImageList(image_list) @@ -873,7 +877,7 @@ def __init__(self, *args, **kwargs): add_button.Bind(wx.EVT_BUTTON, self.add_selected_to_pipeline) vert_sizer.Add(add_button, 0, wx.ALIGN_RIGHT | wx.ALL, 5) self.image_controls.SetSizer(vert_sizer) - splitter.SplitVertically(self.browse_controls, self.image_controls, 200) + splitter.SplitVertically(self.browse_controls, self.image_controls, 300) self.Layout() @@ -956,11 +960,12 @@ def refresh_group_members(self): self.members_box.Clear() self.members_box.AppendItems(list(self.users_in_group.keys())) - def fetch_children(self, event): target_id = event.GetItem() + if self.tree.GetChildrenCount(target_id, recursively=False) > 0: + # Already loaded + return data = self.tree.GetItemData(target_id) - self.tree.DeleteChildren(target_id) subject_type = data['type'] target_type = self.levels.get(subject_type, None) if target_type is None: @@ -971,20 +976,29 @@ def fetch_children(self, event): sub_str = "orphaned=true&" else: sub_str = f"id={subject}&" - url = f"https://{self.credentials.server}/webclient/api/{target_type}/?{sub_str}experimenter_id=-1&page=0&group={self.current_group}&bsession={self.credentials.session_key}" - + if target_type == 'wells': + url = f"https://{self.credentials.server}/api/v0/m/plates/{subject}/wells/?bsession={self.credentials.session_key}" + else: + url = f"https://{self.credentials.server}/webclient/api/{target_type}/?{sub_str}experimenter_id=-1&page=0&group={self.current_group}&bsession={self.credentials.session_key}" try: result = requests.get(url, timeout=5) except requests.exceptions.ConnectTimeout: LOGGER.error("Server request timed out") return - result.raise_for_status() + except Exception: + LOGGER.error("Server request failed", exc_info=True) + return result = result.json() if 'images' in result: image_map = {entry['id']: entry['name'] for entry in result['images']} data['images'] = image_map self.tree.SetItemData(target_id, data) - self.populate_tree(result, target_id) + if 'meta' in result: + # This is the plates API + self.populate_tree_screen(result, target_id) + result = result['data'] + else: + self.populate_tree(result, target_id) def fetch_containers(self): url = f"https://{self.credentials.server}/webclient/api/containers/?id={self.current_user}&page=0&group={self.current_group}&bsession={self.credentials.session_key}" @@ -1097,9 +1111,31 @@ def populate_tree(self, data, parent=None): new_id = self.tree.AppendItem(parent, f"{entry['name']}", data=entry) if image is not None: self.tree.SetItemImage(new_id, image, wx.TreeItemIcon_Normal) - if entry.get('childCount', 0) > 0: + if entry.get('childCount', 0) > 0 or item_type == 'plates': self.tree.SetItemHasChildren(new_id) + def populate_tree_screen(self, data, parent=None): + # Tree data from screens API + wells = data['data'] + rows = string.ascii_uppercase + for well_dict in wells: + name = f"Well {rows[well_dict['Row']]}{well_dict['Column'] + 1:02}" + well_dict['type'] = 'wells' + well_id = self.tree.AppendItem(parent, name) + well_dict['images'] = {} + for field_dict in well_dict['WellSamples']: + image_data = field_dict['Image'] + image_id = image_data['@id'] + image_name = image_data['Name'] + refined_image_data = { + 'type': 'images', + 'id': image_id, + 'name': image_name + } + well_dict['images'][image_id] = image_name + self.tree.AppendItem(well_id, image_name, data=refined_image_data) + self.tree.SetItemData(well_id, well_dict) + class ImagePanel(wx.Panel): ''' @@ -1171,4 +1207,4 @@ def OnPaint(self, evt): # Todo: Make better drag selection # Todo: Split GUI into a helper module -# Todo: Handle wells/fields +# Todo: Thumbnail queue From 37c3933972df81861f6620e641e0b6adff558e5b Mon Sep 17 00:00:00 2001 From: David Stirling Date: Tue, 1 Aug 2023 14:02:38 +0100 Subject: [PATCH 06/32] Better drag selection --- active_plugins/omeroreader.py | 60 +++++++++++++++++++++++++++++++++-- 1 file changed, 58 insertions(+), 2 deletions(-) diff --git a/active_plugins/omeroreader.py b/active_plugins/omeroreader.py index 1a7489d..6a52cb3 100644 --- a/active_plugins/omeroreader.py +++ b/active_plugins/omeroreader.py @@ -863,7 +863,7 @@ def __init__(self, *args, **kwargs): vert_sizer = wx.BoxSizer(wx.VERTICAL) - self.tile_panel = wx.ScrolledWindow(self.image_controls, 1) + self.tile_panel = TilePanel(self.image_controls, 1) self.tile_panel.url_loader = self.url_loader self.tiler_sizer = wx.WrapSizer(wx.HORIZONTAL) @@ -1166,6 +1166,9 @@ def __init__(self, thumbnail, parent, omero_id, name, server, size=128): self.Bind(wx.EVT_LEFT_DOWN, self.select) self.Bind(wx.EVT_RIGHT_DOWN, self.right_click) self.SetClientSize((self.size_x, self.size_y)) + # We need to pass these events up to the parent panel. + self.Bind(wx.EVT_MOTION, self.pass_event) + self.Bind(wx.EVT_LEFT_UP, self.pass_event) def select(self, e): self.selected = not self.selected @@ -1173,6 +1176,11 @@ def select(self, e): e.StopPropagation() e.Skip() + @staticmethod + def pass_event(e): + e.ResumePropagation(1) + e.Skip() + def right_click(self, event): popupmenu = wx.Menu() add_file_item = popupmenu.Append(-1, "Add to file list") @@ -1205,6 +1213,54 @@ def OnPaint(self, evt): dc.DrawRectangle(0, 0, self.size_x, self.size_y) return dc -# Todo: Make better drag selection + + class TilePanel(wx.ScrolledWindow): + + def __init__(self, *args, **kwargs): + super(self.__class__, self).__init__(*args, **kwargs) + self.select_source = None + self.select_box = None + self.Bind(wx.EVT_MOTION, self.on_motion) + self.Bind(wx.EVT_PAINT, self.on_paint) + self.Bind(wx.EVT_LEFT_UP, self.on_release) + + def deselect_all(self): + for child in self.GetChildren(): + if isinstance(child, ImagePanel): + child.selected = False + + def on_motion(self, evt): + if not evt.LeftIsDown(): + self.select_source = None + self.select_box = None + return + self.SetFocusIgnoringChildren() + if self.select_source is None: + self.select_source = evt.Position + if not evt.ShiftDown(): + self.deselect_all() + return + else: + self.select_box = wx.Rect(self.select_source, evt.Position) + for child in self.GetChildren(): + if isinstance(child, ImagePanel): + if not evt.ShiftDown(): + child.selected = False + if child.GetRect().Intersects(self.select_box): + child.selected = True + self.Refresh() + + def on_release(self, e): + self.select_source = None + self.select_box = None + self.Refresh() + + def on_paint(self, e): + dc = wx.PaintDC(self) + dc.SetPen(wx.Pen("BLUE", 3, style=wx.PENSTYLE_SHORT_DASH)) + dc.SetBrush(wx.Brush("BLUE", style=wx.TRANSPARENT)) + if self.select_box is not None: + dc.DrawRectangle(self.select_box) + # Todo: Split GUI into a helper module # Todo: Thumbnail queue From e2d4320bd0e4ed7c0a77735586622a750cc692da Mon Sep 17 00:00:00 2001 From: David Stirling Date: Tue, 1 Aug 2023 17:07:29 +0100 Subject: [PATCH 07/32] Move GUI/login components to subdirectory --- active_plugins/omero_helper/__init__.py | 0 active_plugins/omero_helper/connect.py | 133 ++++ active_plugins/omero_helper/gui.py | 726 ++++++++++++++++++++ active_plugins/omeroreader.py | 846 +----------------------- 4 files changed, 865 insertions(+), 840 deletions(-) create mode 100644 active_plugins/omero_helper/__init__.py create mode 100644 active_plugins/omero_helper/connect.py create mode 100644 active_plugins/omero_helper/gui.py diff --git a/active_plugins/omero_helper/__init__.py b/active_plugins/omero_helper/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/active_plugins/omero_helper/connect.py b/active_plugins/omero_helper/connect.py new file mode 100644 index 0000000..63c46aa --- /dev/null +++ b/active_plugins/omero_helper/connect.py @@ -0,0 +1,133 @@ +import atexit +import os +import logging +import importlib.util + +from cellprofiler_core.preferences import config_read_typed, get_headless +from cellprofiler_core.preferences import get_omero_server, get_omero_port, get_omero_user +import omero + +LOGGER = logging.getLogger(__name__) + +TOKEN_MODULE = importlib.util.find_spec("omero_user_token") +TOKENS_AVAILABLE = TOKEN_MODULE is not None +if TOKENS_AVAILABLE: + # Only load and enable user tokens if dependency is installed + omero_user_token = importlib.util.module_from_spec(TOKEN_MODULE) + TOKEN_MODULE.loader.exec_module(omero_user_token) + + +def login(e=None, server=None): + CREDENTIALS.get_tokens() + if CREDENTIALS.tokens: + if server is None: + # URL didn't specify which server we want. Just try whichever token is available + server = list(CREDENTIALS.tokens.keys())[0] + connected = CREDENTIALS.try_token(server) + if get_headless(): + if connected: + LOGGER.info("Connected to ", CREDENTIALS.server) + else: + LOGGER.warning("Failed to connect, was user token invalid?") + return connected + else: + from .gui import login_gui + login_gui(connected, server=None) + + +class LoginHelper: + def __init__(self): + self.server = get_omero_server() + self.port = get_omero_port() + self.username = get_omero_user() + self.passwd = "" + self.session_key = None + self.session = None + self.client = None + self.container_service = None + self.tokens = {} + self.browser_window = None + atexit.register(self.shutdown) + + def get_tokens(self, path=None): + self.tokens.clear() + # Future versions of omero_user_token may support multiple tokens, so we code with that in mind. + if not TOKENS_AVAILABLE: + return + # Check the reader setting which disables tokens. + tokens_enabled = config_read_typed(f"Reader.OMERO.allow_token", bool) + if tokens_enabled is not None and not tokens_enabled: + return + # User tokens sadly default to the home directory. This would override that location. + py_home = os.environ['HOME'] + if path is not None: + os.environ['HOME'] = path + try: + LOGGER.info("Requesting token info") + token = omero_user_token.get_token() + server, port = token[token.find('@') + 1:].split(':') + port = int(port) + LOGGER.info("Connection to {}:{}".format(server, port)) + session_key = token[:token.find('@')] + self.tokens[server] = (server, port, session_key) + except Exception: + LOGGER.error("Failed to get user token", exc_info=True) + if path is not None: + os.environ['HOME'] = py_home + + def try_token(self, address): + if address not in self.tokens: + LOGGER.error(f"Token {address} not found") + return False + else: + server, port, session_key = self.tokens[address] + return self.login(server=server, port=port, session_key=session_key) + + def login(self, server=None, port=None, user=None, passwd=None, session_key=None): + self.client = omero.client(host=server, port=port) + if session_key is not None: + try: + self.session = self.client.joinSession(session_key) + self.client.enableKeepAlive(60) + self.session.detachOnDestroy() + self.server = server + self.port = port + self.session_key = session_key + except Exception as e: + LOGGER.error(f"Failed to join session, token may have expired: {e}") + self.client = None + self.session = None + return False + elif self.username is not None: + try: + self.session = self.client.createSession( + username=user, password=passwd) + self.client.enableKeepAlive(60) + self.session.detachOnDestroy() + self.session_key = self.client.getSessionId() + self.server = server + self.port = port + self.username = user + self.passwd = passwd + except Exception as e: + LOGGER.error(f"Failed to create session: {e}") + self.client = None + self.session = None + return False + else: + self.client = None + self.session = None + raise Exception( + "Not enough details to create a server connection.") + self.container_service = self.session.getContainerService() + return True + + def shutdown(self): + if self.client is not None: + try: + self.client.closeSession() + except Exception as e: + LOGGER.error("Failed to close OMERO session - ", e) + + +CREDENTIALS = LoginHelper() diff --git a/active_plugins/omero_helper/gui.py b/active_plugins/omero_helper/gui.py new file mode 100644 index 0000000..9d1866c --- /dev/null +++ b/active_plugins/omero_helper/gui.py @@ -0,0 +1,726 @@ +import base64 +import functools +import io +import logging +import os +import string + +import requests +import wx +import cellprofiler.gui.plugins_menu +from cellprofiler_core.preferences import config_read_typed, config_write_typed, \ + set_omero_server, set_omero_port, set_omero_user + +from .connect import CREDENTIALS, TOKENS_AVAILABLE, login + +LOGGER = logging.getLogger(__name__) + + +def get_display_server(): + return config_read_typed(f"Reader.OMERO.show_server", bool) + + +def set_display_server(value): + config_write_typed(f"Reader.OMERO.show_server", value, key_type=bool) + + +def login_gui(connected, server=None): + if connected: + from cellprofiler.gui.errordialog import show_warning + show_warning("Connected to OMERO", + f"A token was found and used to connect to the OMERO server at {CREDENTIALS.server}", + get_display_server, + set_display_server) + return + show_login_dlg(server=server) + + +def show_login_dlg(token=True, server=None): + app = wx.GetApp() + frame = app.GetTopWindow() + with OmeroLoginDlg(frame, title="Log into Omero", token=token, server=server) as dlg: + dlg.ShowModal() + + +def login_no_token(e): + show_login_dlg() + + +def browse(e): + if CREDENTIALS.client is None: + login() + app = wx.GetApp() + frame = app.GetTopWindow() + if CREDENTIALS.browser_window is None: + CREDENTIALS.browser_window = OmeroBrowseDlg(frame, title=f"Browse OMERO: {CREDENTIALS.server}") + CREDENTIALS.browser_window.Show() + else: + CREDENTIALS.browser_window.Raise() + + +def inject_plugin_menu_entries(): + cellprofiler.gui.plugins_menu.PLUGIN_MENU_ENTRIES.extend([ + (login, wx.NewId(), "Connect to OMERO", "Establish an OMERO connection"), + (login_no_token, wx.NewId(), "Connect to OMERO (no token)", "Establish an OMERO connection," + " but without using user tokens"), + (browse, wx.NewId(), "Browse OMERO for images", "Browse an OMERO server and add images to the pipeline") + ]) + + +class OmeroLoginDlg(wx.Dialog): + + def __init__(self, *args, token=True, server=None, **kwargs): + super(self.__class__, self).__init__(*args, **kwargs) + self.credentials = CREDENTIALS + self.token = token + self.SetSizer(wx.BoxSizer(wx.VERTICAL)) + if server is None: + server = self.credentials.server + sizer = wx.BoxSizer(wx.VERTICAL) + self.Sizer.Add(sizer, 1, wx.EXPAND | wx.ALL, 6) + sub_sizer = wx.BoxSizer(wx.HORIZONTAL) + sizer.Add(sub_sizer, 0, wx.EXPAND) + + max_width = 0 + max_height = 0 + for label in ("Server:", "Port:", "Username:", "Password:"): + w, h, _, _ = self.GetFullTextExtent(label) + max_width = max(w, max_width) + max_height = max(h, max_height) + + # Add extra padding + lsize = wx.Size(max_width + 5, max_height) + sub_sizer.Add( + wx.StaticText(self, label="Server:", size=lsize), + 0, wx.ALIGN_CENTER_VERTICAL) + self.omero_server_ctrl = wx.TextCtrl(self, value=server) + sub_sizer.Add(self.omero_server_ctrl, 1, wx.EXPAND) + + sizer.AddSpacer(5) + sub_sizer = wx.BoxSizer(wx.HORIZONTAL) + sizer.Add(sub_sizer, 0, wx.EXPAND) + sub_sizer.Add( + wx.StaticText(self, label="Port:", size=lsize), + 0, wx.ALIGN_CENTER_VERTICAL) + self.omero_port_ctrl = wx.lib.intctrl.IntCtrl(self, value=self.credentials.port) + sub_sizer.Add(self.omero_port_ctrl, 1, wx.EXPAND) + + sizer.AddSpacer(5) + sub_sizer = wx.BoxSizer(wx.HORIZONTAL) + sizer.Add(sub_sizer, 0, wx.EXPAND) + sub_sizer.Add( + wx.StaticText(self, label="User:", size=lsize), + 0, wx.ALIGN_CENTER_VERTICAL) + self.omero_user_ctrl = wx.TextCtrl(self, value=self.credentials.username) + sub_sizer.Add(self.omero_user_ctrl, 1, wx.EXPAND) + + sizer.AddSpacer(5) + sub_sizer = wx.BoxSizer(wx.HORIZONTAL) + sizer.Add(sub_sizer, 0, wx.EXPAND) + sub_sizer.Add( + wx.StaticText(self, label="Password:", size=lsize), + 0, wx.ALIGN_CENTER_VERTICAL) + self.omero_password_ctrl = wx.TextCtrl(self, value="", style=wx.TE_PASSWORD | wx.TE_PROCESS_ENTER) + self.omero_password_ctrl.Bind(wx.EVT_TEXT_ENTER, self.on_connect_pressed) + if self.credentials.username is not None: + self.omero_password_ctrl.SetFocus() + sub_sizer.Add(self.omero_password_ctrl, 1, wx.EXPAND) + + sizer.AddSpacer(5) + sub_sizer = wx.BoxSizer(wx.HORIZONTAL) + sizer.Add(sub_sizer, 0, wx.EXPAND) + connect_button = wx.Button(self, label="Connect") + connect_button.Bind(wx.EVT_BUTTON, self.on_connect_pressed) + sub_sizer.Add(connect_button, 0, wx.EXPAND) + sub_sizer.AddSpacer(5) + + self.message_ctrl = wx.StaticText(self, label="Not connected") + sub_sizer.Add(self.message_ctrl, 1, wx.EXPAND) + + self.token_button = wx.Button(self, label="Set Token") + self.token_button.Bind(wx.EVT_BUTTON, self.on_set_pressed) + self.token_button.Disable() + self.token_button.SetToolTip("Use these credentials to set a long-lasting token for automatic login") + sub_sizer.Add(self.token_button, 0, wx.EXPAND) + sub_sizer.AddSpacer(5) + + button_sizer = wx.StdDialogButtonSizer() + self.Sizer.Add(button_sizer, 0, wx.EXPAND) + + cancel_button = wx.Button(self, wx.ID_CANCEL) + button_sizer.AddButton(cancel_button) + cancel_button.Bind(wx.EVT_BUTTON, self.on_cancel) + + self.ok_button = wx.Button(self, wx.ID_OK) + button_sizer.AddButton(self.ok_button) + self.ok_button.Bind(wx.EVT_BUTTON, self.on_ok) + self.ok_button.Enable(False) + button_sizer.Realize() + + self.omero_password_ctrl.Bind(wx.EVT_TEXT, self.mark_dirty) + self.omero_port_ctrl.Bind(wx.EVT_TEXT, self.mark_dirty) + self.omero_server_ctrl.Bind(wx.EVT_TEXT, self.mark_dirty) + self.omero_user_ctrl.Bind(wx.EVT_TEXT, self.mark_dirty) + self.Layout() + + def mark_dirty(self, event): + if self.ok_button.IsEnabled(): + self.ok_button.Enable(False) + self.message_ctrl.Label = "Please connect with your new credentials" + self.message_ctrl.ForegroundColour = "black" + + def on_connect_pressed(self, event): + if self.credentials.client is not None and self.credentials.server == self.omero_server_ctrl.GetValue(): + # Already connected, accept another 'Connect' command as an ok to close + self.EndModal(wx.OK) + self.connect() + + def on_set_pressed(self, event): + if self.credentials.client is None or not TOKENS_AVAILABLE: + return + import omero_user_token + token_path = omero_user_token.assert_and_get_token_path() + if os.path.exists(token_path): + dlg2 = wx.MessageDialog(self, + "Existing omero_user_token will be overwritten. Proceed?", + "Overwrite existing token?", + wx.YES_NO | wx.CANCEL | wx.ICON_WARNING) + result = dlg2.ShowModal() + if result != wx.ID_YES: + LOGGER.debug("Cancelled") + return + token = omero_user_token.setter( + self.credentials.server, + self.credentials.port, + self.credentials.username, + self.credentials.passwd, + -1) + if token: + LOGGER.info("Set OMERO user token") + self.message_ctrl.Label = "Connected. Token Set!" + self.message_ctrl.ForegroundColour = "forest green" + else: + LOGGER.error("Failed to set OMERO user token") + self.message_ctrl.Label = "Failed to set token." + self.message_ctrl.ForegroundColour = "red" + self.message_ctrl.Refresh() + + def connect(self): + try: + server = self.omero_server_ctrl.GetValue() + port = self.omero_port_ctrl.GetValue() + user = self.omero_user_ctrl.GetValue() + passwd = self.omero_password_ctrl.GetValue() + except: + self.message_ctrl.Label = ( + "The port number must be an integer between 0 and 65535 (try 4064)" + ) + self.message_ctrl.ForegroundColour = "red" + self.message_ctrl.Refresh() + return False + self.message_ctrl.ForegroundColour = "black" + self.message_ctrl.Label = "Connecting..." + self.message_ctrl.Refresh() + # Allow UI to update before connecting + wx.Yield() + success = self.credentials.login(server, port, user, passwd) + if success: + self.message_ctrl.Label = "Connected" + self.message_ctrl.ForegroundColour = "forest green" + self.token_button.Enable() + self.message_ctrl.Refresh() + set_omero_server(server) + set_omero_port(port) + set_omero_user(user) + self.ok_button.Enable(True) + return True + else: + self.message_ctrl.Label = "Failed to log onto server" + self.message_ctrl.ForegroundColour = "red" + self.message_ctrl.Refresh() + self.token_button.Disable() + return False + + def on_cancel(self, event): + self.EndModal(wx.CANCEL) + + def on_ok(self, event): + self.EndModal(wx.OK) + + +class OmeroBrowseDlg(wx.Frame): + def __init__(self, *args, **kwargs): + super(self.__class__, self).__init__(*args, + style=wx.RESIZE_BORDER | wx.CAPTION | wx.CLOSE_BOX, + size=(900, 600), + **kwargs) + self.credentials = CREDENTIALS + self.admin_service = self.credentials.session.getAdminService() + self.url_loader = self.Parent.pipeline.add_urls + + self.Bind(wx.EVT_CLOSE, self.close_browser) + + ec = self.admin_service.getEventContext() + # Exclude sys groups + self.groups = [self.admin_service.getGroup(v) for v in ec.memberOfGroups if v > 1] + self.group_names = [group.name.val for group in self.groups] + self.current_group = self.groups[0].id.getValue() + self.users_in_group = {'All Members': -1} + self.users_in_group.update({ + x.omeName.val: x.id.val for x in self.groups[0].linkedExperimenterList() + }) + self.current_user = -1 + self.levels = {'projects': 'datasets', + 'datasets': 'images', + 'screens': 'plates', + 'plates': 'wells', + 'orphaned': 'images' + } + + splitter = wx.SplitterWindow(self, -1, style=wx.SP_BORDER) + self.browse_controls = wx.Panel(splitter, -1) + b = wx.BoxSizer(wx.VERTICAL) + + self.groups_box = wx.Choice(self.browse_controls, choices=self.group_names) + self.groups_box.Bind(wx.EVT_CHOICE, self.switch_group) + + self.members_box = wx.Choice(self.browse_controls, choices=list(self.users_in_group.keys())) + self.members_box.Bind(wx.EVT_CHOICE, self.switch_member) + + b.Add(self.groups_box, 0, wx.EXPAND) + b.Add(self.members_box, 0, wx.EXPAND) + self.container = self.credentials.session.getContainerService() + + self.tree = wx.TreeCtrl(self.browse_controls, style=wx.TR_HAS_BUTTONS | wx.TR_HIDE_ROOT) + image_list = wx.ImageList(16, 13) + image_data = { + 'projects': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAANCAYAAACgu+4kAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAA5NpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuMy1jMDExIDY2LjE0NTY2MSwgMjAxMi8wMi8wNi0xNDo1NjoyNyAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wTU09Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9tbS8iIHhtbG5zOnN0UmVmPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvc1R5cGUvUmVzb3VyY2VSZWYjIiB4bWxuczp4bXA9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC8iIHhtcE1NOk9yaWdpbmFsRG9jdW1lbnRJRD0ieG1wLmRpZDpDNDk5ODU0N0U5MjA2ODExODhDNkJBNzRDM0U2QkE2NyIgeG1wTU06RG9jdW1lbnRJRD0ieG1wLmRpZDpFNjZDNDcxNzc5NEUxMUUxOTY2OEJEQjhGOUExQ0Y3RCIgeG1wTU06SW5zdGFuY2VJRD0ieG1wLmlpZDpFNjZDNDcxNjc5NEUxMUUxOTY2OEJEQjhGOUExQ0Y3RCIgeG1wOkNyZWF0b3JUb29sPSJBZG9iZSBQaG90b3Nob3AgQ1M2ICgxMy4wIDIwMTIwMzA1Lm0uNDE1IDIwMTIvMDMvMDU6MjE6MDA6MDApICAoTWFjaW50b3NoKSI+IDx4bXBNTTpEZXJpdmVkRnJvbSBzdFJlZjppbnN0YW5jZUlEPSJ4bXAuaWlkOkM1NzAxRDEzMjkyMTY4MTE4OEM2QkE3NEMzRTZCQTY3IiBzdFJlZjpkb2N1bWVudElEPSJ4bXAuZGlkOkM0OTk4NTQ3RTkyMDY4MTE4OEM2QkE3NEMzRTZCQTY3Ii8+IDwvcmRmOkRlc2NyaXB0aW9uPiA8L3JkZjpSREY+IDwveDp4bXBtZXRhPiA8P3hwYWNrZXQgZW5kPSJyIj8+vd9MhwAAALhJREFUeNpi/P//P0NEXsd/BhxgxaQKRgY8gAXGMNbVwpA8e/kaAyHAGJhWe5iNnctGVkoaQ/Lxs6cMv35+w6f/CMvfP39svDytscrqaijgtX3t5u02IAMYXrx5z0AOAOll+fPnF8Ov37/IMgCkl+XP799Af5JpAFAv0IBfDD9//STTAKALfgMJcl0A0gvxwi8KvPAXFIhkGvAXHIjAqPjx4zuZsQCMxn9//my9eOaEFQN54ChAgAEAzRBnWnEZWFQAAAAASUVORK5CYII=', + 'datasets': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAANCAYAAACgu+4kAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAA5NpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuMy1jMDExIDY2LjE0NTY2MSwgMjAxMi8wMi8wNi0xNDo1NjoyNyAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wTU09Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9tbS8iIHhtbG5zOnN0UmVmPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvc1R5cGUvUmVzb3VyY2VSZWYjIiB4bWxuczp4bXA9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC8iIHhtcE1NOk9yaWdpbmFsRG9jdW1lbnRJRD0ieG1wLmRpZDpDNDk5ODU0N0U5MjA2ODExODhDNkJBNzRDM0U2QkE2NyIgeG1wTU06RG9jdW1lbnRJRD0ieG1wLmRpZDpFNjZGMDA2ODc5NEUxMUUxOTY2OEJEQjhGOUExQ0Y3RCIgeG1wTU06SW5zdGFuY2VJRD0ieG1wLmlpZDpFNjZDNDcxQTc5NEUxMUUxOTY2OEJEQjhGOUExQ0Y3RCIgeG1wOkNyZWF0b3JUb29sPSJBZG9iZSBQaG90b3Nob3AgQ1M2ICgxMy4wIDIwMTIwMzA1Lm0uNDE1IDIwMTIvMDMvMDU6MjE6MDA6MDApICAoTWFjaW50b3NoKSI+IDx4bXBNTTpEZXJpdmVkRnJvbSBzdFJlZjppbnN0YW5jZUlEPSJ4bXAuaWlkOkM1NzAxRDEzMjkyMTY4MTE4OEM2QkE3NEMzRTZCQTY3IiBzdFJlZjpkb2N1bWVudElEPSJ4bXAuZGlkOkM0OTk4NTQ3RTkyMDY4MTE4OEM2QkE3NEMzRTZCQTY3Ii8+IDwvcmRmOkRlc2NyaXB0aW9uPiA8L3JkZjpSREY+IDwveDp4bXBtZXRhPiA8P3hwYWNrZXQgZW5kPSJyIj8+l9tKdwAAAKZJREFUeNpi/P//P0NUm+d/BhxgWdV2RgY8gAXGMNMwwpA8deMcAyHAEljsdJhTmJ3h1YfnWBUA5f/j0X+E5d+//zYhrv5YZU108du+cNlKG6AB/xjefn7FQA4A6WX59/cfw5/ff8gz4C/IAKApf/78pswFv3/9Jt8Ff8FeIM+AvzAv/KbUC/8oC8T/DN9//iLTBf8ZWP7/+7f18MELVgzkgaMAAQYAgLlmT8qQW/sAAAAASUVORK5CYII=', + 'screens': 'iVBORw0KGgoAAAANSUhEUgAAABEAAAANCAYAAABPeYUaAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAA5NpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuMy1jMDExIDY2LjE0NTY2MSwgMjAxMi8wMi8wNi0xNDo1NjoyNyAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wTU09Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9tbS8iIHhtbG5zOnN0UmVmPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvc1R5cGUvUmVzb3VyY2VSZWYjIiB4bWxuczp4bXA9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC8iIHhtcE1NOk9yaWdpbmFsRG9jdW1lbnRJRD0ieG1wLmRpZDpDNDk5ODU0N0U5MjA2ODExODhDNkJBNzRDM0U2QkE2NyIgeG1wTU06RG9jdW1lbnRJRD0ieG1wLmRpZDo5NkU0QUUzNjc4NEExMUUxOTY2OEJEQjhGOUExQ0Y3RCIgeG1wTU06SW5zdGFuY2VJRD0ieG1wLmlpZDo5NkU0QUUzNTc4NEExMUUxOTY2OEJEQjhGOUExQ0Y3RCIgeG1wOkNyZWF0b3JUb29sPSJBZG9iZSBQaG90b3Nob3AgQ1M2ICgxMy4wIDIwMTIwMzA1Lm0uNDE1IDIwMTIvMDMvMDU6MjE6MDA6MDApICAoTWFjaW50b3NoKSI+IDx4bXBNTTpEZXJpdmVkRnJvbSBzdFJlZjppbnN0YW5jZUlEPSJ4bXAuaWlkOkM1NzAxRDEzMjkyMTY4MTE4OEM2QkE3NEMzRTZCQTY3IiBzdFJlZjpkb2N1bWVudElEPSJ4bXAuZGlkOkM0OTk4NTQ3RTkyMDY4MTE4OEM2QkE3NEMzRTZCQTY3Ii8+IDwvcmRmOkRlc2NyaXB0aW9uPiA8L3JkZjpSREY+IDwveDp4bXBtZXRhPiA8P3hwYWNrZXQgZW5kPSJyIj8+wkwZRwAAAQxJREFUeNpi/P//PwMjIyODqbHhfwYc4PTZ84y45ED6WZAFPFxdwQYig+27djEQAowgk6wtLcGu4GBnY0C38vvPX3gNOHr8OCPcJaHh4QwsLCwYiv78+YNV87+/fxnWrlkDZsN1PXr0iGHPnj1wRS4uLnj5jg4OcDYTjPH7928w3WxmhcJ3zmtC4Zc2mUH4SC6EG/LrF8TvtaeOofD3TqpD4XfXnULho3gHJOjt7Q2XePHiBV7+s6dPsRuydetWuISuri5evra2Nm7vVKUZoPDTratR+EZV1RjewTCkbdYFFP7Mo60o/HNtrSgBjeEdMzMzuMRToJ/x8Z88eYJpyKcPH8AYGRDiwwBAgAEAvXKdXsBF6t8AAAAASUVORK5CYII=', + 'plates': 'iVBORw0KGgoAAAANSUhEUgAAAA8AAAANCAYAAAB2HjRBAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAASBJREFUeNpiZAACRVXNhnfv3tX//v2bgQjwAIgDv316d4FR38hM4MHDh++trSwZ+Pn4COr8+OkTw4GDhx4ANSuygARANj59+ozhxNFDKAplFVQYHj+4gyEGBAogguniuVMfkCUf7Z4IxsjgyqOJYIwOWGCM////g+mfj68woIt9/Ikphqr53z8wrZo0mwFdzFoVUwxF8z+giRbWDijOevjwAVYxrDafOHoARaGElBxWMRhgQvgF4px98x+BMbLYPSD/HpoYqrP/QQLi2Y2fDOhi37CIoYU2xMSYTlUGdDEdLGIgwAgiNHUM/r9//55BR1sbxX+PHj1kkJOTR43zq1cZBAUFGa5fuQDWy3D69Jn7IAO4+IQIYpA6kHqQPoAAAwCQE6mYLjTwJwAAAABJRU5ErkJggg==' + } + self.image_codes = {} + for name, dat in image_data.items(): + decodedImgData = base64.b64decode(dat) + bio = io.BytesIO(decodedImgData) + img = wx.Image(bio) + img = img.Scale(16, 13) + self.image_codes[name] = image_list.Add(img.ConvertToBitmap()) + self.tree.AssignImageList(image_list) + + self.tree.Bind(wx.EVT_TREE_ITEM_EXPANDING, self.fetch_children) + self.tree.Bind(wx.EVT_TREE_SEL_CHANGING, self.select_tree) + self.tree.Bind(wx.EVT_TREE_SEL_CHANGED, self.update_thumbnails) + self.tree.Bind(wx.EVT_TREE_BEGIN_DRAG, self.process_drag) + + data = self.fetch_containers() + + self.populate_tree(data) + b.Add(self.tree, 1, wx.EXPAND) + + self.browse_controls.SetSizer(b) + + self.image_controls = wx.Panel(splitter, -1) + + vert_sizer = wx.BoxSizer(wx.VERTICAL) + + self.tile_panel = TilePanel(self.image_controls, 1) + self.tile_panel.url_loader = self.url_loader + + self.tiler_sizer = wx.WrapSizer(wx.HORIZONTAL) + self.tile_panel.SetSizer(self.tiler_sizer) + self.tile_panel.SetScrollbars(0, 20, 0, 20) + + self.update_thumbnails() + vert_sizer.Add(self.tile_panel, wx.EXPAND) + + add_button = wx.Button(self.image_controls, wx.NewId(), "Add to file list") + add_button.Bind(wx.EVT_BUTTON, self.add_selected_to_pipeline) + vert_sizer.Add(add_button, 0, wx.ALIGN_RIGHT | wx.ALL, 5) + self.image_controls.SetSizer(vert_sizer) + splitter.SplitVertically(self.browse_controls, self.image_controls, 300) + + self.Layout() + + def close_browser(self, event): + self.credentials.browser_window = None + event.Skip() + + def add_selected_to_pipeline(self, e=None): + displayed = self.tile_panel.GetChildren() + all_urls = [] + selected_urls = [] + for item in displayed: + if isinstance(item, ImagePanel): + all_urls.append(item.url) + if item.selected: + selected_urls.append(item.url) + if selected_urls: + self.url_loader(selected_urls) + else: + self.url_loader(all_urls) + + def process_drag(self, event): + # We have our own custom handler here + data = wx.FileDataObject() + for file_url in self.fetch_file_list_from_tree(event): + data.AddFile(file_url) + drop_src = wx.DropSource(self) + drop_src.SetData(data) + drop_src.DoDragDrop(wx.Drag_CopyOnly) + + def fetch_file_list_from_tree(self, event): + files = [] + + def recurse_for_images(tree_id): + if not self.tree.IsExpanded(tree_id): + self.tree.Expand(tree_id) + data = self.tree.GetItemData(tree_id) + item_type = data['type'] + if item_type == 'images': + files.append(f"https://{self.credentials.server}/webclient/?show=image-{data['id']}") + elif 'images' in data: + for omero_id, _ in data['images'].items(): + files.append(f"https://{self.credentials.server}/webclient/?show=image-{omero_id}") + else: + child_id, cookie = self.tree.GetFirstChild(tree_id) + while child_id.IsOk(): + recurse_for_images(child_id) + child_id, cookie = self.tree.GetNextChild(tree_id, cookie) + + recurse_for_images(event.GetItem()) + return files + + def select_tree(self, event): + target_id = event.GetItem() + self.tree.Expand(target_id) + + def next_level(self, level): + return self.levels.get(level, None) + + def switch_group(self, e=None): + new_group = self.groups_box.GetCurrentSelection() + self.current_group = self.groups[new_group].id.getValue() + self.current_user = -1 + self.refresh_group_members() + data = self.fetch_containers() + self.populate_tree(data) + + def switch_member(self, e=None): + new_member = self.members_box.GetStringSelection() + self.current_user = self.users_in_group.get(new_member, -1) + data = self.fetch_containers() + self.populate_tree(data) + + def refresh_group_members(self): + self.users_in_group = {'All Members': -1} + group = self.groups[self.groups_box.GetCurrentSelection()] + self.users_in_group.update({ + x.omeName.val: x.id.val for x in group.linkedExperimenterList() + }) + self.members_box.Clear() + self.members_box.AppendItems(list(self.users_in_group.keys())) + + def fetch_children(self, event): + target_id = event.GetItem() + if self.tree.GetChildrenCount(target_id, recursively=False) > 0: + # Already loaded + return + data = self.tree.GetItemData(target_id) + subject_type = data['type'] + target_type = self.levels.get(subject_type, None) + if target_type is None: + # We're at the bottom level already + return + subject = data['id'] + if subject == -1: + sub_str = "orphaned=true&" + else: + sub_str = f"id={subject}&" + if target_type == 'wells': + url = f"https://{self.credentials.server}/api/v0/m/plates/{subject}/wells/?bsession={self.credentials.session_key}" + else: + url = f"https://{self.credentials.server}/webclient/api/{target_type}/?{sub_str}experimenter_id=-1&page=0&group={self.current_group}&bsession={self.credentials.session_key}" + try: + result = requests.get(url, timeout=5) + except requests.exceptions.ConnectTimeout: + LOGGER.error("Server request timed out") + return + except Exception: + LOGGER.error("Server request failed", exc_info=True) + return + result = result.json() + if 'images' in result: + image_map = {entry['id']: entry['name'] for entry in result['images']} + data['images'] = image_map + self.tree.SetItemData(target_id, data) + if 'meta' in result: + # This is the plates API + self.populate_tree_screen(result, target_id) + result = result['data'] + else: + self.populate_tree(result, target_id) + + def fetch_containers(self): + url = f"https://{self.credentials.server}/webclient/api/containers/?id={self.current_user}&page=0&group={self.current_group}&bsession={self.credentials.session_key}" + try: + data = requests.get(url, timeout=5) + except requests.exceptions.ConnectTimeout: + LOGGER.error("Server request timed out") + return {} + data.raise_for_status() + return data.json() + + def fetch_thumbnails(self, id_list): + if not id_list: + return {} + id_list = [str(x) for x in id_list] + chunk_size = 10 + buffer = {x: "" for x in id_list} + for i in range(0, len(id_list), chunk_size): + ids_to_get = '&id='.join(id_list[i:i + chunk_size]) + url = f"https://{self.credentials.server}/webclient/get_thumbnails/128/?&bsession={self.credentials.session_key}&id={ids_to_get}" + LOGGER.debug(f"Fetching {url}") + try: + data = requests.get(url, timeout=5) + except requests.exceptions.ConnectTimeout: + continue + if data.status_code != 200: + LOGGER.warning(f"Server error: {data.status_code} - {data.reason}") + else: + buffer.update(data.json()) + return buffer + + @functools.lru_cache(maxsize=20) + def fetch_large_thumbnail(self, id): + # Get a large thumbnail for single image display mode. We cache the last 20. + url = f"https://{self.credentials.server}/webgateway/render_thumbnail/{id}/450/450/?bsession={self.credentials.session_key}" + LOGGER.debug(f"Fetching {url}") + try: + data = requests.get(url, timeout=5) + except requests.exceptions.ConnectTimeout: + LOGGER.error("Thumbnail request timed out") + return False + if data.status_code != 200: + LOGGER.warning("Server error:", data.status_code, data.reason) + return False + elif not data.content: + return False + io_bytes = io.BytesIO(data.content) + return wx.Image(io_bytes) + + def update_thumbnails(self, event=None): + self.tiler_sizer.Clear(delete_windows=True) + if not event: + return + target_id = event.GetItem() + item_data = self.tree.GetItemData(target_id) + if item_data.get('type', None) == 'images': + image_id = item_data['id'] + img_name = item_data['name'] + thumb_img = self.fetch_large_thumbnail(image_id) + if not thumb_img or not thumb_img.IsOk(): + thumb_img = self.get_error_thumbnail(450) + else: + thumb_img = thumb_img.ConvertToBitmap() + tile = ImagePanel(thumb_img, self.tile_panel, image_id, img_name, self.credentials.server, size=450) + tile.selected = True + self.tiler_sizer.Add(tile, 0, wx.ALL, 5) + else: + image_targets = item_data.get('images', {}) + if not image_targets: + return + + id_list = list(image_targets.keys()) + data = self.fetch_thumbnails(id_list) + for image_id, image_data in data.items(): + img_name = image_targets[int(image_id)] + start_data = image_data.find('/9') + if start_data == -1: + img = self.get_error_thumbnail(128) + else: + decoded = base64.b64decode(image_data[start_data:]) + bio = io.BytesIO(decoded) + img = wx.Image(bio) + if not img.IsOk(): + img = self.get_error_thumbnail(128) + else: + img = img.ConvertToBitmap() + tile = ImagePanel(img, self.tile_panel, image_id, img_name, self.credentials.server) + self.tiler_sizer.Add(tile, 0, wx.ALL, 5) + self.tiler_sizer.Layout() + self.image_controls.Layout() + self.image_controls.Refresh() + + @functools.lru_cache(maxsize=10) + def get_error_thumbnail(self, size): + # Draw an image with an error icon. Cache the result since we may need the error icon repeatedly. + artist = wx.ArtProvider() + size //= 2 + return artist.GetBitmap(wx.ART_WARNING, size=(size, size)) + + def populate_tree(self, data, parent=None): + if parent is None: + self.tree.DeleteAllItems() + parent = self.tree.AddRoot("Server") + for item_type, items in data.items(): + image = self.image_codes.get(item_type, None) + if not isinstance(items, list): + items = [items] + for entry in items: + entry['type'] = item_type + new_id = self.tree.AppendItem(parent, f"{entry['name']}", data=entry) + if image is not None: + self.tree.SetItemImage(new_id, image, wx.TreeItemIcon_Normal) + if entry.get('childCount', 0) > 0 or item_type == 'plates': + self.tree.SetItemHasChildren(new_id) + + def populate_tree_screen(self, data, parent=None): + # Tree data from screens API + wells = data['data'] + rows = string.ascii_uppercase + for well_dict in wells: + name = f"Well {rows[well_dict['Row']]}{well_dict['Column'] + 1:02}" + well_dict['type'] = 'wells' + well_id = self.tree.AppendItem(parent, name) + well_dict['images'] = {} + for field_dict in well_dict['WellSamples']: + image_data = field_dict['Image'] + image_id = image_data['@id'] + image_name = image_data['Name'] + refined_image_data = { + 'type': 'images', + 'id': image_id, + 'name': image_name + } + well_dict['images'][image_id] = image_name + self.tree.AppendItem(well_id, image_name, data=refined_image_data) + self.tree.SetItemData(well_id, well_dict) + + +class ImagePanel(wx.Panel): + ''' + ImagePanels are wxPanels that display a wxBitmap and store multiple + image channels which can be recombined to mix different bitmaps. + ''' + + def __init__(self, thumbnail, parent, omero_id, name, server, size=128): + """ + thumbnail -- wx Bitmap + parent -- parent window to the wx.Panel + + """ + self.parent = parent + self.bitmap = thumbnail + self.selected = False + self.omero_id = omero_id + self.url = f"https://{server}/webclient/?show=image-{omero_id}" + self.name = name + if len(name) > 17: + self.shortname = name[:14] + '...' + else: + self.shortname = name + self.size_x = size + self.size_y = size + 30 + wx.Panel.__init__(self, parent, wx.NewId(), size=(self.size_x, self.size_y)) + self.Bind(wx.EVT_PAINT, self.OnPaint) + self.Bind(wx.EVT_LEFT_DOWN, self.select) + self.Bind(wx.EVT_RIGHT_DOWN, self.right_click) + self.SetClientSize((self.size_x, self.size_y)) + # We need to pass these events up to the parent panel. + self.Bind(wx.EVT_MOTION, self.pass_event) + self.Bind(wx.EVT_LEFT_UP, self.pass_event) + + def select(self, e): + self.selected = not self.selected + self.Refresh() + e.StopPropagation() + e.Skip() + + def pass_event(self, e): + x, y = e.GetPosition() + w, h = self.GetPosition() + e.SetPosition((x + w, y + h)) + e.ResumePropagation(1) + e.Skip() + + def right_click(self, event): + popupmenu = wx.Menu() + add_file_item = popupmenu.Append(-1, "Add to file list") + self.Bind(wx.EVT_MENU, self.add_to_pipeline, add_file_item) + add_file_item = popupmenu.Append(-1, "Show in OMERO.web") + self.Bind(wx.EVT_MENU, self.open_in_browser, add_file_item) + # Show menu + self.PopupMenu(popupmenu, event.GetPosition()) + + def add_to_pipeline(self, e): + self.parent.url_loader([self.url]) + + def open_in_browser(self, e): + wx.LaunchDefaultBrowser(self.url) + + def OnPaint(self, evt): + dc = wx.PaintDC(self) + dc.Clear() + dc.DrawBitmap(self.bitmap, (self.size_x - self.bitmap.Width) // 2, + ((self.size_x - self.bitmap.Height) // 2) + 20) + rect = wx.Rect(0, 0, self.size_x, self.size_x + 20) + dc.DrawLabel(self.shortname, rect, alignment=wx.ALIGN_CENTER_HORIZONTAL | wx.ALIGN_TOP) + dc.SetPen(wx.Pen("GREY", style=wx.PENSTYLE_SOLID)) + dc.SetBrush(wx.Brush("BLACK", wx.TRANSPARENT)) + dc.DrawRectangle(rect) + # Outline the whole image + if self.selected: + dc.SetPen(wx.Pen("BLUE", 3)) + dc.SetBrush(wx.Brush("BLACK", style=wx.TRANSPARENT)) + dc.DrawRectangle(0, 0, self.size_x, self.size_y) + return dc + + +class TilePanel(wx.ScrolledWindow): + + def __init__(self, *args, **kwargs): + super(self.__class__, self).__init__(*args, **kwargs) + self.select_source = None + self.select_box = None + self.Bind(wx.EVT_MOTION, self.on_motion) + self.Bind(wx.EVT_PAINT, self.on_paint) + self.Bind(wx.EVT_LEFT_UP, self.on_release) + + def deselect_all(self): + for child in self.GetChildren(): + if isinstance(child, ImagePanel): + child.selected = False + + def on_motion(self, evt): + if not evt.LeftIsDown(): + self.select_source = None + self.select_box = None + return + self.SetFocusIgnoringChildren() + if self.select_source is None: + self.select_source = evt.Position + if not evt.ShiftDown(): + self.deselect_all() + return + else: + self.select_box = wx.Rect(self.select_source, evt.Position) + for child in self.GetChildren(): + if isinstance(child, ImagePanel): + if not evt.ShiftDown(): + child.selected = False + if child.GetRect().Intersects(self.select_box): + child.selected = True + self.Refresh() + + def on_release(self, e): + self.select_source = None + self.select_box = None + self.Refresh() + + def on_paint(self, e): + dc = wx.PaintDC(self) + dc.SetPen(wx.Pen("BLUE", 3, style=wx.PENSTYLE_SHORT_DASH)) + dc.SetBrush(wx.Brush("BLUE", style=wx.TRANSPARENT)) + if self.select_box is not None: + dc.DrawRectangle(self.select_box) diff --git a/active_plugins/omeroreader.py b/active_plugins/omeroreader.py index 6a52cb3..1f5e403 100644 --- a/active_plugins/omeroreader.py +++ b/active_plugins/omeroreader.py @@ -49,15 +49,7 @@ server connections from timing out while CellProfiler is running, though you may need to reconnect if the PC goes to sleep. """ -import base64 -import functools -import io -import os import collections -import atexit -import string -from io import BytesIO -import requests import urllib.parse from struct import unpack @@ -67,24 +59,18 @@ from cellprofiler_core.preferences import get_headless from cellprofiler_core.constants.image import MD_SIZE_S, MD_SIZE_C, MD_SIZE_Z, MD_SIZE_T, \ MD_SIZE_Y, MD_SIZE_X, MD_SERIES_NAME -from cellprofiler_core.preferences import get_omero_server, get_omero_port, get_omero_user, set_omero_server,\ - set_omero_port, set_omero_user, config_read_typed, config_write_typed from cellprofiler_core.constants.image import PASSTHROUGH_SCHEMES from cellprofiler_core.reader import Reader -import omero import logging import re -import importlib.util - -TOKEN_MODULE = importlib.util.find_spec("omero_user_token") -TOKENS_AVAILABLE = TOKEN_MODULE is not None -if TOKENS_AVAILABLE: - # Only load and enable user tokens if dependency is installed - omero_user_token = importlib.util.module_from_spec(TOKEN_MODULE) - TOKEN_MODULE.loader.exec_module(omero_user_token) +from omero_helper.connect import login, CREDENTIALS +if not get_headless(): + # Load the GUI components and add the plugin menu options + from omero_helper.gui import inject_plugin_menu_entries + inject_plugin_menu_entries() REGEX_INDEX_FROM_FILE_NAME = re.compile(r'\?show=image-(\d+)') @@ -109,7 +95,7 @@ class OMEROReader(Reader): """ Reads images from an OMERO server. """ - reader_name = "OMERO Reader" + reader_name = "OMERO" variable_revision_number = 1 supported_filetypes = {} supported_schemes = {'omero', 'http', 'https'} @@ -443,824 +429,4 @@ def get_settings(): ] -def get_display_server(): - return config_read_typed(f"Reader.{OMEROReader.reader_name}.show_server", bool) - - -def set_display_server(value): - config_write_typed(f"Reader.{OMEROReader.reader_name}.show_server", value, key_type=bool) - - -class LoginHelper: - def __init__(self): - self.server = get_omero_server() - self.port = get_omero_port() - self.username = get_omero_user() - self.passwd = "" - self.session_key = None - self.session = None - self.client = None - self.container_service = None - self.tokens = {} - self.browser_window = None - atexit.register(self.shutdown) - - def get_tokens(self, path=None): - self.tokens.clear() - # Future versions of omero_user_token may support multiple tokens, so we code with that in mind. - if not TOKENS_AVAILABLE: - return - # Check the reader setting which disables tokens. - tokens_enabled = config_read_typed(f"Reader.{OMEROReader.reader_name}.allow_token", bool) - if tokens_enabled is not None and not tokens_enabled: - return - # User tokens sadly default to the home directory. This would override that location. - py_home = os.environ['HOME'] - if path is not None: - os.environ['HOME'] = path - try: - LOGGER.info("Requesting token info") - token = omero_user_token.get_token() - server, port = token[token.find('@') + 1:].split(':') - port = int(port) - LOGGER.info("Connection to {}:{}".format(server, port)) - session_key = token[:token.find('@')] - self.tokens[server] = (server, port, session_key) - except Exception: - LOGGER.error("Failed to get user token", exc_info=True) - if path is not None: - os.environ['HOME'] = py_home - - def try_token(self, address): - if address not in self.tokens: - LOGGER.error(f"Token {address} not found") - return False - else: - server, port, session_key = self.tokens[address] - return self.login(server=server, port=port, session_key=session_key) - - def login(self, server=None, port=None, user=None, passwd=None, session_key=None): - self.client = omero.client(host=server, port=port) - if session_key is not None: - try: - self.session = self.client.joinSession(session_key) - self.client.enableKeepAlive(60) - self.session.detachOnDestroy() - self.server = server - self.port = port - self.session_key = session_key - except Exception as e: - LOGGER.error(f"Failed to join session, token may have expired: {e}") - self.client = None - self.session = None - return False - elif self.username is not None: - try: - self.session = self.client.createSession( - username=user, password=passwd) - self.client.enableKeepAlive(60) - self.session.detachOnDestroy() - self.session_key = self.client.getSessionId() - self.server = server - self.port = port - self.username = user - self.passwd = passwd - except Exception as e: - LOGGER.error(f"Failed to create session: {e}") - self.client = None - self.session = None - return False - else: - self.client = None - self.session = None - raise Exception( - "Not enough details to create a server connection.") - self.container_service = self.session.getContainerService() - return True - - - def shutdown(self): - if self.client is not None: - try: - client.closeSession() - except Exception as e: - LOGGER.error("Failed to close OMERO session - ", e) - - -CREDENTIALS = LoginHelper() - -if not get_headless(): - # We can only construct wx widgets if we're not in headless mode - import wx - import cellprofiler.gui.plugins_menu - - - def show_login_dlg(token=True, server=None): - app = wx.GetApp() - frame = app.GetTopWindow() - with OmeroLoginDlg(frame, title="Log into Omero", token=token, server=server) as dlg: - dlg.ShowModal() - - def login(e=None, server=None): - CREDENTIALS.get_tokens() - if CREDENTIALS.tokens: - if server is None: - # URL didn't specify which server we want. Just try whichever token is available - server = list(CREDENTIALS.tokens.keys())[0] - connected = CREDENTIALS.try_token(server) - if get_headless(): - if connected: - LOGGER.info("Connected to ", CREDENTIALS.server) - else: - LOGGER.warning("Failed to connect, was user token invalid?") - return connected - elif connected: - from cellprofiler.gui.errordialog import show_warning - show_warning("Connected to OMERO", - f"A token was found and used to connect to the OMERO server at {CREDENTIALS.server}", - get_display_server, - set_display_server) - return - show_login_dlg(server=server) - - def login_no_token(e): - show_login_dlg() - - - def browse(e): - if CREDENTIALS.client is None: - login() - app = wx.GetApp() - frame = app.GetTopWindow() - if CREDENTIALS.browser_window is None: - CREDENTIALS.browser_window = OmeroBrowseDlg(frame, title=f"Browse OMERO: {CREDENTIALS.server}") - CREDENTIALS.browser_window.Show() - else: - CREDENTIALS.browser_window.Raise() - - - cellprofiler.gui.plugins_menu.PLUGIN_MENU_ENTRIES.extend([ - (login, wx.NewId(), "Connect to OMERO", "Establish an OMERO connection"), - (login_no_token, wx.NewId(), "Connect to OMERO (no token)", "Establish an OMERO connection," - " but without using user tokens"), - (browse, wx.NewId(), "Browse OMERO for images", "Browse an OMERO server and add images to the pipeline") - ]) - - class OmeroLoginDlg(wx.Dialog): - - def __init__(self, *args, token=True, server=None, **kwargs): - super(self.__class__, self).__init__(*args, **kwargs) - self.credentials = CREDENTIALS - self.token = token - self.SetSizer(wx.BoxSizer(wx.VERTICAL)) - if server is None: - server = self.credentials.server - sizer = wx.BoxSizer(wx.VERTICAL) - self.Sizer.Add(sizer, 1, wx.EXPAND | wx.ALL, 6) - sub_sizer = wx.BoxSizer(wx.HORIZONTAL) - sizer.Add(sub_sizer, 0, wx.EXPAND) - - max_width = 0 - max_height = 0 - for label in ("Server:", "Port:", "Username:", "Password:"): - w, h, _, _ = self.GetFullTextExtent(label) - max_width = max(w, max_width) - max_height = max(h, max_height) - - # Add extra padding - lsize = wx.Size(max_width + 5, max_height) - sub_sizer.Add( - wx.StaticText(self, label="Server:", size=lsize), - 0, wx.ALIGN_CENTER_VERTICAL) - self.omero_server_ctrl = wx.TextCtrl(self, value=server) - sub_sizer.Add(self.omero_server_ctrl, 1, wx.EXPAND) - - sizer.AddSpacer(5) - sub_sizer = wx.BoxSizer(wx.HORIZONTAL) - sizer.Add(sub_sizer, 0, wx.EXPAND) - sub_sizer.Add( - wx.StaticText(self, label="Port:", size=lsize), - 0, wx.ALIGN_CENTER_VERTICAL) - self.omero_port_ctrl = wx.lib.intctrl.IntCtrl(self, value=self.credentials.port) - sub_sizer.Add(self.omero_port_ctrl, 1, wx.EXPAND) - - sizer.AddSpacer(5) - sub_sizer = wx.BoxSizer(wx.HORIZONTAL) - sizer.Add(sub_sizer, 0, wx.EXPAND) - sub_sizer.Add( - wx.StaticText(self, label="User:", size=lsize), - 0, wx.ALIGN_CENTER_VERTICAL) - self.omero_user_ctrl = wx.TextCtrl(self, value=self.credentials.username) - sub_sizer.Add(self.omero_user_ctrl, 1, wx.EXPAND) - - sizer.AddSpacer(5) - sub_sizer = wx.BoxSizer(wx.HORIZONTAL) - sizer.Add(sub_sizer, 0, wx.EXPAND) - sub_sizer.Add( - wx.StaticText(self, label="Password:", size=lsize), - 0, wx.ALIGN_CENTER_VERTICAL) - self.omero_password_ctrl = wx.TextCtrl(self, value="", style=wx.TE_PASSWORD|wx.TE_PROCESS_ENTER) - self.omero_password_ctrl.Bind(wx.EVT_TEXT_ENTER, self.on_connect_pressed) - if self.credentials.username is not None: - self.omero_password_ctrl.SetFocus() - sub_sizer.Add(self.omero_password_ctrl, 1, wx.EXPAND) - - sizer.AddSpacer(5) - sub_sizer = wx.BoxSizer(wx.HORIZONTAL) - sizer.Add(sub_sizer, 0, wx.EXPAND) - connect_button = wx.Button(self, label="Connect") - connect_button.Bind(wx.EVT_BUTTON, self.on_connect_pressed) - sub_sizer.Add(connect_button, 0, wx.EXPAND) - sub_sizer.AddSpacer(5) - - self.message_ctrl = wx.StaticText(self, label="Not connected") - sub_sizer.Add(self.message_ctrl, 1, wx.EXPAND) - - self.token_button = wx.Button(self, label="Set Token") - self.token_button.Bind(wx.EVT_BUTTON, self.on_set_pressed) - self.token_button.Disable() - self.token_button.SetToolTip("Use these credentials to set a long-lasting token for automatic login") - sub_sizer.Add(self.token_button, 0, wx.EXPAND) - sub_sizer.AddSpacer(5) - - - button_sizer = wx.StdDialogButtonSizer() - self.Sizer.Add(button_sizer, 0, wx.EXPAND) - - cancel_button = wx.Button(self, wx.ID_CANCEL) - button_sizer.AddButton(cancel_button) - cancel_button.Bind(wx.EVT_BUTTON, self.on_cancel) - - self.ok_button = wx.Button(self, wx.ID_OK) - button_sizer.AddButton(self.ok_button) - self.ok_button.Bind(wx.EVT_BUTTON, self.on_ok) - self.ok_button.Enable(False) - button_sizer.Realize() - - self.omero_password_ctrl.Bind(wx.EVT_TEXT, self.mark_dirty) - self.omero_port_ctrl.Bind(wx.EVT_TEXT, self.mark_dirty) - self.omero_server_ctrl.Bind(wx.EVT_TEXT, self.mark_dirty) - self.omero_user_ctrl.Bind(wx.EVT_TEXT, self.mark_dirty) - self.Layout() - - def mark_dirty(self, event): - if self.ok_button.IsEnabled(): - self.ok_button.Enable(False) - self.message_ctrl.Label = "Please connect with your new credentials" - self.message_ctrl.ForegroundColour = "black" - - def on_connect_pressed(self, event): - if self.credentials.client is not None and self.credentials.server == self.omero_server_ctrl.GetValue(): - # Already connected, accept another 'Connect' command as an ok to close - self.EndModal(wx.OK) - self.connect() - - def on_set_pressed(self, event): - if self.credentials.client is None or not TOKENS_AVAILABLE: - return - token_path = omero_user_token.assert_and_get_token_path() - if os.path.exists(token_path): - dlg2 = wx.MessageDialog(self, - "Existing omero_user_token will be overwritten. Proceed?", - "Overwrite existing token?", - wx.YES_NO | wx.CANCEL | wx.ICON_WARNING) - result = dlg2.ShowModal() - if result != wx.ID_YES: - LOGGER.debug("Cancelled") - return - token = omero_user_token.setter( - self.credentials.server, - self.credentials.port, - self.credentials.username, - self.credentials.passwd, - -1) - if token: - LOGGER.info("Set OMERO user token") - self.message_ctrl.Label = "Connected. Token Set!" - self.message_ctrl.ForegroundColour = "forest green" - else: - LOGGER.error("Failed to set OMERO user token") - self.message_ctrl.Label = "Failed to set token." - self.message_ctrl.ForegroundColour = "red" - self.message_ctrl.Refresh() - - def connect(self): - try: - server = self.omero_server_ctrl.GetValue() - port = self.omero_port_ctrl.GetValue() - user = self.omero_user_ctrl.GetValue() - passwd = self.omero_password_ctrl.GetValue() - except: - self.message_ctrl.Label = ( - "The port number must be an integer between 0 and 65535 (try 4064)" - ) - self.message_ctrl.ForegroundColour = "red" - self.message_ctrl.Refresh() - return False - self.message_ctrl.ForegroundColour = "black" - self.message_ctrl.Label = "Connecting..." - self.message_ctrl.Refresh() - # Allow UI to update before connecting - wx.Yield() - success = self.credentials.login(server, port, user, passwd) - if success: - self.message_ctrl.Label = "Connected" - self.message_ctrl.ForegroundColour = "forest green" - self.token_button.Enable() - self.message_ctrl.Refresh() - set_omero_server(server) - set_omero_port(port) - set_omero_user(user) - self.ok_button.Enable(True) - return True - else: - self.message_ctrl.Label = "Failed to log onto server" - self.message_ctrl.ForegroundColour = "red" - self.message_ctrl.Refresh() - self.token_button.Disable() - return False - - def on_cancel(self, event): - self.EndModal(wx.CANCEL) - - def on_ok(self, event): - self.EndModal(wx.OK) - - - class OmeroBrowseDlg(wx.Frame): - def __init__(self, *args, **kwargs): - super(self.__class__, self).__init__(*args, - style=wx.RESIZE_BORDER | wx.CAPTION | wx.CLOSE_BOX, - size=(900, 600), - **kwargs) - self.credentials = CREDENTIALS - self.admin_service = self.credentials.session.getAdminService() - self.url_loader = self.Parent.pipeline.add_urls - - self.Bind(wx.EVT_CLOSE, self.close_browser) - - ec = self.admin_service.getEventContext() - # Exclude sys groups - self.groups = [self.admin_service.getGroup(v) for v in ec.memberOfGroups if v > 1] - self.group_names = [group.name.val for group in self.groups] - self.current_group = self.groups[0].id.getValue() - self.users_in_group = {'All Members': -1} - self.users_in_group.update({ - x.omeName.val: x.id.val for x in self.groups[0].linkedExperimenterList() - }) - self.current_user = -1 - self.levels = {'projects': 'datasets', - 'datasets': 'images', - 'screens': 'plates', - 'plates': 'wells', - 'orphaned': 'images' - } - - splitter = wx.SplitterWindow(self, -1, style=wx.SP_BORDER) - self.browse_controls = wx.Panel(splitter, -1) - b = wx.BoxSizer(wx.VERTICAL) - - self.groups_box = wx.Choice(self.browse_controls, choices=self.group_names) - self.groups_box.Bind(wx.EVT_CHOICE, self.switch_group) - - self.members_box = wx.Choice(self.browse_controls, choices=list(self.users_in_group.keys())) - self.members_box.Bind(wx.EVT_CHOICE, self.switch_member) - - b.Add(self.groups_box, 0, wx.EXPAND) - b.Add(self.members_box, 0, wx.EXPAND) - self.container = self.credentials.session.getContainerService() - - self.tree = wx.TreeCtrl(self.browse_controls, style=wx.TR_HAS_BUTTONS | wx.TR_HIDE_ROOT) - image_list = wx.ImageList(16, 13) - image_data = { - 'projects': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAANCAYAAACgu+4kAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAA5NpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuMy1jMDExIDY2LjE0NTY2MSwgMjAxMi8wMi8wNi0xNDo1NjoyNyAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wTU09Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9tbS8iIHhtbG5zOnN0UmVmPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvc1R5cGUvUmVzb3VyY2VSZWYjIiB4bWxuczp4bXA9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC8iIHhtcE1NOk9yaWdpbmFsRG9jdW1lbnRJRD0ieG1wLmRpZDpDNDk5ODU0N0U5MjA2ODExODhDNkJBNzRDM0U2QkE2NyIgeG1wTU06RG9jdW1lbnRJRD0ieG1wLmRpZDpFNjZDNDcxNzc5NEUxMUUxOTY2OEJEQjhGOUExQ0Y3RCIgeG1wTU06SW5zdGFuY2VJRD0ieG1wLmlpZDpFNjZDNDcxNjc5NEUxMUUxOTY2OEJEQjhGOUExQ0Y3RCIgeG1wOkNyZWF0b3JUb29sPSJBZG9iZSBQaG90b3Nob3AgQ1M2ICgxMy4wIDIwMTIwMzA1Lm0uNDE1IDIwMTIvMDMvMDU6MjE6MDA6MDApICAoTWFjaW50b3NoKSI+IDx4bXBNTTpEZXJpdmVkRnJvbSBzdFJlZjppbnN0YW5jZUlEPSJ4bXAuaWlkOkM1NzAxRDEzMjkyMTY4MTE4OEM2QkE3NEMzRTZCQTY3IiBzdFJlZjpkb2N1bWVudElEPSJ4bXAuZGlkOkM0OTk4NTQ3RTkyMDY4MTE4OEM2QkE3NEMzRTZCQTY3Ii8+IDwvcmRmOkRlc2NyaXB0aW9uPiA8L3JkZjpSREY+IDwveDp4bXBtZXRhPiA8P3hwYWNrZXQgZW5kPSJyIj8+vd9MhwAAALhJREFUeNpi/P//P0NEXsd/BhxgxaQKRgY8gAXGMNbVwpA8e/kaAyHAGJhWe5iNnctGVkoaQ/Lxs6cMv35+w6f/CMvfP39svDytscrqaijgtX3t5u02IAMYXrx5z0AOAOll+fPnF8Ov37/IMgCkl+XP799Af5JpAFAv0IBfDD9//STTAKALfgMJcl0A0gvxwi8KvPAXFIhkGvAXHIjAqPjx4zuZsQCMxn9//my9eOaEFQN54ChAgAEAzRBnWnEZWFQAAAAASUVORK5CYII=', - 'datasets': 'iVBORw0KGgoAAAANSUhEUgAAABAAAAANCAYAAACgu+4kAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAA5NpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuMy1jMDExIDY2LjE0NTY2MSwgMjAxMi8wMi8wNi0xNDo1NjoyNyAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wTU09Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9tbS8iIHhtbG5zOnN0UmVmPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvc1R5cGUvUmVzb3VyY2VSZWYjIiB4bWxuczp4bXA9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC8iIHhtcE1NOk9yaWdpbmFsRG9jdW1lbnRJRD0ieG1wLmRpZDpDNDk5ODU0N0U5MjA2ODExODhDNkJBNzRDM0U2QkE2NyIgeG1wTU06RG9jdW1lbnRJRD0ieG1wLmRpZDpFNjZGMDA2ODc5NEUxMUUxOTY2OEJEQjhGOUExQ0Y3RCIgeG1wTU06SW5zdGFuY2VJRD0ieG1wLmlpZDpFNjZDNDcxQTc5NEUxMUUxOTY2OEJEQjhGOUExQ0Y3RCIgeG1wOkNyZWF0b3JUb29sPSJBZG9iZSBQaG90b3Nob3AgQ1M2ICgxMy4wIDIwMTIwMzA1Lm0uNDE1IDIwMTIvMDMvMDU6MjE6MDA6MDApICAoTWFjaW50b3NoKSI+IDx4bXBNTTpEZXJpdmVkRnJvbSBzdFJlZjppbnN0YW5jZUlEPSJ4bXAuaWlkOkM1NzAxRDEzMjkyMTY4MTE4OEM2QkE3NEMzRTZCQTY3IiBzdFJlZjpkb2N1bWVudElEPSJ4bXAuZGlkOkM0OTk4NTQ3RTkyMDY4MTE4OEM2QkE3NEMzRTZCQTY3Ii8+IDwvcmRmOkRlc2NyaXB0aW9uPiA8L3JkZjpSREY+IDwveDp4bXBtZXRhPiA8P3hwYWNrZXQgZW5kPSJyIj8+l9tKdwAAAKZJREFUeNpi/P//P0NUm+d/BhxgWdV2RgY8gAXGMNMwwpA8deMcAyHAEljsdJhTmJ3h1YfnWBUA5f/j0X+E5d+//zYhrv5YZU108du+cNlKG6AB/xjefn7FQA4A6WX59/cfw5/ff8gz4C/IAKApf/78pswFv3/9Jt8Ff8FeIM+AvzAv/KbUC/8oC8T/DN9//iLTBf8ZWP7/+7f18MELVgzkgaMAAQYAgLlmT8qQW/sAAAAASUVORK5CYII=', - 'screens': 'iVBORw0KGgoAAAANSUhEUgAAABEAAAANCAYAAABPeYUaAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAA5NpVFh0WE1MOmNvbS5hZG9iZS54bXAAAAAAADw/eHBhY2tldCBiZWdpbj0i77u/IiBpZD0iVzVNME1wQ2VoaUh6cmVTek5UY3prYzlkIj8+IDx4OnhtcG1ldGEgeG1sbnM6eD0iYWRvYmU6bnM6bWV0YS8iIHg6eG1wdGs9IkFkb2JlIFhNUCBDb3JlIDUuMy1jMDExIDY2LjE0NTY2MSwgMjAxMi8wMi8wNi0xNDo1NjoyNyAgICAgICAgIj4gPHJkZjpSREYgeG1sbnM6cmRmPSJodHRwOi8vd3d3LnczLm9yZy8xOTk5LzAyLzIyLXJkZi1zeW50YXgtbnMjIj4gPHJkZjpEZXNjcmlwdGlvbiByZGY6YWJvdXQ9IiIgeG1sbnM6eG1wTU09Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC9tbS8iIHhtbG5zOnN0UmVmPSJodHRwOi8vbnMuYWRvYmUuY29tL3hhcC8xLjAvc1R5cGUvUmVzb3VyY2VSZWYjIiB4bWxuczp4bXA9Imh0dHA6Ly9ucy5hZG9iZS5jb20veGFwLzEuMC8iIHhtcE1NOk9yaWdpbmFsRG9jdW1lbnRJRD0ieG1wLmRpZDpDNDk5ODU0N0U5MjA2ODExODhDNkJBNzRDM0U2QkE2NyIgeG1wTU06RG9jdW1lbnRJRD0ieG1wLmRpZDo5NkU0QUUzNjc4NEExMUUxOTY2OEJEQjhGOUExQ0Y3RCIgeG1wTU06SW5zdGFuY2VJRD0ieG1wLmlpZDo5NkU0QUUzNTc4NEExMUUxOTY2OEJEQjhGOUExQ0Y3RCIgeG1wOkNyZWF0b3JUb29sPSJBZG9iZSBQaG90b3Nob3AgQ1M2ICgxMy4wIDIwMTIwMzA1Lm0uNDE1IDIwMTIvMDMvMDU6MjE6MDA6MDApICAoTWFjaW50b3NoKSI+IDx4bXBNTTpEZXJpdmVkRnJvbSBzdFJlZjppbnN0YW5jZUlEPSJ4bXAuaWlkOkM1NzAxRDEzMjkyMTY4MTE4OEM2QkE3NEMzRTZCQTY3IiBzdFJlZjpkb2N1bWVudElEPSJ4bXAuZGlkOkM0OTk4NTQ3RTkyMDY4MTE4OEM2QkE3NEMzRTZCQTY3Ii8+IDwvcmRmOkRlc2NyaXB0aW9uPiA8L3JkZjpSREY+IDwveDp4bXBtZXRhPiA8P3hwYWNrZXQgZW5kPSJyIj8+wkwZRwAAAQxJREFUeNpi/P//PwMjIyODqbHhfwYc4PTZ84y45ED6WZAFPFxdwQYig+27djEQAowgk6wtLcGu4GBnY0C38vvPX3gNOHr8OCPcJaHh4QwsLCwYiv78+YNV87+/fxnWrlkDZsN1PXr0iGHPnj1wRS4uLnj5jg4OcDYTjPH7928w3WxmhcJ3zmtC4Zc2mUH4SC6EG/LrF8TvtaeOofD3TqpD4XfXnULho3gHJOjt7Q2XePHiBV7+s6dPsRuydetWuISuri5evra2Nm7vVKUZoPDTratR+EZV1RjewTCkbdYFFP7Mo60o/HNtrSgBjeEdMzMzuMRToJ/x8Z88eYJpyKcPH8AYGRDiwwBAgAEAvXKdXsBF6t8AAAAASUVORK5CYII=', - 'plates': 'iVBORw0KGgoAAAANSUhEUgAAAA8AAAANCAYAAAB2HjRBAAAAGXRFWHRTb2Z0d2FyZQBBZG9iZSBJbWFnZVJlYWR5ccllPAAAASBJREFUeNpiZAACRVXNhnfv3tX//v2bgQjwAIgDv316d4FR38hM4MHDh++trSwZ+Pn4COr8+OkTw4GDhx4ANSuygARANj59+ozhxNFDKAplFVQYHj+4gyEGBAogguniuVMfkCUf7Z4IxsjgyqOJYIwOWGCM////g+mfj68woIt9/Ikphqr53z8wrZo0mwFdzFoVUwxF8z+giRbWDijOevjwAVYxrDafOHoARaGElBxWMRhgQvgF4px98x+BMbLYPSD/HpoYqrP/QQLi2Y2fDOhi37CIoYU2xMSYTlUGdDEdLGIgwAgiNHUM/r9//55BR1sbxX+PHj1kkJOTR43zq1cZBAUFGa5fuQDWy3D69Jn7IAO4+IQIYpA6kHqQPoAAAwCQE6mYLjTwJwAAAABJRU5ErkJggg==' - } - self.image_codes = {} - for name, dat in image_data.items(): - decodedImgData = base64.b64decode(dat) - bio = BytesIO(decodedImgData) - img = wx.Image(bio) - img = img.Scale(16, 13) - self.image_codes[name] = image_list.Add(img.ConvertToBitmap()) - self.tree.AssignImageList(image_list) - - self.tree.Bind(wx.EVT_TREE_ITEM_EXPANDING, self.fetch_children) - self.tree.Bind(wx.EVT_TREE_SEL_CHANGING, self.select_tree) - self.tree.Bind(wx.EVT_TREE_SEL_CHANGED, self.update_thumbnails) - self.tree.Bind(wx.EVT_TREE_BEGIN_DRAG, self.process_drag) - - data = self.fetch_containers() - - self.populate_tree(data) - b.Add(self.tree, 1, wx.EXPAND) - - self.browse_controls.SetSizer(b) - - self.image_controls = wx.Panel(splitter, -1) - - vert_sizer = wx.BoxSizer(wx.VERTICAL) - - self.tile_panel = TilePanel(self.image_controls, 1) - self.tile_panel.url_loader = self.url_loader - - self.tiler_sizer = wx.WrapSizer(wx.HORIZONTAL) - self.tile_panel.SetSizer(self.tiler_sizer) - self.tile_panel.SetScrollbars(0, 20, 0, 20) - - self.update_thumbnails() - vert_sizer.Add(self.tile_panel, wx.EXPAND) - - add_button = wx.Button(self.image_controls, wx.NewId(), "Add to file list") - add_button.Bind(wx.EVT_BUTTON, self.add_selected_to_pipeline) - vert_sizer.Add(add_button, 0, wx.ALIGN_RIGHT | wx.ALL, 5) - self.image_controls.SetSizer(vert_sizer) - splitter.SplitVertically(self.browse_controls, self.image_controls, 300) - - self.Layout() - - def close_browser(self, event): - self.credentials.browser_window = None - event.Skip() - - def add_selected_to_pipeline(self, e=None): - displayed = self.tile_panel.GetChildren() - all_urls = [] - selected_urls = [] - for item in displayed: - if isinstance(item, ImagePanel): - all_urls.append(item.url) - if item.selected: - selected_urls.append(item.url) - if selected_urls: - self.url_loader(selected_urls) - else: - self.url_loader(all_urls) - - def process_drag(self, event): - # We have our own custom handler here - data = wx.FileDataObject() - for file_url in self.fetch_file_list_from_tree(event): - data.AddFile(file_url) - drop_src = wx.DropSource(self) - drop_src.SetData(data) - drop_src.DoDragDrop(wx.Drag_CopyOnly) - - def fetch_file_list_from_tree(self, event): - files = [] - - def recurse_for_images(tree_id): - if not self.tree.IsExpanded(tree_id): - self.tree.Expand(tree_id) - data = self.tree.GetItemData(tree_id) - item_type = data['type'] - if item_type == 'images': - files.append(f"https://{self.credentials.server}/webclient/?show=image-{data['id']}") - elif 'images' in data: - for omero_id, _ in data['images'].items(): - files.append(f"https://{self.credentials.server}/webclient/?show=image-{omero_id}") - else: - child_id, cookie = self.tree.GetFirstChild(tree_id) - while child_id.IsOk(): - recurse_for_images(child_id) - child_id, cookie = self.tree.GetNextChild(tree_id, cookie) - - recurse_for_images(event.GetItem()) - return files - - def select_tree(self, event): - target_id = event.GetItem() - self.tree.Expand(target_id) - - def next_level(self, level): - return self.levels.get(level, None) - - def switch_group(self, e=None): - new_group = self.groups_box.GetCurrentSelection() - self.current_group = self.groups[new_group].id.getValue() - self.current_user = -1 - self.refresh_group_members() - data = self.fetch_containers() - self.populate_tree(data) - - def switch_member(self, e=None): - new_member = self.members_box.GetStringSelection() - self.current_user = self.users_in_group.get(new_member, -1) - data = self.fetch_containers() - self.populate_tree(data) - - def refresh_group_members(self): - self.users_in_group = {'All Members': -1} - group = self.groups[self.groups_box.GetCurrentSelection()] - self.users_in_group.update({ - x.omeName.val: x.id.val for x in group.linkedExperimenterList() - }) - self.members_box.Clear() - self.members_box.AppendItems(list(self.users_in_group.keys())) - - def fetch_children(self, event): - target_id = event.GetItem() - if self.tree.GetChildrenCount(target_id, recursively=False) > 0: - # Already loaded - return - data = self.tree.GetItemData(target_id) - subject_type = data['type'] - target_type = self.levels.get(subject_type, None) - if target_type is None: - # We're at the bottom level already - return - subject = data['id'] - if subject == -1: - sub_str = "orphaned=true&" - else: - sub_str = f"id={subject}&" - if target_type == 'wells': - url = f"https://{self.credentials.server}/api/v0/m/plates/{subject}/wells/?bsession={self.credentials.session_key}" - else: - url = f"https://{self.credentials.server}/webclient/api/{target_type}/?{sub_str}experimenter_id=-1&page=0&group={self.current_group}&bsession={self.credentials.session_key}" - try: - result = requests.get(url, timeout=5) - except requests.exceptions.ConnectTimeout: - LOGGER.error("Server request timed out") - return - except Exception: - LOGGER.error("Server request failed", exc_info=True) - return - result = result.json() - if 'images' in result: - image_map = {entry['id']: entry['name'] for entry in result['images']} - data['images'] = image_map - self.tree.SetItemData(target_id, data) - if 'meta' in result: - # This is the plates API - self.populate_tree_screen(result, target_id) - result = result['data'] - else: - self.populate_tree(result, target_id) - - def fetch_containers(self): - url = f"https://{self.credentials.server}/webclient/api/containers/?id={self.current_user}&page=0&group={self.current_group}&bsession={self.credentials.session_key}" - try: - data = requests.get(url, timeout=5) - except requests.exceptions.ConnectTimeout: - LOGGER.error("Server request timed out") - return {} - data.raise_for_status() - return data.json() - - def fetch_thumbnails(self, id_list): - if not id_list: - return {} - id_list = [str(x) for x in id_list] - chunk_size = 10 - buffer = {x: "" for x in id_list} - for i in range(0, len(id_list), chunk_size): - ids_to_get = '&id='.join(id_list[i:i + chunk_size]) - url = f"https://{self.credentials.server}/webclient/get_thumbnails/128/?&bsession={self.credentials.session_key}&id={ids_to_get}" - LOGGER.debug(f"Fetching {url}") - try: - data = requests.get(url, timeout=5) - except requests.exceptions.ConnectTimeout: - continue - if data.status_code != 200: - LOGGER.warning(f"Server error: {data.status_code} - {data.reason}") - else: - buffer.update(data.json()) - return buffer - - @functools.lru_cache(maxsize=20) - def fetch_large_thumbnail(self, id): - # Get a large thumbnail for single image display mode. We cache the last 20. - url = f"https://{self.credentials.server}/webgateway/render_thumbnail/{id}/450/450/?bsession={self.credentials.session_key}" - LOGGER.debug(f"Fetching {url}") - try: - data = requests.get(url, timeout=5) - except requests.exceptions.ConnectTimeout: - LOGGER.error("Thumbnail request timed out") - return False - if data.status_code != 200: - LOGGER.warning("Server error:", data.status_code, data.reason) - return False - elif not data.content: - return False - io_bytes = io.BytesIO(data.content) - return wx.Image(io_bytes) - - def update_thumbnails(self, event=None): - self.tiler_sizer.Clear(delete_windows=True) - if not event: - return - target_id = event.GetItem() - item_data = self.tree.GetItemData(target_id) - if item_data.get('type', None) == 'images': - image_id = item_data['id'] - img_name = item_data['name'] - thumb_img = self.fetch_large_thumbnail(image_id) - if not thumb_img or not thumb_img.IsOk(): - thumb_img = self.get_error_thumbnail(450) - else: - thumb_img = thumb_img.ConvertToBitmap() - tile = ImagePanel(thumb_img, self.tile_panel, image_id, img_name, self.credentials.server, size=450) - tile.selected = True - self.tiler_sizer.Add(tile, 0, wx.ALL, 5) - else: - image_targets = item_data.get('images', {}) - if not image_targets: - return - - id_list = list(image_targets.keys()) - data = self.fetch_thumbnails(id_list) - for image_id, image_data in data.items(): - img_name = image_targets[int(image_id)] - start_data = image_data.find('/9') - if start_data == -1: - img = self.get_error_thumbnail(128) - else: - decoded = base64.b64decode(image_data[start_data:]) - bio = BytesIO(decoded) - img = wx.Image(bio) - if not img.IsOk(): - img = self.get_error_thumbnail(128) - else: - img = img.ConvertToBitmap() - tile = ImagePanel(img, self.tile_panel, image_id, img_name, self.credentials.server) - self.tiler_sizer.Add(tile, 0, wx.ALL, 5) - self.tiler_sizer.Layout() - self.image_controls.Layout() - self.image_controls.Refresh() - - @functools.lru_cache(maxsize=10) - def get_error_thumbnail(self, size): - # Draw an image with an error icon. Cache the result since we may need the error icon repeatedly. - artist = wx.ArtProvider() - size //= 2 - return artist.GetBitmap(wx.ART_WARNING, size=(size, size)) - - def populate_tree(self, data, parent=None): - if parent is None: - self.tree.DeleteAllItems() - parent = self.tree.AddRoot("Server") - for item_type, items in data.items(): - image = self.image_codes.get(item_type, None) - if not isinstance(items, list): - items = [items] - for entry in items: - entry['type'] = item_type - new_id = self.tree.AppendItem(parent, f"{entry['name']}", data=entry) - if image is not None: - self.tree.SetItemImage(new_id, image, wx.TreeItemIcon_Normal) - if entry.get('childCount', 0) > 0 or item_type == 'plates': - self.tree.SetItemHasChildren(new_id) - - def populate_tree_screen(self, data, parent=None): - # Tree data from screens API - wells = data['data'] - rows = string.ascii_uppercase - for well_dict in wells: - name = f"Well {rows[well_dict['Row']]}{well_dict['Column'] + 1:02}" - well_dict['type'] = 'wells' - well_id = self.tree.AppendItem(parent, name) - well_dict['images'] = {} - for field_dict in well_dict['WellSamples']: - image_data = field_dict['Image'] - image_id = image_data['@id'] - image_name = image_data['Name'] - refined_image_data = { - 'type': 'images', - 'id': image_id, - 'name': image_name - } - well_dict['images'][image_id] = image_name - self.tree.AppendItem(well_id, image_name, data=refined_image_data) - self.tree.SetItemData(well_id, well_dict) - - - class ImagePanel(wx.Panel): - ''' - ImagePanels are wxPanels that display a wxBitmap and store multiple - image channels which can be recombined to mix different bitmaps. - ''' - - def __init__(self, thumbnail, parent, omero_id, name, server, size=128): - """ - thumbnail -- wx Bitmap - parent -- parent window to the wx.Panel - - """ - self.parent = parent - self.bitmap = thumbnail - self.selected = False - self.omero_id = omero_id - self.url = f"https://{server}/webclient/?show=image-{omero_id}" - self.name = name - if len(name) > 17: - self.shortname = name[:14] + '...' - else: - self.shortname = name - self.size_x = size - self.size_y = size + 30 - wx.Panel.__init__(self, parent, wx.NewId(), size=(self.size_x, self.size_y)) - self.Bind(wx.EVT_PAINT, self.OnPaint) - self.Bind(wx.EVT_LEFT_DOWN, self.select) - self.Bind(wx.EVT_RIGHT_DOWN, self.right_click) - self.SetClientSize((self.size_x, self.size_y)) - # We need to pass these events up to the parent panel. - self.Bind(wx.EVT_MOTION, self.pass_event) - self.Bind(wx.EVT_LEFT_UP, self.pass_event) - - def select(self, e): - self.selected = not self.selected - self.Refresh() - e.StopPropagation() - e.Skip() - - @staticmethod - def pass_event(e): - e.ResumePropagation(1) - e.Skip() - - def right_click(self, event): - popupmenu = wx.Menu() - add_file_item = popupmenu.Append(-1, "Add to file list") - self.Bind(wx.EVT_MENU, self.add_to_pipeline, add_file_item) - add_file_item = popupmenu.Append(-1, "Show in OMERO.web") - self.Bind(wx.EVT_MENU, self.open_in_browser, add_file_item) - # Show menu - self.PopupMenu(popupmenu, event.GetPosition()) - - def add_to_pipeline(self, e): - self.parent.url_loader([self.url]) - - def open_in_browser(self, e): - wx.LaunchDefaultBrowser(self.url) - - def OnPaint(self, evt): - dc = wx.PaintDC(self) - dc.Clear() - dc.DrawBitmap(self.bitmap, (self.size_x - self.bitmap.Width) // 2, - ((self.size_x - self.bitmap.Height) // 2) + 20) - rect = wx.Rect(0, 0, self.size_x, self.size_x + 20) - dc.DrawLabel(self.shortname, rect, alignment=wx.ALIGN_CENTER_HORIZONTAL | wx.ALIGN_TOP) - dc.SetPen(wx.Pen("GREY", style=wx.PENSTYLE_SOLID)) - dc.SetBrush(wx.Brush("BLACK", wx.TRANSPARENT)) - dc.DrawRectangle(rect) - # Outline the whole image - if self.selected: - dc.SetPen(wx.Pen("BLUE", 3)) - dc.SetBrush(wx.Brush("BLACK", style=wx.TRANSPARENT)) - dc.DrawRectangle(0, 0, self.size_x, self.size_y) - return dc - - - class TilePanel(wx.ScrolledWindow): - - def __init__(self, *args, **kwargs): - super(self.__class__, self).__init__(*args, **kwargs) - self.select_source = None - self.select_box = None - self.Bind(wx.EVT_MOTION, self.on_motion) - self.Bind(wx.EVT_PAINT, self.on_paint) - self.Bind(wx.EVT_LEFT_UP, self.on_release) - - def deselect_all(self): - for child in self.GetChildren(): - if isinstance(child, ImagePanel): - child.selected = False - - def on_motion(self, evt): - if not evt.LeftIsDown(): - self.select_source = None - self.select_box = None - return - self.SetFocusIgnoringChildren() - if self.select_source is None: - self.select_source = evt.Position - if not evt.ShiftDown(): - self.deselect_all() - return - else: - self.select_box = wx.Rect(self.select_source, evt.Position) - for child in self.GetChildren(): - if isinstance(child, ImagePanel): - if not evt.ShiftDown(): - child.selected = False - if child.GetRect().Intersects(self.select_box): - child.selected = True - self.Refresh() - - def on_release(self, e): - self.select_source = None - self.select_box = None - self.Refresh() - - def on_paint(self, e): - dc = wx.PaintDC(self) - dc.SetPen(wx.Pen("BLUE", 3, style=wx.PENSTYLE_SHORT_DASH)) - dc.SetBrush(wx.Brush("BLUE", style=wx.TRANSPARENT)) - if self.select_box is not None: - dc.DrawRectangle(self.select_box) - -# Todo: Split GUI into a helper module # Todo: Thumbnail queue From 279877bf4af381fb29eddcc14ec65ab37b0b5139 Mon Sep 17 00:00:00 2001 From: David Stirling Date: Wed, 2 Aug 2023 13:33:13 +0100 Subject: [PATCH 08/32] Tweaks and docs --- active_plugins/omero_helper/connect.py | 20 ++++++ active_plugins/omero_helper/gui.py | 95 ++++++++++++++++++-------- active_plugins/omeroreader.py | 12 ++-- 3 files changed, 92 insertions(+), 35 deletions(-) diff --git a/active_plugins/omero_helper/connect.py b/active_plugins/omero_helper/connect.py index 63c46aa..640784f 100644 --- a/active_plugins/omero_helper/connect.py +++ b/active_plugins/omero_helper/connect.py @@ -9,6 +9,7 @@ LOGGER = logging.getLogger(__name__) +# Is omero_user_token installed and available? TOKEN_MODULE = importlib.util.find_spec("omero_user_token") TOKENS_AVAILABLE = TOKEN_MODULE is not None if TOKENS_AVAILABLE: @@ -18,6 +19,7 @@ def login(e=None, server=None): + # Attempt to connect to the server, first using a token, then via GUI CREDENTIALS.get_tokens() if CREDENTIALS.tokens: if server is None: @@ -36,6 +38,19 @@ def login(e=None, server=None): class LoginHelper: + """ + This class stores our working set of OMERO credentials and connection objects. + + It behaves as a singleton, so multiple OMERO-using plugins will share credentials. + """ + _instance = None + + def __new__(cls): + # We only allow one instance of this class within CellProfiler + if cls._instance is None: + cls._instance = super().__new__(cls) + return cls._instance + def __init__(self): self.server = get_omero_server() self.port = get_omero_port() @@ -46,10 +61,12 @@ def __init__(self): self.client = None self.container_service = None self.tokens = {} + # Any OMERO browser GUI which is connected to this object self.browser_window = None atexit.register(self.shutdown) def get_tokens(self, path=None): + # Load all tokens from omero_user_token self.tokens.clear() # Future versions of omero_user_token may support multiple tokens, so we code with that in mind. if not TOKENS_AVAILABLE: @@ -76,6 +93,7 @@ def get_tokens(self, path=None): os.environ['HOME'] = py_home def try_token(self, address): + # Attempt to use an omero token to connect to a specific server if address not in self.tokens: LOGGER.error(f"Token {address} not found") return False @@ -84,6 +102,7 @@ def try_token(self, address): return self.login(server=server, port=port, session_key=session_key) def login(self, server=None, port=None, user=None, passwd=None, session_key=None): + # Attempt to connect to the server self.client = omero.client(host=server, port=port) if session_key is not None: try: @@ -123,6 +142,7 @@ def login(self, server=None, port=None, user=None, passwd=None, session_key=None return True def shutdown(self): + # Disconnect from the server if self.client is not None: try: self.client.closeSession() diff --git a/active_plugins/omero_helper/gui.py b/active_plugins/omero_helper/gui.py index 9d1866c..d2a0c9c 100644 --- a/active_plugins/omero_helper/gui.py +++ b/active_plugins/omero_helper/gui.py @@ -17,6 +17,7 @@ def get_display_server(): + # Should we display the 'connection successful' message after using a token? return config_read_typed(f"Reader.OMERO.show_server", bool) @@ -25,6 +26,7 @@ def set_display_server(value): def login_gui(connected, server=None): + # Login via GUI or display a prompt notifying that we're already connected if connected: from cellprofiler.gui.errordialog import show_warning show_warning("Connected to OMERO", @@ -35,22 +37,21 @@ def login_gui(connected, server=None): show_login_dlg(server=server) -def show_login_dlg(token=True, server=None): +def show_login_dlg(e=None, server=None): + # Show the login GUI app = wx.GetApp() frame = app.GetTopWindow() - with OmeroLoginDlg(frame, title="Log into Omero", token=token, server=server) as dlg: + with OmeroLoginDlg(frame, title="Log into Omero", server=server) as dlg: dlg.ShowModal() -def login_no_token(e): - show_login_dlg() - - def browse(e): + # Show the browser dialog if CREDENTIALS.client is None: login() app = wx.GetApp() frame = app.GetTopWindow() + # Only allow a single instance, raise the window if it already exists. if CREDENTIALS.browser_window is None: CREDENTIALS.browser_window = OmeroBrowseDlg(frame, title=f"Browse OMERO: {CREDENTIALS.server}") CREDENTIALS.browser_window.Show() @@ -59,16 +60,19 @@ def browse(e): def inject_plugin_menu_entries(): + # Add plugin menu entries to the main CellProfiler GUI cellprofiler.gui.plugins_menu.PLUGIN_MENU_ENTRIES.extend([ (login, wx.NewId(), "Connect to OMERO", "Establish an OMERO connection"), - (login_no_token, wx.NewId(), "Connect to OMERO (no token)", "Establish an OMERO connection," + (show_login_dlg, wx.NewId(), "Connect to OMERO (no token)", "Establish an OMERO connection," " but without using user tokens"), (browse, wx.NewId(), "Browse OMERO for images", "Browse an OMERO server and add images to the pipeline") ]) class OmeroLoginDlg(wx.Dialog): - + """ + A dialog pane to provide and use OMERO login credentials. + """ def __init__(self, *args, token=True, server=None, **kwargs): super(self.__class__, self).__init__(*args, **kwargs) self.credentials = CREDENTIALS @@ -249,6 +253,9 @@ def on_ok(self, event): class OmeroBrowseDlg(wx.Frame): + """ + An OMERO server browser intended for browsing images and adding them to the main file list + """ def __init__(self, *args, **kwargs): super(self.__class__, self).__init__(*args, style=wx.RESIZE_BORDER | wx.CAPTION | wx.CLOSE_BOX, @@ -343,10 +350,12 @@ def __init__(self, *args, **kwargs): self.Layout() def close_browser(self, event): + # Disconnect the browser window from the login helper self.credentials.browser_window = None event.Skip() def add_selected_to_pipeline(self, e=None): + # Add selected images to the pipeline as URLs displayed = self.tile_panel.GetChildren() all_urls = [] selected_urls = [] @@ -361,7 +370,8 @@ def add_selected_to_pipeline(self, e=None): self.url_loader(all_urls) def process_drag(self, event): - # We have our own custom handler here + # CellProfiler's file list uses a custom handler. + # This will mimic dropping actual files onto it. data = wx.FileDataObject() for file_url in self.fetch_file_list_from_tree(event): data.AddFile(file_url) @@ -370,9 +380,11 @@ def process_drag(self, event): drop_src.DoDragDrop(wx.Drag_CopyOnly) def fetch_file_list_from_tree(self, event): + # Generate a list of OMERO URLs when the user tries to drag an entry. files = [] def recurse_for_images(tree_id): + # Search the tree for images and add to a list of URLs if not self.tree.IsExpanded(tree_id): self.tree.Expand(tree_id) data = self.tree.GetItemData(tree_id) @@ -392,13 +404,12 @@ def recurse_for_images(tree_id): return files def select_tree(self, event): + # The tree is fetched as it's being expanded. Clicking on an object needs to trigger child expansion. target_id = event.GetItem() self.tree.Expand(target_id) - def next_level(self, level): - return self.levels.get(level, None) - def switch_group(self, e=None): + # Change OMERO group new_group = self.groups_box.GetCurrentSelection() self.current_group = self.groups[new_group].id.getValue() self.current_user = -1 @@ -407,12 +418,14 @@ def switch_group(self, e=None): self.populate_tree(data) def switch_member(self, e=None): + # Change OMERO user filter new_member = self.members_box.GetStringSelection() self.current_user = self.users_in_group.get(new_member, -1) data = self.fetch_containers() self.populate_tree(data) def refresh_group_members(self): + # Update the available user list when the group changes self.users_in_group = {'All Members': -1} group = self.groups[self.groups_box.GetCurrentSelection()] self.users_in_group.update({ @@ -422,6 +435,7 @@ def refresh_group_members(self): self.members_box.AppendItems(list(self.users_in_group.keys())) def fetch_children(self, event): + # Load the next level in the tree for a target object. target_id = event.GetItem() if self.tree.GetChildrenCount(target_id, recursively=False) > 0: # Already loaded @@ -462,6 +476,7 @@ def fetch_children(self, event): self.populate_tree(result, target_id) def fetch_containers(self): + # Grab the base project/dataset structure for the tree view. url = f"https://{self.credentials.server}/webclient/api/containers/?id={self.current_user}&page=0&group={self.current_group}&bsession={self.credentials.session_key}" try: data = requests.get(url, timeout=5) @@ -472,6 +487,7 @@ def fetch_containers(self): return data.json() def fetch_thumbnails(self, id_list): + # Get thumbnails for images if not id_list: return {} id_list = [str(x) for x in id_list] @@ -492,9 +508,9 @@ def fetch_thumbnails(self, id_list): return buffer @functools.lru_cache(maxsize=20) - def fetch_large_thumbnail(self, id): + def fetch_large_thumbnail(self, image_id): # Get a large thumbnail for single image display mode. We cache the last 20. - url = f"https://{self.credentials.server}/webgateway/render_thumbnail/{id}/450/450/?bsession={self.credentials.session_key}" + url = f"https://{self.credentials.server}/webgateway/render_thumbnail/{image_id}/450/450/?bsession={self.credentials.session_key}" LOGGER.debug(f"Fetching {url}") try: data = requests.get(url, timeout=5) @@ -510,6 +526,7 @@ def fetch_large_thumbnail(self, id): return wx.Image(io_bytes) def update_thumbnails(self, event=None): + # Show image previews when objects in the tree are clicked on. self.tiler_sizer.Clear(delete_windows=True) if not event: return @@ -560,6 +577,7 @@ def get_error_thumbnail(self, size): return artist.GetBitmap(wx.ART_WARNING, size=(size, size)) def populate_tree(self, data, parent=None): + # Build the tree view if parent is None: self.tree.DeleteAllItems() parent = self.tree.AddRoot("Server") @@ -576,7 +594,7 @@ def populate_tree(self, data, parent=None): self.tree.SetItemHasChildren(new_id) def populate_tree_screen(self, data, parent=None): - # Tree data from screens API + # Fill the tree data from the screens API wells = data['data'] rows = string.ascii_uppercase for well_dict in wells: @@ -599,16 +617,18 @@ def populate_tree_screen(self, data, parent=None): class ImagePanel(wx.Panel): - ''' - ImagePanels are wxPanels that display a wxBitmap and store multiple - image channels which can be recombined to mix different bitmaps. - ''' + """ + The ImagePanel displays an image's name and a preview thumbnail as a wx.Bitmap. + """ def __init__(self, thumbnail, parent, omero_id, name, server, size=128): """ - thumbnail -- wx Bitmap - parent -- parent window to the wx.Panel - + thumbnail - wx.Bitmap + parent - parent window to the wx.Panel + omero_id - OMERO id of the image + name - name to display + server - the server the image lives on + size - int dimension of the thumbnail to display """ self.parent = parent self.bitmap = thumbnail @@ -632,19 +652,24 @@ def __init__(self, thumbnail, parent, omero_id, name, server, size=128): self.Bind(wx.EVT_LEFT_UP, self.pass_event) def select(self, e): + # Mark a panel as selected self.selected = not self.selected self.Refresh() e.StopPropagation() e.Skip() def pass_event(self, e): + # We need to pass mouse events up to the containing TilePanel. + # To do this we need to correct the event position to be relative to the parent. x, y = e.GetPosition() w, h = self.GetPosition() e.SetPosition((x + w, y + h)) + # Now we send the event upwards to be caught by the parent. e.ResumePropagation(1) e.Skip() def right_click(self, event): + # Show right click menu popupmenu = wx.Menu() add_file_item = popupmenu.Append(-1, "Add to file list") self.Bind(wx.EVT_MENU, self.add_to_pipeline, add_file_item) @@ -654,12 +679,15 @@ def right_click(self, event): self.PopupMenu(popupmenu, event.GetPosition()) def add_to_pipeline(self, e): + # Add image to the pipeline self.parent.url_loader([self.url]) def open_in_browser(self, e): + # Open in OMERO.web wx.LaunchDefaultBrowser(self.url) def OnPaint(self, evt): + # Custom paint handler to display image/label/selection marker. dc = wx.PaintDC(self) dc.Clear() dc.DrawBitmap(self.bitmap, (self.size_x - self.bitmap.Width) // 2, @@ -671,20 +699,22 @@ def OnPaint(self, evt): dc.DrawRectangle(rect) # Outline the whole image if self.selected: - dc.SetPen(wx.Pen("BLUE", 3)) - dc.SetBrush(wx.Brush("BLACK", style=wx.TRANSPARENT)) + dc.SetPen(wx.Pen("SLATE BLUE", 3)) dc.DrawRectangle(0, 0, self.size_x, self.size_y) return dc class TilePanel(wx.ScrolledWindow): + """ + A scrollable window which will contain image panels and allow selection of them by drawing a rectangle. + """ def __init__(self, *args, **kwargs): super(self.__class__, self).__init__(*args, **kwargs) self.select_source = None self.select_box = None - self.Bind(wx.EVT_MOTION, self.on_motion) - self.Bind(wx.EVT_PAINT, self.on_paint) + self.Bind(wx.EVT_MOTION, self.OnMotion) + self.Bind(wx.EVT_PAINT, self.OnPaint) self.Bind(wx.EVT_LEFT_UP, self.on_release) def deselect_all(self): @@ -692,19 +722,22 @@ def deselect_all(self): if isinstance(child, ImagePanel): child.selected = False - def on_motion(self, evt): + def OnMotion(self, evt): + # Handle drag selection if not evt.LeftIsDown(): + # Not dragging self.select_source = None self.select_box = None return self.SetFocusIgnoringChildren() if self.select_source is None: self.select_source = evt.Position - if not evt.ShiftDown(): - self.deselect_all() return else: self.select_box = wx.Rect(self.select_source, evt.Position) + if self.select_box.Width < 5 and self.select_box.Height < 5: + # Don't start selecting until a reasonable box size is drawn + return for child in self.GetChildren(): if isinstance(child, ImagePanel): if not evt.ShiftDown(): @@ -714,11 +747,13 @@ def on_motion(self, evt): self.Refresh() def on_release(self, e): + # Cease dragging self.select_source = None self.select_box = None self.Refresh() - def on_paint(self, e): + def OnPaint(self, e): + # Draw selection box. dc = wx.PaintDC(self) dc.SetPen(wx.Pen("BLUE", 3, style=wx.PENSTYLE_SHORT_DASH)) dc.SetBrush(wx.Brush("BLUE", style=wx.TRANSPARENT)) diff --git a/active_plugins/omeroreader.py b/active_plugins/omeroreader.py index 1f5e403..53972a8 100644 --- a/active_plugins/omeroreader.py +++ b/active_plugins/omeroreader.py @@ -72,6 +72,7 @@ from omero_helper.gui import inject_plugin_menu_entries inject_plugin_menu_entries() +# Isolates image numbers from OMERO URLs REGEX_INDEX_FROM_FILE_NAME = re.compile(r'\?show=image-(\d+)') # Inject omero as a URI scheme which CellProfiler should accept as an image entry. @@ -79,6 +80,7 @@ LOGGER = logging.getLogger(__name__) +# Maps OMERO pixel types to numpy PIXEL_TYPES = { "int8": ['b', numpy.int8, (-128, 127)], "uint8": ['B', numpy.uint8, (0, 255)], @@ -264,9 +266,9 @@ def read_volume(self, ) def read_planes(self, z=0, c=None, t=0, tile=None): - ''' + """ Creates RawPixelsStore and reads planes from the OMERO server. - ''' + """ channels = [] if c is None: channel_count = self.pixels.getSizeC().val @@ -313,9 +315,9 @@ def read_planes(self, z=0, c=None, t=0, tile=None): raw_pixels_store.close() def read_planes_volumetric(self, z=None, c=None, t=None, tile=None): - ''' + """ Creates RawPixelsStore and reads planes from the OMERO server. - ''' + """ if t is not None and z is not None: raise ValueError(f"Specified parameters {z=}, {t=} would not produce a 3D image") if z is None: @@ -383,7 +385,7 @@ def supports_format(cls, image_file, allow_open=False, volume=False): # Yes please return 1 elif "?show=image" in image_file.url.lower(): - # Looks enough like an OMERO URL that I'll have a go. + # Looks enough like an OMERO URL that we'll have a go. return 2 return -1 From 779837da0d898e06026b658d6b94fd0facb3c9bb Mon Sep 17 00:00:00 2001 From: David Stirling Date: Wed, 2 Aug 2023 13:40:45 +0100 Subject: [PATCH 09/32] Docs --- active_plugins/omeroreader.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/active_plugins/omeroreader.py b/active_plugins/omeroreader.py index 53972a8..9cce36f 100644 --- a/active_plugins/omeroreader.py +++ b/active_plugins/omeroreader.py @@ -2,6 +2,10 @@ An image reader which connects to OMERO to load data # Installation - +Add this file plus the `omero_helper` directory into your CellProfiler plugins folder. Install dependencies into +your CellProfiler Python environment. + +## Installing dependencies - This depends on platform. At the most basic level you'll need the `omero-py` package. For headless run and more convenient server login you'll also want the `omero_user_token` package. @@ -15,6 +19,11 @@ From there pip install omero-py should do the rest. +You'll probably also want the `omero_user_token` package to help manage logins (`pip install omero_user_token`). +This allows you to set reusable login tokens for quick reconnection to a server. These tokens are required for using +headless mode. + + # Usage - Like the old functionality from <=CP4, connection to OMERO is triggered through a login dialog within the GUI which should appear automatically when needed. Enter your credentials and hit 'connect'. Once connected you should be able to @@ -35,6 +44,20 @@ *File->Import->File List* to load a text file containing one image per line. A LoadData CSV can also be used. As of CP5 it is also now possible to copy and paste text (e.g. URLs) directly into the file list in the Images module. +In the Plugins menu you'll also find an option to browse an OMERO server for images and add them to your file list. +This provides an alternative method for constructing your file list. Images will be added to the list in the +OMERO URL format. + +# Tokens - +omero_user_token creates a long-lasting session token based on your login credentials, which can then be reconnected to +at a later time. The CellProfiler plugin will detect and use these tokens to connect to a server automatically. Use the +`Connect to OMERO (No token)` option in the Plugins menu if you need to switch servers. + +Within the connect dialog you'll find a new 'Set Token' button which allows you to create these tokens after making a +successful connection. These tokens are important when working in headless mode, but also mean that you no longer +need to enter your credentials each time you login via the GUI. Current omero_user_token builds support a single token +at a time, which will be stored in your user home directory. + # Working with data - Unlike previous iterations of this integration, the CP5 plugin has full support for channel and plane indexing. The previous reader misbehaved in that it would only load the first channel from OMERO if greyscale is requested. From 404a8694850da9d2482f04f78f2ea98ac9c48e17 Mon Sep 17 00:00:00 2001 From: David Stirling Date: Thu, 3 Aug 2023 16:52:00 +0100 Subject: [PATCH 10/32] Move thumbnail loading onto a thread --- active_plugins/omero_helper/gui.py | 205 +++++++++++++++++------------ active_plugins/omeroreader.py | 3 - 2 files changed, 118 insertions(+), 90 deletions(-) diff --git a/active_plugins/omero_helper/gui.py b/active_plugins/omero_helper/gui.py index d2a0c9c..4d5a93e 100644 --- a/active_plugins/omero_helper/gui.py +++ b/active_plugins/omero_helper/gui.py @@ -3,7 +3,10 @@ import io import logging import os +import queue import string +import threading +import time import requests import wx @@ -352,6 +355,8 @@ def __init__(self, *args, **kwargs): def close_browser(self, event): # Disconnect the browser window from the login helper self.credentials.browser_window = None + # Tell the thumbnail generator to shut down + self.tile_panel.active = False event.Skip() def add_selected_to_pipeline(self, e=None): @@ -455,14 +460,12 @@ def fetch_children(self, event): url = f"https://{self.credentials.server}/api/v0/m/plates/{subject}/wells/?bsession={self.credentials.session_key}" else: url = f"https://{self.credentials.server}/webclient/api/{target_type}/?{sub_str}experimenter_id=-1&page=0&group={self.current_group}&bsession={self.credentials.session_key}" + LOGGER.debug(f"Fetching {url}") try: - result = requests.get(url, timeout=5) - except requests.exceptions.ConnectTimeout: + result = requests.get(url, timeout=15) + except requests.exceptions.Timeout: LOGGER.error("Server request timed out") return - except Exception: - LOGGER.error("Server request failed", exc_info=True) - return result = result.json() if 'images' in result: image_map = {entry['id']: entry['name'] for entry in result['images']} @@ -480,102 +483,41 @@ def fetch_containers(self): url = f"https://{self.credentials.server}/webclient/api/containers/?id={self.current_user}&page=0&group={self.current_group}&bsession={self.credentials.session_key}" try: data = requests.get(url, timeout=5) - except requests.exceptions.ConnectTimeout: + except requests.exceptions.Timeout: LOGGER.error("Server request timed out") return {} data.raise_for_status() return data.json() - def fetch_thumbnails(self, id_list): - # Get thumbnails for images - if not id_list: - return {} - id_list = [str(x) for x in id_list] - chunk_size = 10 - buffer = {x: "" for x in id_list} - for i in range(0, len(id_list), chunk_size): - ids_to_get = '&id='.join(id_list[i:i + chunk_size]) - url = f"https://{self.credentials.server}/webclient/get_thumbnails/128/?&bsession={self.credentials.session_key}&id={ids_to_get}" - LOGGER.debug(f"Fetching {url}") - try: - data = requests.get(url, timeout=5) - except requests.exceptions.ConnectTimeout: - continue - if data.status_code != 200: - LOGGER.warning(f"Server error: {data.status_code} - {data.reason}") - else: - buffer.update(data.json()) - return buffer - - @functools.lru_cache(maxsize=20) - def fetch_large_thumbnail(self, image_id): - # Get a large thumbnail for single image display mode. We cache the last 20. - url = f"https://{self.credentials.server}/webgateway/render_thumbnail/{image_id}/450/450/?bsession={self.credentials.session_key}" - LOGGER.debug(f"Fetching {url}") - try: - data = requests.get(url, timeout=5) - except requests.exceptions.ConnectTimeout: - LOGGER.error("Thumbnail request timed out") - return False - if data.status_code != 200: - LOGGER.warning("Server error:", data.status_code, data.reason) - return False - elif not data.content: - return False - io_bytes = io.BytesIO(data.content) - return wx.Image(io_bytes) - def update_thumbnails(self, event=None): # Show image previews when objects in the tree are clicked on. self.tiler_sizer.Clear(delete_windows=True) + # Empty out any pending tile thumbnails + with self.tile_panel.thumbnail_queue.mutex: + self.tile_panel.thumbnail_queue.queue.clear() if not event: return target_id = event.GetItem() item_data = self.tree.GetItemData(target_id) if item_data.get('type', None) == 'images': + # We're displaying a single image image_id = item_data['id'] img_name = item_data['name'] - thumb_img = self.fetch_large_thumbnail(image_id) - if not thumb_img or not thumb_img.IsOk(): - thumb_img = self.get_error_thumbnail(450) - else: - thumb_img = thumb_img.ConvertToBitmap() - tile = ImagePanel(thumb_img, self.tile_panel, image_id, img_name, self.credentials.server, size=450) + tile = ImagePanel(self.tile_panel, image_id, img_name, self.credentials.server, size=450) tile.selected = True self.tiler_sizer.Add(tile, 0, wx.ALL, 5) else: + # We're displaying a series of images image_targets = item_data.get('images', {}) if not image_targets: return - - id_list = list(image_targets.keys()) - data = self.fetch_thumbnails(id_list) - for image_id, image_data in data.items(): - img_name = image_targets[int(image_id)] - start_data = image_data.find('/9') - if start_data == -1: - img = self.get_error_thumbnail(128) - else: - decoded = base64.b64decode(image_data[start_data:]) - bio = io.BytesIO(decoded) - img = wx.Image(bio) - if not img.IsOk(): - img = self.get_error_thumbnail(128) - else: - img = img.ConvertToBitmap() - tile = ImagePanel(img, self.tile_panel, image_id, img_name, self.credentials.server) + for image_id, image_name in image_targets.items(): + tile = ImagePanel(self.tile_panel, image_id, image_name, self.credentials.server) self.tiler_sizer.Add(tile, 0, wx.ALL, 5) self.tiler_sizer.Layout() self.image_controls.Layout() self.image_controls.Refresh() - @functools.lru_cache(maxsize=10) - def get_error_thumbnail(self, size): - # Draw an image with an error icon. Cache the result since we may need the error icon repeatedly. - artist = wx.ArtProvider() - size //= 2 - return artist.GetBitmap(wx.ART_WARNING, size=(size, size)) - def populate_tree(self, data, parent=None): # Build the tree view if parent is None: @@ -619,11 +561,12 @@ def populate_tree_screen(self, data, parent=None): class ImagePanel(wx.Panel): """ The ImagePanel displays an image's name and a preview thumbnail as a wx.Bitmap. + + Tiles are initialised with a loading icon, call update_thumbnail once image data has arrived for display. """ - def __init__(self, thumbnail, parent, omero_id, name, server, size=128): + def __init__(self, parent, omero_id, name, server, size=128): """ - thumbnail - wx.Bitmap parent - parent window to the wx.Panel omero_id - OMERO id of the image name - name to display @@ -631,18 +574,27 @@ def __init__(self, thumbnail, parent, omero_id, name, server, size=128): size - int dimension of the thumbnail to display """ self.parent = parent - self.bitmap = thumbnail + self.bitmap = None self.selected = False self.omero_id = omero_id self.url = f"https://{server}/webclient/?show=image-{omero_id}" self.name = name - if len(name) > 17: - self.shortname = name[:14] + '...' + max_len = int(17 / 128 * size) + if len(name) > max_len: + self.shortname = name[:max_len - 3] + '...' else: self.shortname = name self.size_x = size self.size_y = size + 30 wx.Panel.__init__(self, parent, wx.NewId(), size=(self.size_x, self.size_y)) + indicator_size = 64 + self.loading = wx.ActivityIndicator(self, + size=wx.Size(indicator_size, indicator_size), + pos=((self.size_x - indicator_size) // 2, + ((self.size_x - indicator_size) // 2) + 20) + ) + self.loading.Start() + self.parent.thumbnail_queue.put((omero_id, size, self.update_thumbnail)) self.Bind(wx.EVT_PAINT, self.OnPaint) self.Bind(wx.EVT_LEFT_DOWN, self.select) self.Bind(wx.EVT_RIGHT_DOWN, self.right_click) @@ -686,21 +638,32 @@ def open_in_browser(self, e): # Open in OMERO.web wx.LaunchDefaultBrowser(self.url) + def update_thumbnail(self, bitmap): + # Replace the temporary loading icon with a thumbnail image + if not self.__nonzero__() or self.IsBeingDeleted(): + # Skip update if the tile has already been deleted from the panel + return + if self.loading is not None: + # Remove the loading widget + self.loading.Destroy() + self.bitmap = bitmap + self.Refresh() + def OnPaint(self, evt): # Custom paint handler to display image/label/selection marker. dc = wx.PaintDC(self) dc.Clear() - dc.DrawBitmap(self.bitmap, (self.size_x - self.bitmap.Width) // 2, - ((self.size_x - self.bitmap.Height) // 2) + 20) + if self.bitmap is not None: + dc.DrawBitmap(self.bitmap, (self.size_x - self.bitmap.Width) // 2, + ((self.size_x - self.bitmap.Height) // 2) + 20) rect = wx.Rect(0, 0, self.size_x, self.size_x + 20) dc.DrawLabel(self.shortname, rect, alignment=wx.ALIGN_CENTER_HORIZONTAL | wx.ALIGN_TOP) - dc.SetPen(wx.Pen("GREY", style=wx.PENSTYLE_SOLID)) + if self.selected: + dc.SetPen(wx.Pen("SLATE BLUE", 3, style=wx.PENSTYLE_SOLID)) + else: + dc.SetPen(wx.Pen("GREY", 1, style=wx.PENSTYLE_SOLID)) dc.SetBrush(wx.Brush("BLACK", wx.TRANSPARENT)) dc.DrawRectangle(rect) - # Outline the whole image - if self.selected: - dc.SetPen(wx.Pen("SLATE BLUE", 3)) - dc.DrawRectangle(0, 0, self.size_x, self.size_y) return dc @@ -713,10 +676,69 @@ def __init__(self, *args, **kwargs): super(self.__class__, self).__init__(*args, **kwargs) self.select_source = None self.select_box = None + + self.credentials = CREDENTIALS + + self.active = True + self.thumbnail_queue = queue.Queue() + self.thumbnail_thread = threading.Thread(name="ThumbnailProvider", target=self.thumbnail_loader, daemon=True) + self.thumbnail_thread.start() + self.Bind(wx.EVT_MOTION, self.OnMotion) self.Bind(wx.EVT_PAINT, self.OnPaint) self.Bind(wx.EVT_LEFT_UP, self.on_release) + def thumbnail_loader(self): + # Spin and monitor queue + # Jobs will arrive as tuples of (omero id, thumbnail size, tile update function). + chunk_size = 10 + LOGGER.debug("Starting thumbnail loader") + size = 0 + callback_map = {} + while self.active: + if self.thumbnail_queue.empty(): + time.sleep(0.1) + for _ in range(chunk_size): + if not self.thumbnail_queue.empty(): + omero_id, size, callback = self.thumbnail_queue.get() + callback_map[str(omero_id)] = callback + else: + break + if callback_map: + ids_to_fetch = list(callback_map.keys()) + ids_str = '&id='.join(ids_to_fetch) + url = f"https://{self.credentials.server}/webclient/get_thumbnails/{size}/?&bsession={self.credentials.session_key}&id={ids_str}" + LOGGER.debug(f"Fetching {url}") + result = {} + try: + data = requests.get(url, timeout=10) + if data.status_code != 200: + LOGGER.warning(f"Server error: {data.status_code} - {data.reason}") + else: + result.update(data.json()) + except requests.exceptions.Timeout: + LOGGER.error("URL fetch timed out") + except Exception: + LOGGER.error("Unable to retrieve data", exc_info=True) + for omero_id, callback in callback_map.items(): + image_data = result.get(omero_id, "") + start_data = image_data.find('/9') + if start_data == -1: + LOGGER.info(f"No thumbnail data was returned for image {omero_id}") + img = self.get_error_thumbnail(size) + else: + decoded = base64.b64decode(image_data[start_data:]) + bio = io.BytesIO(decoded) + img = wx.Image(bio) + if not img.IsOk(): + LOGGER.info(f"Thumbnail data was invalid for image {omero_id}") + img = self.get_error_thumbnail(size) + else: + img = img.ConvertToBitmap() + # Update the tile in question. This must be scheduled on the main GUI thread to avoid crashes. + wx.CallAfter(callback, img) + callback_map = {} + def deselect_all(self): for child in self.GetChildren(): if isinstance(child, ImagePanel): @@ -759,3 +781,12 @@ def OnPaint(self, e): dc.SetBrush(wx.Brush("BLUE", style=wx.TRANSPARENT)) if self.select_box is not None: dc.DrawRectangle(self.select_box) + + @functools.lru_cache(maxsize=10) + def get_error_thumbnail(self, size): + # Draw an image with an error icon. Cache the result since we may need the error icon repeatedly. + artist = wx.ArtProvider() + size //= 2 + return artist.GetBitmap(wx.ART_WARNING, size=(size, size)) + +# TODO: Paginate well loading diff --git a/active_plugins/omeroreader.py b/active_plugins/omeroreader.py index 9cce36f..21570b3 100644 --- a/active_plugins/omeroreader.py +++ b/active_plugins/omeroreader.py @@ -452,6 +452,3 @@ def get_settings(): bool, True) ] - - -# Todo: Thumbnail queue From 934de8e97cecc90211f1ae8b8e2c809e72ae0153 Mon Sep 17 00:00:00 2001 From: David Stirling Date: Fri, 4 Aug 2023 11:55:01 +0100 Subject: [PATCH 11/32] Add install system and docs --- active_plugins/omeroreader.py | 5 +++ .../supported_plugins.md | 33 ++++++++++--------- setup.py | 29 ++++++++++++---- 3 files changed, 44 insertions(+), 23 deletions(-) diff --git a/active_plugins/omeroreader.py b/active_plugins/omeroreader.py index 21570b3..d0f2386 100644 --- a/active_plugins/omeroreader.py +++ b/active_plugins/omeroreader.py @@ -2,6 +2,11 @@ An image reader which connects to OMERO to load data # Installation - +Easy mode - clone the plugins repository and point your CellProfiler plugins folder to this folder. +Navigate to /active_plugins/ and run `pip install -e .[omero]` to install dependencies. + +## Manual Installation + Add this file plus the `omero_helper` directory into your CellProfiler plugins folder. Install dependencies into your CellProfiler Python environment. diff --git a/documentation/CP-plugins-documentation/supported_plugins.md b/documentation/CP-plugins-documentation/supported_plugins.md index 5ccdda9..503b26d 100644 --- a/documentation/CP-plugins-documentation/supported_plugins.md +++ b/documentation/CP-plugins-documentation/supported_plugins.md @@ -9,19 +9,20 @@ See [using plugins](using_plugins.md) for how to set up CellProfiler for plugin Most plugin documentation can be found within the plugin itself and can be accessed through CellProfiler help. Those plugins that do have extra documentation contain links below. -| Plugin | Description | Requires installation of dependencies? | Install flag | -|--------|-------------|----------------------------------------|--------------| -| CalculateMoments | CalculateMoments extracts moments statistics from a given distribution of pixel values. | No | | -| CallBarcodes | CallBarcodes is used for assigning a barcode to an object based on the channel with the strongest intensity for a given number of cycles. It is used for optical sequencing by synthesis (SBS). | No | | -| CompensateColors | CompensateColors determines how much signal in any given channel is because of bleed-through from another channel and removes the bleed-through. It can be performed across an image or masked to objects and provides a number of preprocessing and rescaling options to allow for troubleshooting if input image intensities are not well matched. | No | | -| DistanceTransform | DistanceTransform computes the distance transform of a binary image. The distance of each foreground pixel is computed to the nearest background pixel and the resulting image is then scaled so that the largest distance is 1. | No | | -| EnhancedMeasureTexture| EnhancedMeasureTexture measures the degree and nature of textures within an image or objects in a more comprehensive/tuneable manner than the MeasureTexture module native to CellProfiler. | No | | -| HistogramEqualization | HistogramEqualization increases the global contrast of a low-contrast image or volume. Histogram equalization redistributes intensities to utilize the full range of intensities, such that the most common frequencies are more distinct. This module can perform either global or local histogram equalization. | No | | -| HistogramMatching | HistogramMatching manipulates the pixel intensity values an input image and matches them to the histogram of a reference image. It can be used as a way to normalize intensities across different 2D or 3D images or different frames of the same 3D image. It allows you to choose which frame to use as the reference. | No | | -| PixelShuffle | PixelShuffle takes the intensity of each pixel in an image and randomly shuffles its position. | No | | -| Predict | Predict allows you to use an ilastik pixel classifier to generate a probability image. CellProfiler supports two types of ilastik projects: Pixel Classification and Autocontext (2-stage). | No | | -| [RunCellpose](RunCellPose.md) | RunCellpose allows you to run Cellpose within CellProfiler. Cellpose is a generalist machine-learning algorithm for cellular segmentation and is a great starting point for segmenting non-round cells. You can use pre-trained Cellpose models or your custom model with this plugin. You can use a GPU with this module to dramatically increase your speed/efficiency. | Yes | `cellpose` | -| RunImageJScript | RunImageJScript allows you to run any supported ImageJ script directly within CellProfiler. It is significantly more performant than RunImageJMacro, and is also less likely to leave behind temporary files. | Yes | XXXXX | -| RunOmnipose | RunOmnipose allows you to run Omnipose within CellProfiler. Omnipose is a general image segmentation tool that builds on Cellpose. | Yes | `omnipose` | -| RunStarDist | RunStarDist allows you to run StarDist within CellProfiler. StarDist is a machine-learning algorithm for object detection with star-convex shapes making it best suited for nuclei or round-ish cells. You can use pre-trained StarDist models or your custom model with this plugin. You can use a GPU with this module to dramatically increase your speed/efficiency. RunStarDist is generally faster than RunCellpose. | Yes | `stardist` | -| VarianceTransform | This module allows you to calculate the variance of an image, using a determined window size. It also has the option to find the optimal window size from a predetermined range to obtain the maximum variance of an image. | No | | \ No newline at end of file +| Plugin | Description | Requires installation of dependencies? | Install flag | +|-------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------|--------------| +| CalculateMoments | CalculateMoments extracts moments statistics from a given distribution of pixel values. | No | | +| CallBarcodes | CallBarcodes is used for assigning a barcode to an object based on the channel with the strongest intensity for a given number of cycles. It is used for optical sequencing by synthesis (SBS). | No | | +| CompensateColors | CompensateColors determines how much signal in any given channel is because of bleed-through from another channel and removes the bleed-through. It can be performed across an image or masked to objects and provides a number of preprocessing and rescaling options to allow for troubleshooting if input image intensities are not well matched. | No | | +| DistanceTransform | DistanceTransform computes the distance transform of a binary image. The distance of each foreground pixel is computed to the nearest background pixel and the resulting image is then scaled so that the largest distance is 1. | No | | +| EnhancedMeasureTexture | EnhancedMeasureTexture measures the degree and nature of textures within an image or objects in a more comprehensive/tuneable manner than the MeasureTexture module native to CellProfiler. | No | | +| HistogramEqualization | HistogramEqualization increases the global contrast of a low-contrast image or volume. Histogram equalization redistributes intensities to utilize the full range of intensities, such that the most common frequencies are more distinct. This module can perform either global or local histogram equalization. | No | | +| HistogramMatching | HistogramMatching manipulates the pixel intensity values an input image and matches them to the histogram of a reference image. It can be used as a way to normalize intensities across different 2D or 3D images or different frames of the same 3D image. It allows you to choose which frame to use as the reference. | No | | +| PixelShuffle | PixelShuffle takes the intensity of each pixel in an image and randomly shuffles its position. | No | | +| Predict | Predict allows you to use an ilastik pixel classifier to generate a probability image. CellProfiler supports two types of ilastik projects: Pixel Classification and Autocontext (2-stage). | No | | +| [RunCellpose](RunCellPose.md) | RunCellpose allows you to run Cellpose within CellProfiler. Cellpose is a generalist machine-learning algorithm for cellular segmentation and is a great starting point for segmenting non-round cells. You can use pre-trained Cellpose models or your custom model with this plugin. You can use a GPU with this module to dramatically increase your speed/efficiency. | Yes | `cellpose` | +| RunImageJScript | RunImageJScript allows you to run any supported ImageJ script directly within CellProfiler. It is significantly more performant than RunImageJMacro, and is also less likely to leave behind temporary files. | Yes | XXXXX | +| RunOmnipose | RunOmnipose allows you to run Omnipose within CellProfiler. Omnipose is a general image segmentation tool that builds on Cellpose. | Yes | `omnipose` | +| RunStarDist | RunStarDist allows you to run StarDist within CellProfiler. StarDist is a machine-learning algorithm for object detection with star-convex shapes making it best suited for nuclei or round-ish cells. You can use pre-trained StarDist models or your custom model with this plugin. You can use a GPU with this module to dramatically increase your speed/efficiency. RunStarDist is generally faster than RunCellpose. | Yes | `stardist` | +| VarianceTransform | This module allows you to calculate the variance of an image, using a determined window size. It also has the option to find the optimal window size from a predetermined range to obtain the maximum variance of an image. | No | | +| OMEROReader | This reader allows you to connect to and load images from OMERO servers. | Yes | 'omero' | \ No newline at end of file diff --git a/setup.py b/setup.py index 03d583a..b2ac821 100644 --- a/setup.py +++ b/setup.py @@ -24,14 +24,29 @@ "pyimagej" ] +# The zeroc-ice version OMERO needs is very difficult to build, so here are some premade wheels for a bunch of platforms +omero_deps = [ + "zeroc-ice @ https://github.com/glencoesoftware/zeroc-ice-py-macos-x86_64/releases/download/20220722/zeroc_ice-3.6.5-cp310-cp310-macosx_10_15_x86_64.whl ; sys_platform == 'darwin' and python_version == '3.10'", + "zeroc-ice @ https://github.com/glencoesoftware/zeroc-ice-py-macos-x86_64/releases/download/20220722/zeroc_ice-3.6.5-cp39-cp39-macosx_10_15_x86_64.whl ; sys_platform == 'darwin' and python_version == '3.9'", + "zeroc-ice @ https://github.com/glencoesoftware/zeroc-ice-py-macos-x86_64/releases/download/20220722/zeroc_ice-3.6.5-cp38-cp38-macosx_10_15_x86_64.whl ; sys_platform == 'darwin' and python_version == '3.8'", + "zeroc-ice @ https://github.com/glencoesoftware/zeroc-ice-py-ubuntu2204-x86_64/releases/download/20221004/zeroc_ice-3.6.5-cp310-cp310-linux_x86_64.whl ; 'Ubuntu' in platform_version and python_version == '3.10'", + "zeroc-ice @ https://github.com/glencoesoftware/zeroc-ice-py-linux-x86_64/releases/download/20221003/zeroc_ice-3.6.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl ; sys_platform == 'linux' and python_version == '3.10' and 'Ubuntu' not in platform_version", + "zeroc-ice @ https://github.com/glencoesoftware/zeroc-ice-py-linux-x86_64/releases/download/20221003/zeroc_ice-3.6.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl ; sys_platform == 'linux' and python_version == '3.9'", + "zeroc-ice @ https://github.com/glencoesoftware/zeroc-ice-py-linux-x86_64/releases/download/20221003/zeroc_ice-3.6.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl ; sys_platform == 'linux' and python_version == '3.8'", + "omero-py", + "omero-user-token", +] + + setup( name="cellprofiler_plugins", packages=setuptools.find_packages(), - install_requires = install_deps, - extras_require = { - "cellpose": cellpose_deps, - "omnipose": omnipose_deps, - "stardist": stardist_deps, - "imagejscript": imagejscript_deps, - } + install_requires=install_deps, + extras_require={ + "cellpose": cellpose_deps, + "omnipose": omnipose_deps, + "stardist": stardist_deps, + "imagejscript": imagejscript_deps, + "omero": omero_deps, + }, ) From 52b644fbaf7210a0aa7adcec0401fa8782fee6d4 Mon Sep 17 00:00:00 2001 From: David Stirling Date: Sun, 6 Aug 2023 14:13:12 +0100 Subject: [PATCH 12/32] Require OMERO token, fix analysis mode by passing session to workers with temp dir token --- active_plugins/omero_helper/connect.py | 105 +++++++++++++++++++------ active_plugins/omero_helper/gui.py | 6 +- active_plugins/omeroreader.py | 3 +- 3 files changed, 87 insertions(+), 27 deletions(-) diff --git a/active_plugins/omero_helper/connect.py b/active_plugins/omero_helper/connect.py index 640784f..e9c647d 100644 --- a/active_plugins/omero_helper/connect.py +++ b/active_plugins/omero_helper/connect.py @@ -1,40 +1,63 @@ import atexit import os import logging -import importlib.util +import tempfile -from cellprofiler_core.preferences import config_read_typed, get_headless +from cellprofiler_core.preferences import config_read_typed, get_headless, get_temporary_directory from cellprofiler_core.preferences import get_omero_server, get_omero_port, get_omero_user import omero +from omero.gateway import BlitzGateway +import omero_user_token LOGGER = logging.getLogger(__name__) +OMERO_CREDENTIAL_FILE = os.path.join(get_temporary_directory(), "OMERO_CP.token") -# Is omero_user_token installed and available? -TOKEN_MODULE = importlib.util.find_spec("omero_user_token") -TOKENS_AVAILABLE = TOKEN_MODULE is not None -if TOKENS_AVAILABLE: - # Only load and enable user tokens if dependency is installed - omero_user_token = importlib.util.module_from_spec(TOKEN_MODULE) - TOKEN_MODULE.loader.exec_module(omero_user_token) - -def login(e=None, server=None): +def login(e=None, server=None, token_path=None): # Attempt to connect to the server, first using a token, then via GUI - CREDENTIALS.get_tokens() + CREDENTIALS.get_tokens(token_path) if CREDENTIALS.tokens: if server is None: # URL didn't specify which server we want. Just try whichever token is available server = list(CREDENTIALS.tokens.keys())[0] connected = CREDENTIALS.try_token(server) - if get_headless(): - if connected: - LOGGER.info("Connected to ", CREDENTIALS.server) - else: - LOGGER.warning("Failed to connect, was user token invalid?") - return connected + else: + connected = CREDENTIALS.client is not None + if get_headless(): + if connected: + LOGGER.info(f"Connected to {CREDENTIALS.server}") + elif CREDENTIALS.try_temp_token(): + connected = True + LOGGER.info(f"Connected to {CREDENTIALS.server}") else: - from .gui import login_gui - login_gui(connected, server=None) + LOGGER.warning("Failed to connect, was user token invalid?") + return connected + else: + from .gui import login_gui + login_gui(connected, server=None) + + +def get_temporary_dir(): + temporary_directory = get_temporary_directory() + if not ( + os.path.exists(temporary_directory) and os.access(temporary_directory, os.W_OK) + ): + temporary_directory = tempfile.gettempdir() + return temporary_directory + + +def clear_temporary_file(): + LOGGER.debug("Checking for OMERO credential file to delete") + if os.path.exists(OMERO_CREDENTIAL_FILE): + os.unlink(OMERO_CREDENTIAL_FILE) + LOGGER.debug(f"Cleared {OMERO_CREDENTIAL_FILE}") + + +if not get_headless(): + if os.path.exists(OMERO_CREDENTIAL_FILE): + LOGGER.warning("Existing credential file was found") + # Main GUI process should clear any temporary tokens + atexit.register(clear_temporary_file) class LoginHelper: @@ -59,18 +82,25 @@ def __init__(self): self.session_key = None self.session = None self.client = None + self.gateway = None self.container_service = None self.tokens = {} # Any OMERO browser GUI which is connected to this object self.browser_window = None atexit.register(self.shutdown) + def get_gateway(self): + if self.client is None: + raise Exception("Client connection not initialised") + if self.gateway is None: + LOGGER.debug("Constructing BlitzGateway") + self.gateway = BlitzGateway(client_obj=self.client) + return self.gateway + def get_tokens(self, path=None): # Load all tokens from omero_user_token self.tokens.clear() # Future versions of omero_user_token may support multiple tokens, so we code with that in mind. - if not TOKENS_AVAILABLE: - return # Check the reader setting which disables tokens. tokens_enabled = config_read_typed(f"Reader.OMERO.allow_token", bool) if tokens_enabled is not None and not tokens_enabled: @@ -101,8 +131,37 @@ def try_token(self, address): server, port, session_key = self.tokens[address] return self.login(server=server, port=port, session_key=session_key) + def create_temp_token(self): + # Store a temporary OMERO token based on our active session + # This allows the workers to use that session in Analysis mode. + if self.client is None: + raise ValueError("Client not initialised, cannot make token") + if os.path.exists(OMERO_CREDENTIAL_FILE): + LOGGER.warning(f"Token already exists at {OMERO_CREDENTIAL_FILE}, overwriting") + os.unlink(OMERO_CREDENTIAL_FILE) + try: + token = f"{self.session_key}@{self.server}:{self.port}" + with open(OMERO_CREDENTIAL_FILE, 'w') as token_file: + token_file.write(token) + LOGGER.debug(f"Made temp token for {self.server}") + except: + LOGGER.error("Unable to write temporary token", exc_info=True) + + def try_temp_token(self): + # Look for and attempt to connect to OMERO using a temporary token. + if not os.path.exists(OMERO_CREDENTIAL_FILE): + LOGGER.error(f"No temporary OMERO token found. Cannot connect to server.") + return False + with open(OMERO_CREDENTIAL_FILE, 'r') as token_path: + token = token_path.read().strip() + server, port = token[token.find('@') + 1:].split(':') + port = int(port) + session_key = token[:token.find('@')] + LOGGER.info(f"Using connection details for {self.server}") + return self.login(server=server, port=port, session_key=session_key) + def login(self, server=None, port=None, user=None, passwd=None, session_key=None): - # Attempt to connect to the server + # Attempt to connect to the server using provided connection credentials self.client = omero.client(host=server, port=port) if session_key is not None: try: diff --git a/active_plugins/omero_helper/gui.py b/active_plugins/omero_helper/gui.py index 4d5a93e..6a0852e 100644 --- a/active_plugins/omero_helper/gui.py +++ b/active_plugins/omero_helper/gui.py @@ -14,7 +14,7 @@ from cellprofiler_core.preferences import config_read_typed, config_write_typed, \ set_omero_server, set_omero_port, set_omero_user -from .connect import CREDENTIALS, TOKENS_AVAILABLE, login +from .connect import CREDENTIALS, login LOGGER = logging.getLogger(__name__) @@ -46,6 +46,8 @@ def show_login_dlg(e=None, server=None): frame = app.GetTopWindow() with OmeroLoginDlg(frame, title="Log into Omero", server=server) as dlg: dlg.ShowModal() + if CREDENTIALS.client is not None: + CREDENTIALS.create_temp_token() def browse(e): @@ -183,7 +185,7 @@ def on_connect_pressed(self, event): self.connect() def on_set_pressed(self, event): - if self.credentials.client is None or not TOKENS_AVAILABLE: + if self.credentials.client is None: return import omero_user_token token_path = omero_user_token.assert_and_get_token_path() diff --git a/active_plugins/omeroreader.py b/active_plugins/omeroreader.py index d0f2386..e737fda 100644 --- a/active_plugins/omeroreader.py +++ b/active_plugins/omeroreader.py @@ -11,8 +11,7 @@ your CellProfiler Python environment. ## Installing dependencies - -This depends on platform. At the most basic level you'll need the `omero-py` package. For headless run and -more convenient server login you'll also want the `omero_user_token` package. +This depends on platform. At the most basic level you'll need the `omero-py` package and the `omero_user_token` package. Both should be possible to pip install on Windows. On MacOS, you'll probably have trouble with the zeroc-ice dependency. omero-py uses an older version and so needs specific wheels. Fortunately we've built some for you. From dc9cf06bb17ad10b331e4b5f9c77de3080819184 Mon Sep 17 00:00:00 2001 From: David Stirling Date: Tue, 8 Aug 2023 11:28:29 +0100 Subject: [PATCH 13/32] Add ExportToOMEROTable module --- active_plugins/exporttoomerotable.py | 847 +++++++++++++++++++++++++++ active_plugins/omeroreader.py | 4 +- 2 files changed, 849 insertions(+), 2 deletions(-) create mode 100644 active_plugins/exporttoomerotable.py diff --git a/active_plugins/exporttoomerotable.py b/active_plugins/exporttoomerotable.py new file mode 100644 index 0000000..d45e7a3 --- /dev/null +++ b/active_plugins/exporttoomerotable.py @@ -0,0 +1,847 @@ +""" +ExportToOMEROTable +================== + +**ExportToOMEROTable** exports measurements directly into an +OMERO.table stored on an OMERO server. + +An uploaded table is viewable in OMERO.web, it will be uploaded as an attachment to +an existing OMERO object. + +# Installation - +Easy mode - clone the plugins repository and point your CellProfiler plugins folder to this folder. +Navigate to /active_plugins/ and run `pip install -e .[omero]` to install dependencies. + +## Manual Installation + +Add this file plus the `omero_helper` directory into your CellProfiler plugins folder. Install dependencies into +your CellProfiler Python environment. + +## Installing dependencies - +This depends on platform. At the most basic level you'll need the `omero-py` package and the `omero_user_token` package. + +Both should be possible to pip install on Windows. On MacOS, you'll probably have trouble with the zeroc-ice dependency. +omero-py uses an older version and so needs specific wheels. Fortunately we've built some for you. +Macos - https://github.com/glencoesoftware/zeroc-ice-py-macos-x86_64/releases/latest +Linux (Generic) - https://github.com/glencoesoftware/zeroc-ice-py-linux-x86_64/releases/latest +Ubuntu 22.04 - https://github.com/glencoesoftware/zeroc-ice-py-ubuntu2204-x86_64/releases/latest + +Download the .whl file from whichever is most appropriate and run `pip install `. + +From there pip install omero-py should do the rest. + +You'll also want the `omero_user_token` package to help manage logins (`pip install omero_user_token`). +This allows you to set reusable login tokens for quick reconnection to a server. These tokens are required for using +headless mode/analysis mode. + +# Limitations + +- OMERO tables cannot have their columns changed after being initialised. For now, +this means that measurements cannot be added after the pipeline finishes (e.g. per-well averages). +For most use cases you can export all measurements produced by the pipeline, it'll be results from +complex modules like the LAP tracker in TrackObjects which cannot be fully exported. + +- Groupings from the Groups module are currently not implemented. Everything goes into a single table. +This may be added in a future version, but for now a single table per image/object type is created +without support for splitting (much like ExportToDatabase). + +- There is a limit to how much data can be transmitted to OMERO in a single operation. This +causes issues when creating very large tables. In practice you may encounter issues when trying to +export a table with more than ~600 columns, depending on the column name lengths. + +| + +============ ============ =============== +Supports 2D? Supports 3D? Respects masks? +============ ============ =============== +YES YES YES +============ ============ =============== + + +""" + +import functools +import logging +import math +import os +import re +from collections.abc import Iterable + +import cellprofiler_core.pipeline +import cellprofiler_core.utilities.legacy +from cellprofiler_core.constants.measurement import AGG_MEAN +from cellprofiler_core.constants.measurement import AGG_MEDIAN +from cellprofiler_core.constants.measurement import AGG_STD_DEV +from cellprofiler_core.constants.measurement import EXPERIMENT +from cellprofiler_core.constants.measurement import M_NUMBER_OBJECT_NUMBER +from cellprofiler_core.constants.measurement import NEIGHBORS +from cellprofiler_core.constants.measurement import OBJECT +from cellprofiler_core.module import Module +from cellprofiler_core.preferences import get_headless +from cellprofiler_core.setting import Binary +from cellprofiler_core.setting import ValidationError +from cellprofiler_core.setting.choice import Choice +from cellprofiler_core.setting.do_something import DoSomething +from cellprofiler_core.setting.subscriber import LabelListSubscriber +from cellprofiler_core.setting.text import Integer, Text +from cellprofiler_core.utilities.measurement import agg_ignore_feature +from cellprofiler_core.constants.measurement import COLTYPE_INTEGER +from cellprofiler_core.constants.measurement import COLTYPE_FLOAT +from cellprofiler_core.constants.measurement import COLTYPE_VARCHAR +from cellprofiler_core.constants.measurement import COLTYPE_BLOB +from cellprofiler_core.constants.measurement import COLTYPE_MEDIUMBLOB +from cellprofiler_core.constants.measurement import COLTYPE_LONGBLOB + +import omero +import omero.grid + +from omero_helper.connect import CREDENTIALS, login + +LOGGER = logging.getLogger(__name__) + +############################################## +# +# Keyword for the cached data +# +############################################## +# Measurement column info +D_MEASUREMENT_COLUMNS = "MeasurementColumns" +# OMERO table locations/metadata. +OMERO_TABLE_KEY = "OMERO_tables" + +"""The column name for the image number column""" +C_IMAGE_NUMBER = "ImageNumber" + +"""The column name for the object number column""" +C_OBJECT_NUMBER = "ObjectNumber" + + +############################################## +# +# Choices for which objects to include +# +############################################## + +"""Put all objects in the database""" +O_ALL = "All" +"""Don't put any objects in the database""" +O_NONE = "None" +"""Select the objects you want from a list""" +O_SELECT = "Select..." + + +############################################## +# +# Constants for interacting with the OMERO tables API +# +############################################## +# Map from CellProfiler - OMERO column type +COLUMN_TYPES = { + COLTYPE_INTEGER: omero.grid.LongColumn, + COLTYPE_FLOAT: omero.grid.DoubleColumn, + COLTYPE_VARCHAR: omero.grid.StringColumn, + COLTYPE_BLOB: omero.grid.StringColumn, + COLTYPE_MEDIUMBLOB: omero.grid.StringColumn, + COLTYPE_LONGBLOB: omero.grid.StringColumn, +} + +# OMERO columns with special meaning +SPECIAL_NAMES = { + 'roi': omero.grid.RoiColumn, + 'image': omero.grid.ImageColumn, + 'dataset': omero.grid.DatasetColumn, + 'well': omero.grid.WellColumn, + 'field': omero.grid.ImageColumn, + 'wellsample': omero.grid.ImageColumn, + 'plate': omero.grid.PlateColumn, +} + +# Link annotations needed for each parent type +LINK_TYPES = { + "Image": omero.model.ImageAnnotationLinkI, + "Dataset": omero.model.DatasetAnnotationLinkI, + "Screen": omero.model.ScreenAnnotationLinkI, + "Plate": omero.model.PlateAnnotationLinkI, + "Well": omero.model.WellAnnotationLinkI, +} + +# OMERO types for each parent type +OBJECT_TYPES = { + "Image": omero.model.ImageI, + "Dataset": omero.model.DatasetI, + "Screen": omero.model.ScreenI, + "Plate": omero.model.PlateI, + "Well": omero.model.WellI, +} + + +class ExportToOMEROTable(Module): + module_name = "ExportToOMEROTable" + variable_revision_number = 1 + category = ["File Processing", "Data Tools"] + + def create_settings(self): + self.target_object_type = Choice( + "OMERO parent object type", + ["Image", "Dataset", "Project", "Screen", "Plate"], + doc="""\ + The created OMERO.table must be associated with an existing object + in OMERO. Select the type of object you'd like to attach the table + to.""" + ) + + self.target_object_id = Integer( + text="OMERO ID of the parent object", + minval=1, + doc="""\ + The created OMERO.table must be associated with an existing object + in OMERO. Enter the OMERO ID of the object you'd like to associate + the table(s) with. This ID can be found by locating the target object + in OMERO.web (ID and type is displayed in the right panel).""", + ) + + + self.test_connection_button = DoSomething( + "Test the OMERO connection", + "Test connection", + self.test_connection, + doc="""\ +This button test the connection to the OMERO server specified using +the settings entered by the user.""", + ) + + self.want_table_prefix = Binary( + "Add a prefix to table names?", + True, + doc="""\ +Select whether you want to add a prefix to your table names. The default +table names are *Per\_Image* for the per-image table and *Per\_Object* +for the per-object table. Adding a prefix can be useful for bookkeeping +purposes. + +- Select "*{YES}*" to add a user-specified prefix to the default table + names. If you want to distinguish multiple sets of data written to + the same database, you probably want to use a prefix. +- Select "*{NO}*" to use the default table names. For a one-time export + of data, this option is fine. + +Whether you chose to use a prefix or not, CellProfiler will warn you if +your choice entails overwriting an existing table. +""".format( + **{"YES": "Yes", "NO": "No"} + ), + ) + + self.table_prefix = Text( + "Table prefix", + "MyExpt_", + doc="""\ +*(Used if "Add a prefix to table names?" is selected)* + +Enter the table prefix you want to use. +""", + ) + + + self.wants_agg_mean = Binary( + "Calculate the per-image mean values of object measurements?", + True, + doc="""\ +Select "*Yes*" for **ExportToOMEROTable** to calculate population +statistics over all the objects in each image and store the results in +the database. For instance, if you are measuring the area of the Nuclei +objects and you check the box for this option, **ExportToOMEROTable** will +create a column in the Per\_Image table called +“Mean\_Nuclei\_AreaShape\_Area”. + +You may not want to use **ExportToOMEROTable** to calculate these +population statistics if your pipeline generates a large number of +per-object measurements; doing so might exceed table column limits. +""", + ) + + self.wants_agg_median = Binary( + "Calculate the per-image median values of object measurements?", + False, + doc="""\ +Select "*Yes*" for **ExportToOMEROTable** to calculate population +statistics over all the objects in each image and store the results in +the database. For instance, if you are measuring the area of the Nuclei +objects and you check the box for this option, **ExportToOMEROTable** will +create a column in the Per\_Image table called +“Median\_Nuclei\_AreaShape\_Area”. + +You may not want to use **ExportToOMEROTable** to calculate these +population statistics if your pipeline generates a large number of +per-object measurements; doing so might exceed table column limits. +""", + ) + + self.wants_agg_std_dev = Binary( + "Calculate the per-image standard deviation values of object measurements?", + False, + doc="""\ +Select "*Yes*" for **ExportToOMEROTable** to calculate population +statistics over all the objects in each image and store the results in +the database. For instance, if you are measuring the area of the Nuclei +objects and you check the box for this option, **ExportToOMEROTable** will +create a column in the Per\_Image table called +“StDev\_Nuclei\_AreaShape\_Area”. + +You may not want to use **ExportToOMEROTable** to calculate these +population statistics if your pipeline generates a large number of +per-object measurements; doing so might exceed database column limits. +""", + ) + + self.objects_choice = Choice( + "Export measurements for all objects to OMERO?", + [O_ALL, O_NONE, O_SELECT], + doc="""\ +This option lets you choose the objects whose measurements will be saved +in the Per\_Object and Per\_Well(s) OMERO tables. + +- *{O_ALL}:* Export measurements from all objects. +- *{O_NONE}:* Do not export data to a Per\_Object table. Save only + Per\_Image measurements (which nonetheless include + population statistics from objects). +- *{O_SELECT}:* Select the objects you want to export from a list. +""".format( + **{"O_ALL": O_ALL, "O_NONE": O_NONE, "O_SELECT": O_SELECT} + ), + ) + + self.objects_list = LabelListSubscriber( + "Select object tables to export", + [], + doc="""\ + *(Used only when "Within objects" or "Both" are selected)* + + Select the objects to be measured.""", + ) + + def visible_settings(self): + result = [self.target_object_type, self.target_object_id, + self.test_connection_button, self.want_table_prefix] + if self.want_table_prefix.value: + result += [self.table_prefix] + # Aggregations + result += [self.wants_agg_mean, self.wants_agg_median, self.wants_agg_std_dev] + # Table choices (1 / separate object tables, etc) + result += [self.objects_choice] + if self.objects_choice == O_SELECT: + result += [self.objects_list] + return result + + def settings(self): + result = [ + self.target_object_type, + self.target_object_id, + self.want_table_prefix, + self.table_prefix, + self.wants_agg_mean, + self.wants_agg_median, + self.wants_agg_std_dev, + self.objects_choice, + self.objects_list, + ] + return result + + def help_settings(self): + return [ + self.target_object_type, + self.target_object_id, + self.want_table_prefix, + self.table_prefix, + self.wants_agg_mean, + self.wants_agg_median, + self.wants_agg_std_dev, + self.objects_choice, + self.objects_list, + ] + + def validate_module(self, pipeline): + if self.want_table_prefix.value: + if not re.match("^[A-Za-z][A-Za-z0-9_]+$", self.table_prefix.value): + raise ValidationError("Invalid table prefix", self.table_prefix) + + if self.objects_choice == O_SELECT: + if len(self.objects_list.value) == 0: + raise ValidationError( + "Please choose at least one object", self.objects_choice + ) + + def validate_module_warnings(self, pipeline): + """Warn user re: Test mode """ + if pipeline.test_mode: + raise ValidationError( + "ExportToOMEROTable does not produce output in Test Mode", self.target_object_id + ) + + def test_connection(self): + """Check to make sure the OMERO server is remotely accessible""" + # CREDENTIALS is a singleton so we can safely grab it here. + if CREDENTIALS.client is None: + login() + if CREDENTIALS.client is None: + msg = "OMERO connection failed" + else: + msg = f"Connected to {CREDENTIALS.server}" + else: + msg = f"Already connected to {CREDENTIALS.server}" + if CREDENTIALS.client is not None: + try: + self.get_omero_parent() + msg += f"\n\nFound parent object {self.target_object_id}" + except ValueError as ve: + msg += f"\n\n{ve}" + + import wx + wx.MessageBox(msg) + + def make_full_filename(self, file_name, workspace=None, image_set_index=None): + """Convert a file name into an absolute path + + We do a few things here: + * apply metadata from an image set to the file name if an + image set is specified + * change the relative path into an absolute one using the "." and "&" + convention + * Create any directories along the path + """ + if image_set_index is not None and workspace is not None: + file_name = workspace.measurements.apply_metadata( + file_name, image_set_index + ) + measurements = None if workspace is None else workspace.measurements + path_name = self.directory.get_absolute_path(measurements, image_set_index) + file_name = os.path.join(path_name, file_name) + path, file = os.path.split(file_name) + if not os.path.isdir(path): + os.makedirs(path) + return os.path.join(path, file) + + @staticmethod + def connect_to_omero(): + if CREDENTIALS.client is None: + if get_headless(): + connected = login() + if not connected: + raise ValueError("No OMERO connection established") + else: + login() + if CREDENTIALS.client is None: + raise ValueError("OMERO connection failed") + + def prepare_run(self, workspace): + """Prepare to run the pipeline. + Establish a connection to OMERO and create the necessary tables.""" + # Reset shared state + self.get_dictionary().clear() + + pipeline = workspace.pipeline + if pipeline.test_mode: + # Don't generate in test mode + return + + if pipeline.in_batch_mode(): + return True + + # Verify that we're able to connect to a server + self.connect_to_omero() + + shared_state = self.get_dictionary() + + # Add a list of measurement columns into the module state, and fix their order. + if D_MEASUREMENT_COLUMNS not in shared_state: + shared_state[D_MEASUREMENT_COLUMNS] = pipeline.get_measurement_columns() + shared_state[D_MEASUREMENT_COLUMNS] = self.filter_measurement_columns( + shared_state[D_MEASUREMENT_COLUMNS] + ) + + # Build a list of tables to create + column_defs = shared_state[D_MEASUREMENT_COLUMNS] + desired_tables = ["Image"] + if self.objects_choice == O_SELECT: + desired_tables += self.objects_list.value + elif self.objects_choice == O_ALL: + desired_tables += self.get_object_names(pipeline) + + # Construct a list of tables in the format (CP name, OMERO name, OMERO ID, CP columns) + omero_table_list = [] + parent = self.get_omero_parent() + + workspace.display_data.header = ["Output", "Table Name", "OMERO ID", "Server Location"] + workspace.display_data.columns = [] + + for table_name in desired_tables: + true_name = self.get_table_name(table_name) + table_cols = [("", "ImageNumber", COLTYPE_INTEGER)] + if table_name != "Image": + table_cols.append(("", "ObjectNumber", COLTYPE_INTEGER)) + if table_name == OBJECT: + target_names = set(self.get_object_names(pipeline)) + else: + target_names = {table_name} + table_cols.extend([col for col in column_defs if col[0] in target_names]) + if table_name == "Image": + # Add any aggregate measurements + table_cols.extend(self.get_aggregate_columns(workspace.pipeline)) + omero_id = self.create_omero_table(parent, true_name, table_cols) + omero_table_list.append((table_name, true_name, omero_id, table_cols)) + table_path = f"https://{CREDENTIALS.server}/webclient/omero_table/{omero_id}" + LOGGER.info(f"Created table at {table_path}") + workspace.display_data.columns.append((table_name, true_name, omero_id, table_path)) + + shared_state[OMERO_TABLE_KEY] = omero_table_list + LOGGER.debug("Stored OMERO table info into shared state") + return True + + def get_omero_conn(self): + self.connect_to_omero() + return CREDENTIALS.get_gateway() + + def get_omero_parent(self): + conn = self.get_omero_conn() + parent_id = self.target_object_id.value + parent_type = self.target_object_type.value + old_group = conn.SERVICE_OPTS.getOmeroGroup() + # Search across groups + conn.SERVICE_OPTS.setOmeroGroup(-1) + parent_ob = conn.getObject(parent_type, parent_id) + if parent_ob is None: + raise ValueError(f"{parent_type} ID {parent_id} not found on server") + conn.SERVICE_OPTS.setOmeroGroup(old_group) + return parent_ob + + def create_omero_table(self, parent, table_name, column_defs): + """Creates a new OMERO table""" + conn = self.get_omero_conn() + parent_type = self.target_object_type.value + parent_id = self.target_object_id.value + parent_group = parent.details.group.id.val + + columns = self.generate_omero_columns(column_defs) + if len(columns) > 500: + LOGGER.warning(f"Large number of columns in table ({len(columns)})." + f"Plugin may encounter issues sending data to OMERO.") + resources = conn.c.sf.sharedResources(_ctx={ + "omero.group": str(parent_group)}) + repository_id = resources.repositories().descriptions[0].getId().getValue() + + table = None + try: + table = resources.newTable(repository_id, table_name, _ctx={ + "omero.group": str(parent_group)}) + table.initialize(columns) + LOGGER.info("Table creation complete, linking to image") + orig_file = table.getOriginalFile() + + # create file link + link_obj = LINK_TYPES[parent_type]() + target_obj = OBJECT_TYPES[parent_type](parent_id, False) + # create annotation + annotation = omero.model.FileAnnotationI() + # link table to annotation object + annotation.file = orig_file + + link_obj.link(target_obj, annotation) + conn.getUpdateService().saveObject(link_obj, _ctx={ + "omero.group": str(parent_group)}) + LOGGER.debug("Saved annotation link") + + LOGGER.info(f"Created table {table_name} under " + f"{parent_type} {parent_id}") + return orig_file.id.val + except Exception: + raise + finally: + if table is not None: + table.close() + + def get_omero_table(self, table_id): + conn = self.get_omero_conn() + old_group = conn.SERVICE_OPTS.getOmeroGroup() + # Search across groups + conn.SERVICE_OPTS.setOmeroGroup(-1) + table_file = conn.getObject("OriginalFile", table_id) + if table_file is None: + raise ValueError(f"OriginalFile ID {table_id} not found on server") + resources = conn.c.sf.sharedResources() + table = resources.openTable(table_file._obj) + conn.SERVICE_OPTS.setOmeroGroup(old_group) + return table + + def generate_omero_columns(self, column_defs): + omero_columns = [] + for object_name, measurement, column_type in column_defs: + if object_name: + column_name = f"{object_name}_{measurement}" + else: + column_name = measurement + cleaned_name = column_name.replace('/', '\\') + split_type = column_type.split('(', 1) + cleaned_type = split_type[0] + if column_name in SPECIAL_NAMES and column_type.kind == 'i': + col_class = SPECIAL_NAMES[column_name] + elif cleaned_type in COLUMN_TYPES: + col_class = COLUMN_TYPES[cleaned_type] + else: + raise NotImplementedError(f"Column type " + f"{cleaned_type} not supported") + if col_class == omero.grid.StringColumn: + if len(split_type) == 1: + max_len = 128 + else: + max_len = int(split_type[1][:-1]) + col = col_class(cleaned_name, "", max_len, []) + else: + col = col_class(cleaned_name, "", []) + omero_columns.append(col) + return omero_columns + + def run(self, workspace): + if workspace.pipeline.test_mode: + return + shared_state = self.get_dictionary() + omero_map = shared_state[OMERO_TABLE_KEY] + # Re-establish server connection + self.connect_to_omero() + + for table_type, table_name, table_file_id, table_columns in omero_map: + table = None + try: + table = self.get_omero_table(table_file_id) + self.write_data_to_omero(workspace, table_type, table, table_columns) + except: + LOGGER.error(f"Unable to write to table {table_name}", exc_info=True) + raise + finally: + if table is not None: + table.close() + + def write_data_to_omero(self, workspace, table_type, omero_table, column_list): + measurements = workspace.measurements + table_columns = omero_table.getHeaders() + # Collect any extra aggregate columns we might need. + extra_data = {C_IMAGE_NUMBER: measurements.image_set_number} + if table_type == "Image": + extra_data.update(measurements.compute_aggregate_measurements( + measurements.image_set_number, self.agg_names + )) + else: + extra_data[C_OBJECT_NUMBER] = measurements.get_measurement(table_type, M_NUMBER_OBJECT_NUMBER) + extra_data[C_IMAGE_NUMBER] = [extra_data[C_IMAGE_NUMBER]] * len(extra_data[C_OBJECT_NUMBER]) + + for omero_column, (col_type, col_name, _) in zip(table_columns, column_list): + if col_type: + true_name = f"{col_type}_{col_name}" + else: + true_name = col_name + if true_name in extra_data: + value = extra_data[true_name] + elif not measurements.has_current_measurements(col_type, col_name): + LOGGER.warning(f"Column not available: {true_name}") + continue + else: + value = measurements.get_measurement(col_type, col_name) + if isinstance(value, str): + value = [value] + elif isinstance(value, Iterable): + value = list(value) + elif value is None and isinstance(omero_column, omero.grid.DoubleColumn): + # Replace None with NaN + value = [math.nan] + elif value is None and isinstance(omero_column, omero.grid.LongColumn): + # Missing values not supported + value = [-1] + else: + value = [value] + omero_column.values = value + try: + omero_table.addData(table_columns) + except Exception as e: + LOGGER.error("Data upload was unsuccessful", exc_info=True) + raise + LOGGER.info(f"OMERO data uploaded for {table_type}") + + def should_stop_writing_measurements(self): + """All subsequent modules should not write measurements""" + return True + + def ignore_object(self, object_name, strict=False): + """Ignore objects (other than 'Image') if this returns true + + If strict is True, then we ignore objects based on the object selection + """ + if object_name in (EXPERIMENT, NEIGHBORS,): + return True + if strict and self.objects_choice == O_NONE: + return True + if strict and self.objects_choice == O_SELECT and object_name != "Image": + return object_name not in self.objects_list.value + return False + + def ignore_feature( + self, + object_name, + feature_name, + strict=False, + ): + """Return true if we should ignore a feature""" + if ( + self.ignore_object(object_name, strict) + or feature_name.startswith("Number_") + or feature_name.startswith("Description_") + or feature_name.startswith("ModuleError_") + or feature_name.startswith("TimeElapsed_") + or (feature_name.startswith("ExecutionTime_")) + ): + return True + return False + + def get_aggregate_columns(self, pipeline): + """Get object aggregate columns for the PerImage table + + pipeline - the pipeline being run + image_set_list - for cacheing column data + post_group - true if only getting aggregates available post-group, + false for getting aggregates available after run, + None to get all + + returns a tuple: + result[0] - object_name = name of object generating the aggregate + result[1] - feature name + result[2] - aggregation operation + result[3] - column name in Image database + """ + columns = self.get_pipeline_measurement_columns(pipeline) + ob_tables = self.get_object_names(pipeline) + result = [] + for ob_table in ob_tables: + for obname, feature, ftype in columns: + if ( + obname == ob_table + and (not self.ignore_feature(obname, feature)) + and (not agg_ignore_feature(feature)) + ): + feature_name = f"{obname}_{feature}" + # create per_image aggregate column defs + result += [ + (aggname, feature_name, ftype) + for aggname in self.agg_names + ] + return result + + def get_object_names(self, pipeline): + """Get the names of the objects whose measurements are being taken""" + column_defs = self.get_pipeline_measurement_columns(pipeline) + obnames = set([c[0] for c in column_defs]) + # + # In alphabetical order + # + obnames = sorted(obnames) + return [obname for obname in obnames if not self.ignore_object(obname, True) + and obname not in ("Image", EXPERIMENT, NEIGHBORS,)] + + @property + def agg_names(self): + """The list of selected aggregate names""" + return [ + name + for name, setting in ( + (AGG_MEAN, self.wants_agg_mean), + (AGG_MEDIAN, self.wants_agg_median), + (AGG_STD_DEV, self.wants_agg_std_dev), + ) + if setting.value + ] + + def display(self, workspace, figure): + figure.set_subplots((1, 1)) + if workspace.pipeline.test_mode: + figure.subplot_table(0, 0, [["Data not written to database in test mode"]]) + else: + figure.subplot_table( + 0, + 0, + workspace.display_data.columns, + col_labels=workspace.display_data.header, + ) + + def display_post_run(self, workspace, figure): + if not workspace.display_data.columns: + # Nothing to display + return + figure.set_subplots((1, 1)) + figure.subplot_table( + 0, + 0, + workspace.display_data.columns, + col_labels=workspace.display_data.header, + ) + + def get_table_prefix(self): + if self.want_table_prefix.value: + return self.table_prefix.value + return "" + + def get_table_name(self, object_name): + """Return the table name associated with a given object + + object_name - name of object or "Image", "Object" or "Well" + """ + return self.get_table_prefix() + "Per_" + object_name + + def get_pipeline_measurement_columns( + self, pipeline + ): + """Get the measurement columns for this pipeline, possibly cached""" + d = self.get_dictionary() + if D_MEASUREMENT_COLUMNS not in d: + d[D_MEASUREMENT_COLUMNS] = pipeline.get_measurement_columns() + d[D_MEASUREMENT_COLUMNS] = self.filter_measurement_columns( + d[D_MEASUREMENT_COLUMNS] + ) + return d[D_MEASUREMENT_COLUMNS] + + def filter_measurement_columns(self, columns): + """Filter out and properly sort measurement columns""" + # Unlike ExportToDb we also filter out complex columns here, + # since post-group measurements can't easily be added to an OMERO.table + columns = [ + x for x in columns + if not self.ignore_feature(x[0], x[1], strict=True) and len(x) == 3 + ] + + # + # put Image ahead of any other object + # put Number_ObjectNumber ahead of any other column + # + def cmpfn(x, y): + if x[0] != y[0]: + if x[0] == "Image": + return -1 + elif y[0] == "Image": + return 1 + else: + return cellprofiler_core.utilities.legacy.cmp(x[0], y[0]) + if x[1] == M_NUMBER_OBJECT_NUMBER: + return -1 + if y[1] == M_NUMBER_OBJECT_NUMBER: + return 1 + return cellprofiler_core.utilities.legacy.cmp(x[1], y[1]) + + columns = sorted(columns, key=functools.cmp_to_key(cmpfn)) + # + # Remove all but the last duplicate + # + duplicate = [ + c0[0] == c1[0] and c0[1] == c1[1] + for c0, c1 in zip(columns[:-1], columns[1:]) + ] + [False] + columns = [x for x, y in zip(columns, duplicate) if not y] + return columns + + def volumetric(self): + return True diff --git a/active_plugins/omeroreader.py b/active_plugins/omeroreader.py index e737fda..64a830d 100644 --- a/active_plugins/omeroreader.py +++ b/active_plugins/omeroreader.py @@ -23,9 +23,9 @@ From there pip install omero-py should do the rest. -You'll probably also want the `omero_user_token` package to help manage logins (`pip install omero_user_token`). +You'll also want the `omero_user_token` package to help manage logins (`pip install omero_user_token`). This allows you to set reusable login tokens for quick reconnection to a server. These tokens are required for using -headless mode. +headless mode/analysis moe. # Usage - From 0b9b90acd2503373a29a14622ae4d1c7b22ea813 Mon Sep 17 00:00:00 2001 From: David Stirling Date: Tue, 8 Aug 2023 16:38:08 +0100 Subject: [PATCH 14/32] Add SaveImagesToOMERO plugin --- active_plugins/saveimagestoomero.py | 590 ++++++++++++++++++ .../supported_plugins.md | 4 +- 2 files changed, 593 insertions(+), 1 deletion(-) create mode 100644 active_plugins/saveimagestoomero.py diff --git a/active_plugins/saveimagestoomero.py b/active_plugins/saveimagestoomero.py new file mode 100644 index 0000000..da60946 --- /dev/null +++ b/active_plugins/saveimagestoomero.py @@ -0,0 +1,590 @@ +""" +SaveImagesToOMERO +================== + +**SaveImagesToOMERO** saves image or movies directly onto an +OMERO server. + +# Installation - +Easy mode - clone the plugins repository and point your CellProfiler plugins folder to this folder. +Navigate to /active_plugins/ and run `pip install -e .[omero]` to install dependencies. + +## Manual Installation + +Add this file plus the `omero_helper` directory into your CellProfiler plugins folder. Install dependencies into +your CellProfiler Python environment. + +## Installing dependencies - +This depends on platform. At the most basic level you'll need the `omero-py` package and the `omero_user_token` package. + +Both should be possible to pip install on Windows. On MacOS, you'll probably have trouble with the zeroc-ice dependency. +omero-py uses an older version and so needs specific wheels. Fortunately we've built some for you. +Macos - https://github.com/glencoesoftware/zeroc-ice-py-macos-x86_64/releases/latest +Linux (Generic) - https://github.com/glencoesoftware/zeroc-ice-py-linux-x86_64/releases/latest +Ubuntu 22.04 - https://github.com/glencoesoftware/zeroc-ice-py-ubuntu2204-x86_64/releases/latest + +Download the .whl file from whichever is most appropriate and run `pip install `. + +From there pip install omero-py should do the rest. + +You'll also want the `omero_user_token` package to help manage logins (`pip install omero_user_token`). +This allows you to set reusable login tokens for quick reconnection to a server. These tokens are required for using +headless mode/analysis mode. + +| + +============ ============ =============== +Supports 2D? Supports 3D? Respects masks? +============ ============ =============== +YES YES YES +============ ============ =============== + +""" + +import logging +import os + +import numpy +import skimage +from cellprofiler_core.module import Module +from cellprofiler_core.preferences import get_headless +from cellprofiler_core.setting import Binary +from cellprofiler_core.setting import ValidationError +from cellprofiler_core.setting.choice import Choice +from cellprofiler_core.setting.do_something import DoSomething +from cellprofiler_core.setting.subscriber import ImageSubscriber, FileImageSubscriber +from cellprofiler_core.setting.text import Integer, Text +from cellprofiler_core.constants.setting import get_name_providers + +from cellprofiler.modules import _help + +from omero_helper.connect import CREDENTIALS, login + +LOGGER = logging.getLogger(__name__) + +FN_FROM_IMAGE = "From image filename" +FN_SEQUENTIAL = "Sequential numbers" +FN_SINGLE_NAME = "Single name" + +SINGLE_NAME_TEXT = "Enter single file name" +SEQUENTIAL_NUMBER_TEXT = "Enter file prefix" + +BIT_DEPTH_8 = "8-bit integer" +BIT_DEPTH_16 = "16-bit integer" +BIT_DEPTH_FLOAT = "32-bit floating point" +BIT_DEPTH_RAW = "No conversion" + +WS_EVERY_CYCLE = "Every cycle" +WS_FIRST_CYCLE = "First cycle" +WS_LAST_CYCLE = "Last cycle" + + +class SaveImagesToOMERO(Module): + module_name = "SaveImagesToOMERO" + variable_revision_number = 1 + category = ["File Processing"] + + def create_settings(self): + self.target_object_id = Integer( + text="OMERO ID of the parent dataset", + minval=1, + doc="""\ + The created images must be added to an OMERO dataset. + Enter the OMERO ID of the Dataset object you'd like to associate + the the image with. This ID can be found by locating the target object + in OMERO.web (ID and type is displayed in the right panel). + + To use a new dataset, first create this in OMERO.web and then enter the ID here""", + ) + + self.test_connection_button = DoSomething( + "Test the OMERO connection", + "Test connection", + self.test_connection, + doc="""\ +This button test the connection to the OMERO server specified using +the settings entered by the user.""", + ) + + self.image_name = ImageSubscriber( + "Select the image to save", doc="Select the image you want to save." + ) + + self.bit_depth = Choice( + "Image bit depth conversion", + [BIT_DEPTH_8, BIT_DEPTH_16, BIT_DEPTH_FLOAT, BIT_DEPTH_RAW], + BIT_DEPTH_RAW, + doc=f"""\ + Select the bit-depth at which you want to save the images. CellProfiler + typically works with images scaled into the 0-1 range. This setting lets + you transform that into other scales. + + Selecting *{BIT_DEPTH_RAW}* will attempt to upload data without applying + any transformations. This could be used to save integer labels + in 32-bit float format if you had more labels than the 16-bit format can + handle (without rescaling to the 0-1 range of *{BIT_DEPTH_FLOAT}*). + N.B. data compatibility with OMERO is not checked. + + *{BIT_DEPTH_8}* and *{BIT_DEPTH_16}* will attempt to rescale values to + be in the range 0-255 and 0-65535 respectively. These are typically + used with external tools. + + *{BIT_DEPTH_FLOAT}* saves the image as floating-point decimals with + 32-bit precision. When the input data is integer or binary type, pixel + values are scaled within the range (0, 1). Floating point data is not + rescaled.""", + ) + + self.file_name_method = Choice( + "Select method for constructing file names", + [FN_FROM_IMAGE, FN_SEQUENTIAL, FN_SINGLE_NAME], + FN_FROM_IMAGE, + doc="""\ + *(Used only if saving non-movie files)* + + Several choices are available for constructing the image file name: + + - *{FN_FROM_IMAGE}:* The filename will be constructed based on the + original filename of an input image specified in **NamesAndTypes**. + You will have the opportunity to prefix or append additional text. + + If you have metadata associated with your images, you can append + text to the image filename using a metadata tag. This is especially + useful if you want your output given a unique label according to the + metadata corresponding to an image group. The name of the metadata to + substitute can be provided for each image for each cycle using the + **Metadata** module. + - *{FN_SEQUENTIAL}:* Same as above, but in addition, each filename + will have a number appended to the end that corresponds to the image + cycle number (starting at 1). + - *{FN_SINGLE_NAME}:* A single name will be given to the file. Since + the filename is fixed, this file will be overwritten with each cycle. + In this case, you would probably want to save the image on the last + cycle (see the *Select how often to save* setting). The exception to + this is to use a metadata tag to provide a unique label, as mentioned + in the *{FN_FROM_IMAGE}* option. + + {USING_METADATA_TAGS_REF} + + {USING_METADATA_HELP_REF} + """.format( + **{ + "FN_FROM_IMAGE": FN_FROM_IMAGE, + "FN_SEQUENTIAL": FN_SEQUENTIAL, + "FN_SINGLE_NAME": FN_SINGLE_NAME, + "USING_METADATA_HELP_REF": _help.USING_METADATA_HELP_REF, + "USING_METADATA_TAGS_REF": _help.USING_METADATA_TAGS_REF, + } + ), + ) + + self.file_image_name = FileImageSubscriber( + "Select image name for file prefix", + "None", + doc="""\ + *(Used only when “{FN_FROM_IMAGE}” is selected for constructing the filename)* + + Select an image loaded using **NamesAndTypes**. The original filename + will be used as the prefix for the output filename.""".format( + **{"FN_FROM_IMAGE": FN_FROM_IMAGE} + ), + ) + + self.single_file_name = Text( + SINGLE_NAME_TEXT, + "OrigBlue", + metadata=True, + doc="""\ + *(Used only when “{FN_SEQUENTIAL}” or “{FN_SINGLE_NAME}” are selected + for constructing the filename)* + + Specify the filename text here. If you have metadata associated with + your images, enter the filename text with the metadata tags. + {USING_METADATA_TAGS_REF} + Do not enter the file extension in this setting; it will be appended + automatically.""".format( + **{ + "FN_SEQUENTIAL": FN_SEQUENTIAL, + "FN_SINGLE_NAME": FN_SINGLE_NAME, + "USING_METADATA_TAGS_REF": _help.USING_METADATA_TAGS_REF, + } + ), + ) + + self.number_of_digits = Integer( + "Number of digits", + 4, + doc="""\ + *(Used only when “{FN_SEQUENTIAL}” is selected for constructing the filename)* + + Specify the number of digits to be used for the sequential numbering. + Zeros will be used to left-pad the digits. If the number specified here + is less than that needed to contain the number of image sets, the latter + will override the value entered.""".format( + **{"FN_SEQUENTIAL": FN_SEQUENTIAL} + ), + ) + + self.wants_file_name_suffix = Binary( + "Append a suffix to the image file name?", + False, + doc="""\ + Select "*{YES}*" to add a suffix to the image’s file name. Select "*{NO}*" + to use the image name as-is. + """.format( + **{"NO": "No", "YES": "Yes"} + ), + ) + + self.file_name_suffix = Text( + "Text to append to the image name", + "", + metadata=True, + doc="""\ + *(Used only when constructing the filename from the image filename)* + + Enter the text that should be appended to the filename specified above. + If you have metadata associated with your images, you may use metadata tags. + + {USING_METADATA_TAGS_REF} + + Do not enter the file extension in this setting; it will be appended + automatically. + """.format( + **{"USING_METADATA_TAGS_REF": _help.USING_METADATA_TAGS_REF} + ), + ) + + self.when_to_save = Choice( + "When to save", + [WS_EVERY_CYCLE, WS_FIRST_CYCLE, WS_LAST_CYCLE], + WS_EVERY_CYCLE, + doc="""\ + Specify at what point during pipeline execution to save file(s). + + - *{WS_EVERY_CYCLE}:* Useful for when the image of interest is + created every cycle and is not dependent on results from a prior + cycle. + - *{WS_FIRST_CYCLE}:* Useful for when you are saving an aggregate + image created on the first cycle, e.g., + **CorrectIlluminationCalculate** with the *All* setting used on + images obtained directly from **NamesAndTypes**. + - *{WS_LAST_CYCLE}:* Useful for when you are saving an aggregate image + completed on the last cycle, e.g., **CorrectIlluminationCalculate** + with the *All* setting used on intermediate images generated during + each cycle.""".format( + **{ + "WS_EVERY_CYCLE": WS_EVERY_CYCLE, + "WS_FIRST_CYCLE": WS_FIRST_CYCLE, + "WS_LAST_CYCLE": WS_LAST_CYCLE, + } + ), + ) + + def settings(self): + result = [ + self.target_object_id, + self.image_name, + self.bit_depth, + self.file_name_method, + self.file_image_name, + self.single_file_name, + self.number_of_digits, + self.wants_file_name_suffix, + self.file_name_suffix, + self.when_to_save, + ] + return result + + def visible_settings(self): + result = [self.target_object_id, self.test_connection_button, + self.image_name, self.bit_depth, self.file_name_method] + + if self.file_name_method == FN_FROM_IMAGE: + result += [self.file_image_name, self.wants_file_name_suffix] + if self.wants_file_name_suffix: + result.append(self.file_name_suffix) + elif self.file_name_method == FN_SEQUENTIAL: + self.single_file_name.text = SEQUENTIAL_NUMBER_TEXT + result.append(self.single_file_name) + result.append(self.number_of_digits) + elif self.file_name_method == FN_SINGLE_NAME: + self.single_file_name.text = SINGLE_NAME_TEXT + result.append(self.single_file_name) + else: + raise NotImplementedError( + "Unhandled file name method: %s" % self.file_name_method + ) + result.append(self.when_to_save) + return result + + def help_settings(self): + return [ + self.target_object_id, + self.image_name, + self.bit_depth, + self.file_name_method, + self.file_image_name, + self.single_file_name, + self.number_of_digits, + self.wants_file_name_suffix, + self.file_name_suffix, + self.when_to_save, + ] + + def validate_module(self, pipeline): + # Make sure metadata tags exist + if self.file_name_method == FN_SINGLE_NAME or ( + self.file_name_method == FN_FROM_IMAGE and self.wants_file_name_suffix.value + ): + text_str = ( + self.single_file_name.value + if self.file_name_method == FN_SINGLE_NAME + else self.file_name_suffix.value + ) + undefined_tags = pipeline.get_undefined_metadata_tags(text_str) + if len(undefined_tags) > 0: + raise ValidationError( + "%s is not a defined metadata tag. Check the metadata specifications in your load modules" + % undefined_tags[0], + self.single_file_name + if self.file_name_method == FN_SINGLE_NAME + else self.file_name_suffix, + ) + if self.when_to_save in (WS_FIRST_CYCLE, WS_EVERY_CYCLE): + # + # Make sure that the image name is available on every cycle + # + for setting in get_name_providers(pipeline, self.image_name): + if setting.provided_attributes.get("available_on_last"): + # + # If we fell through, then you can only save on the last cycle + # + raise ValidationError( + "%s is only available after processing all images in an image group" + % self.image_name.value, + self.when_to_save, + ) + + def test_connection(self): + """Check to make sure the OMERO server is remotely accessible""" + # CREDENTIALS is a singleton so we can safely grab it here. + if CREDENTIALS.client is None: + login() + if CREDENTIALS.client is None: + msg = "OMERO connection failed" + else: + msg = f"Connected to {CREDENTIALS.server}" + else: + msg = f"Already connected to {CREDENTIALS.server}" + if CREDENTIALS.client is not None: + try: + self.get_omero_parent() + msg += f"\n\nFound parent object {self.target_object_id}" + except ValueError as ve: + msg += f"\n\n{ve}" + + import wx + wx.MessageBox(msg) + + def make_full_filename(self, file_name, workspace=None, image_set_index=None): + """Convert a file name into an absolute path + + We do a few things here: + * apply metadata from an image set to the file name if an + image set is specified + * change the relative path into an absolute one using the "." and "&" + convention + * Create any directories along the path + """ + if image_set_index is not None and workspace is not None: + file_name = workspace.measurements.apply_metadata( + file_name, image_set_index + ) + measurements = None if workspace is None else workspace.measurements + path_name = self.directory.get_absolute_path(measurements, image_set_index) + file_name = os.path.join(path_name, file_name) + path, file = os.path.split(file_name) + if not os.path.isdir(path): + os.makedirs(path) + return os.path.join(path, file) + + @staticmethod + def connect_to_omero(): + if CREDENTIALS.client is None: + if get_headless(): + connected = login() + if not connected: + raise ValueError("No OMERO connection established") + else: + login() + if CREDENTIALS.client is None: + raise ValueError("OMERO connection failed") + + def prepare_run(self, workspace): + """Prepare to run the pipeline. + Establish a connection to OMERO.""" + pipeline = workspace.pipeline + + if pipeline.in_batch_mode(): + return True + + # Verify that we're able to connect to a server + self.connect_to_omero() + + return True + + def get_omero_conn(self): + self.connect_to_omero() + return CREDENTIALS.get_gateway() + + def get_omero_parent(self): + conn = self.get_omero_conn() + parent_id = self.target_object_id.value + parent_type = "Dataset" + old_group = conn.SERVICE_OPTS.getOmeroGroup() + # Search across groups + conn.SERVICE_OPTS.setOmeroGroup(-1) + parent_ob = conn.getObject(parent_type, parent_id) + if parent_ob is None: + raise ValueError(f"{parent_type} ID {parent_id} not found on server") + conn.SERVICE_OPTS.setOmeroGroup(old_group) + return parent_ob + + def run(self, workspace): + if self.show_window: + workspace.display_data.wrote_image = False + + if self.when_to_save == WS_FIRST_CYCLE and workspace.measurements["Image", "Group_Index", ] > 1: + # We're past the first image set + return + elif self.when_to_save == WS_LAST_CYCLE: + # We do this in post group + return + + self.save_image(workspace) + + def save_image(self, workspace): + # Re-establish server connection + self.connect_to_omero() + + filename = self.get_filename(workspace) + + image = workspace.image_set.get_image(self.image_name.value) + + omero_image = self.upload_image_to_omero(image, filename) + + if self.show_window: + workspace.display_data.wrote_image = True + im_id = omero_image.getId() + path = f"https://{CREDENTIALS.server}/webclient/?show=image-{im_id}" + workspace.display_data.header = ["Image Name", "OMERO ID", "Server Location"] + workspace.display_data.columns = [[filename, im_id, path]] + + def post_group(self, workspace, *args): + if self.when_to_save == WS_LAST_CYCLE: + self.save_image(workspace) + + def upload_image_to_omero(self, image, name): + pixels = image.pixel_data.copy() + volumetric = image.volumetric + multichannel = image.multichannel + + if self.bit_depth.value == BIT_DEPTH_8: + pixels = skimage.util.img_as_ubyte(pixels) + elif self.bit_depth.value == BIT_DEPTH_16: + pixels = skimage.util.img_as_uint(pixels) + elif self.bit_depth.value == BIT_DEPTH_FLOAT: + pixels = skimage.util.img_as_float32(pixels) + elif self.bit_depth.value == BIT_DEPTH_RAW: + # No bit depth transformation + pass + else: + raise NotImplementedError(f"Unknown bit depth {self.bit_depth.value}") + + conn = self.get_omero_conn() + parent = self.get_omero_parent() + parent_group = parent.details.group.id.val + old_group = conn.SERVICE_OPTS.getOmeroGroup() + conn.SERVICE_OPTS.setOmeroGroup(parent_group) + shape = pixels.shape + if multichannel: + size_c = shape[-1] + else: + size_c = 1 + if volumetric: + size_z = shape[2] + else: + size_z = 1 + + new_shape = list(shape) + while len(new_shape) < 4: + new_shape.append(1) + upload_pixels = numpy.reshape(pixels, new_shape) + + def slice_iterator(): + for z in range(size_z): + for c in range(size_c): + yield upload_pixels[:, :, z, c] + + # Upload the image data to OMERO + LOGGER.debug("Transmitting data for image") + omero_image = conn.createImageFromNumpySeq( + slice_iterator(), name, size_z, size_c, 1, description="Image uploaded from CellProfiler", + dataset=parent) + LOGGER.debug("Transmission successful") + conn.SERVICE_OPTS.setOmeroGroup(old_group) + return omero_image + + def get_filename(self, workspace): + """Concoct a filename for the current image based on the user settings""" + measurements = workspace.measurements + if self.file_name_method == FN_SINGLE_NAME: + filename = self.single_file_name.value + filename = workspace.measurements.apply_metadata(filename) + elif self.file_name_method == FN_SEQUENTIAL: + filename = self.single_file_name.value + filename = workspace.measurements.apply_metadata(filename) + n_image_sets = workspace.measurements.image_set_count + ndigits = int(numpy.ceil(numpy.log10(n_image_sets + 1))) + ndigits = max((ndigits, self.number_of_digits.value)) + padded_num_string = str(measurements.image_set_number).zfill(ndigits) + filename = "%s%s" % (filename, padded_num_string) + else: + file_name_feature = self.source_file_name_feature + filename = measurements.get_current_measurement("Image", file_name_feature) + filename = os.path.splitext(filename)[0] + if self.wants_file_name_suffix: + suffix = self.file_name_suffix.value + suffix = workspace.measurements.apply_metadata(suffix) + filename += suffix + return filename + + @property + def source_file_name_feature(self): + """The file name measurement for the exemplar disk image""" + return "_".join(("FileName", self.file_image_name.value)) + + def display(self, workspace, figure): + if not workspace.display_data.columns: + # Nothing to display + return + figure.set_subplots((1, 1)) + figure.subplot_table( + 0, + 0, + workspace.display_data.columns, + col_labels=workspace.display_data.header, + ) + + def display_post_run(self, workspace, figure): + self.display(workspace, figure) + + def is_aggregation_module(self): + """SaveImagesToOMERO is an aggregation module when it writes on the last cycle""" + return ( + self.when_to_save == WS_LAST_CYCLE + ) + + def volumetric(self): + return True diff --git a/documentation/CP-plugins-documentation/supported_plugins.md b/documentation/CP-plugins-documentation/supported_plugins.md index 503b26d..cf12417 100644 --- a/documentation/CP-plugins-documentation/supported_plugins.md +++ b/documentation/CP-plugins-documentation/supported_plugins.md @@ -25,4 +25,6 @@ Those plugins that do have extra documentation contain links below. | RunOmnipose | RunOmnipose allows you to run Omnipose within CellProfiler. Omnipose is a general image segmentation tool that builds on Cellpose. | Yes | `omnipose` | | RunStarDist | RunStarDist allows you to run StarDist within CellProfiler. StarDist is a machine-learning algorithm for object detection with star-convex shapes making it best suited for nuclei or round-ish cells. You can use pre-trained StarDist models or your custom model with this plugin. You can use a GPU with this module to dramatically increase your speed/efficiency. RunStarDist is generally faster than RunCellpose. | Yes | `stardist` | | VarianceTransform | This module allows you to calculate the variance of an image, using a determined window size. It also has the option to find the optimal window size from a predetermined range to obtain the maximum variance of an image. | No | | -| OMEROReader | This reader allows you to connect to and load images from OMERO servers. | Yes | 'omero' | \ No newline at end of file +| OMEROReader | This reader allows you to connect to and load images from OMERO servers. | Yes | 'omero' | +| SaveImagesToOMERO | A module to upload resulting images directly onto OMERO servers. | Yes | 'omero' | +| ExportToOMEROTable | A module to upload results tables directly onto OMERO servers. | Yes | 'omero' | From 7a75654ccdd8ae0a4fa282d5bc6dc12ac05598f8 Mon Sep 17 00:00:00 2001 From: David Stirling Date: Tue, 8 Aug 2023 16:39:37 +0100 Subject: [PATCH 15/32] Formatting tweak --- documentation/CP-plugins-documentation/supported_plugins.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/documentation/CP-plugins-documentation/supported_plugins.md b/documentation/CP-plugins-documentation/supported_plugins.md index cf12417..3cdb4d3 100644 --- a/documentation/CP-plugins-documentation/supported_plugins.md +++ b/documentation/CP-plugins-documentation/supported_plugins.md @@ -25,6 +25,6 @@ Those plugins that do have extra documentation contain links below. | RunOmnipose | RunOmnipose allows you to run Omnipose within CellProfiler. Omnipose is a general image segmentation tool that builds on Cellpose. | Yes | `omnipose` | | RunStarDist | RunStarDist allows you to run StarDist within CellProfiler. StarDist is a machine-learning algorithm for object detection with star-convex shapes making it best suited for nuclei or round-ish cells. You can use pre-trained StarDist models or your custom model with this plugin. You can use a GPU with this module to dramatically increase your speed/efficiency. RunStarDist is generally faster than RunCellpose. | Yes | `stardist` | | VarianceTransform | This module allows you to calculate the variance of an image, using a determined window size. It also has the option to find the optimal window size from a predetermined range to obtain the maximum variance of an image. | No | | -| OMEROReader | This reader allows you to connect to and load images from OMERO servers. | Yes | 'omero' | -| SaveImagesToOMERO | A module to upload resulting images directly onto OMERO servers. | Yes | 'omero' | -| ExportToOMEROTable | A module to upload results tables directly onto OMERO servers. | Yes | 'omero' | +| OMEROReader | This reader allows you to connect to and load images from OMERO servers. | Yes | `omero` | +| SaveImagesToOMERO | A module to upload resulting images directly onto OMERO servers. | Yes | `omero` | +| ExportToOMEROTable | A module to upload results tables directly onto OMERO servers. | Yes | `omero` | From 4c1dec9c4a7024348b5e5c60d2cfd0678c8f6e5d Mon Sep 17 00:00:00 2001 From: David Stirling Date: Wed, 9 Aug 2023 12:07:25 +0100 Subject: [PATCH 16/32] Add relationships table support --- active_plugins/exporttoomerotable.py | 51 +++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/active_plugins/exporttoomerotable.py b/active_plugins/exporttoomerotable.py index d45e7a3..e69f290 100644 --- a/active_plugins/exporttoomerotable.py +++ b/active_plugins/exporttoomerotable.py @@ -461,7 +461,7 @@ def prepare_run(self, workspace): # Build a list of tables to create column_defs = shared_state[D_MEASUREMENT_COLUMNS] - desired_tables = ["Image"] + desired_tables = ["Image", "Relationships"] if self.objects_choice == O_SELECT: desired_tables += self.objects_list.value elif self.objects_choice == O_ALL: @@ -487,6 +487,23 @@ def prepare_run(self, workspace): if table_name == "Image": # Add any aggregate measurements table_cols.extend(self.get_aggregate_columns(workspace.pipeline)) + elif table_name == "Relationships": + # Hacky, but relationships are totally different from standard measurements + relationships = pipeline.get_object_relationships() + if not relationships: + # No need for table + continue + table_cols = [ + ("", "Module", COLTYPE_VARCHAR), + ("", "Module Number", COLTYPE_INTEGER), + ("", "Relationship", COLTYPE_VARCHAR), + ("", "First Object Name", COLTYPE_VARCHAR), + ("", "First Image Number", COLTYPE_INTEGER), + ("", "First Object Number", COLTYPE_INTEGER), + ("", "Second Object Name", COLTYPE_VARCHAR), + ("", "Second Image Number", COLTYPE_INTEGER), + ("", "Second Object Number", COLTYPE_INTEGER), + ] omero_id = self.create_omero_table(parent, true_name, table_cols) omero_table_list.append((table_name, true_name, omero_id, table_cols)) table_path = f"https://{CREDENTIALS.server}/webclient/omero_table/{omero_id}" @@ -629,6 +646,38 @@ def write_data_to_omero(self, workspace, table_type, omero_table, column_list): extra_data.update(measurements.compute_aggregate_measurements( measurements.image_set_number, self.agg_names )) + elif table_type == "Relationships": + # We build the Relationships table in the extra data buffer + modules = workspace.pipeline.modules() + # Initialise table variables as empty + extra_data["First Image Number"] = [] + extra_data["Second Image Number"] = [] + extra_data["First Object Number"] = [] + extra_data["Second Object Number"] = [] + extra_data["Module"] = [] + extra_data["Module Number"] = [] + extra_data["Relationship"] = [] + extra_data["First Object Name"] = [] + extra_data["Second Object Name"] = [] + for key in measurements.get_relationship_groups(): + # Add records for each relationship + records = measurements.get_relationships( + key.module_number, + key.relationship, + key.object_name1, + key.object_name2, + ) + module_name = modules[key.module_number].module_name + extra_data["First Image Number"] += list(records["ImageNumber_First"]) + extra_data["Second Image Number"] += list(records["ImageNumber_Second"]) + extra_data["First Object Number"] += list(records["ObjectNumber_First"]) + extra_data["Second Object Number"] += list(records["ObjectNumber_Second"]) + num_records = len(records["ImageNumber_First"]) + extra_data["Module"] += [module_name] * num_records + extra_data["Module Number"] += [key.module_number] * num_records + extra_data["Relationship"] += [key.relationship] * num_records + extra_data["First Object Name"] += [key.object_name1] * num_records + extra_data["Second Object Name"] += [key.object_name2] * num_records else: extra_data[C_OBJECT_NUMBER] = measurements.get_measurement(table_type, M_NUMBER_OBJECT_NUMBER) extra_data[C_IMAGE_NUMBER] = [extra_data[C_IMAGE_NUMBER]] * len(extra_data[C_OBJECT_NUMBER]) From f634a2d2e37921bba7db91ffda6c3e2b46e93d64 Mon Sep 17 00:00:00 2001 From: David Stirling Date: Wed, 6 Sep 2023 14:57:43 +0100 Subject: [PATCH 17/32] Safer home dir manipulation --- active_plugins/omero_helper/connect.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/active_plugins/omero_helper/connect.py b/active_plugins/omero_helper/connect.py index e9c647d..46667ed 100644 --- a/active_plugins/omero_helper/connect.py +++ b/active_plugins/omero_helper/connect.py @@ -1,6 +1,7 @@ import atexit import os import logging +import sys import tempfile from cellprofiler_core.preferences import config_read_typed, get_headless, get_temporary_directory @@ -106,9 +107,10 @@ def get_tokens(self, path=None): if tokens_enabled is not None and not tokens_enabled: return # User tokens sadly default to the home directory. This would override that location. - py_home = os.environ['HOME'] + home_key = "USERPROFILE" if sys.platform == "win32" else "HOME" + py_home = os.environ.get(home_key, "") if path is not None: - os.environ['HOME'] = path + os.environ[home_key] = path try: LOGGER.info("Requesting token info") token = omero_user_token.get_token() @@ -120,7 +122,7 @@ def get_tokens(self, path=None): except Exception: LOGGER.error("Failed to get user token", exc_info=True) if path is not None: - os.environ['HOME'] = py_home + os.environ[home_key] = py_home def try_token(self, address): # Attempt to use an omero token to connect to a specific server From 021f1835d09d3e9437af522b895212f7f5a11338 Mon Sep 17 00:00:00 2001 From: David Stirling Date: Wed, 6 Sep 2023 14:58:02 +0100 Subject: [PATCH 18/32] Handle blanked OMERO server config key --- active_plugins/omero_helper/gui.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/active_plugins/omero_helper/gui.py b/active_plugins/omero_helper/gui.py index 6a0852e..a4b688e 100644 --- a/active_plugins/omero_helper/gui.py +++ b/active_plugins/omero_helper/gui.py @@ -84,7 +84,7 @@ def __init__(self, *args, token=True, server=None, **kwargs): self.token = token self.SetSizer(wx.BoxSizer(wx.VERTICAL)) if server is None: - server = self.credentials.server + server = self.credentials.server or "" sizer = wx.BoxSizer(wx.VERTICAL) self.Sizer.Add(sizer, 1, wx.EXPAND | wx.ALL, 6) sub_sizer = wx.BoxSizer(wx.HORIZONTAL) From c02c49881ed4aa2a3203daa3a5cdf90dee3dd5a9 Mon Sep 17 00:00:00 2001 From: David Stirling Date: Wed, 6 Sep 2023 15:08:37 +0100 Subject: [PATCH 19/32] Apply to username too --- active_plugins/omero_helper/gui.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/active_plugins/omero_helper/gui.py b/active_plugins/omero_helper/gui.py index a4b688e..ecc0d5a 100644 --- a/active_plugins/omero_helper/gui.py +++ b/active_plugins/omero_helper/gui.py @@ -120,7 +120,7 @@ def __init__(self, *args, token=True, server=None, **kwargs): sub_sizer.Add( wx.StaticText(self, label="User:", size=lsize), 0, wx.ALIGN_CENTER_VERTICAL) - self.omero_user_ctrl = wx.TextCtrl(self, value=self.credentials.username) + self.omero_user_ctrl = wx.TextCtrl(self, value=self.credentials.username or "") sub_sizer.Add(self.omero_user_ctrl, 1, wx.EXPAND) sizer.AddSpacer(5) From 0118748af5b8d9b44936ea6277c9a94955fc2439 Mon Sep 17 00:00:00 2001 From: David Stirling Date: Wed, 6 Sep 2023 15:14:51 +0100 Subject: [PATCH 20/32] Username detection fix --- active_plugins/omero_helper/connect.py | 2 +- active_plugins/omero_helper/gui.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/active_plugins/omero_helper/connect.py b/active_plugins/omero_helper/connect.py index 46667ed..61212b8 100644 --- a/active_plugins/omero_helper/connect.py +++ b/active_plugins/omero_helper/connect.py @@ -178,7 +178,7 @@ def login(self, server=None, port=None, user=None, passwd=None, session_key=None self.client = None self.session = None return False - elif self.username is not None: + elif user is not None: try: self.session = self.client.createSession( username=user, password=passwd) diff --git a/active_plugins/omero_helper/gui.py b/active_plugins/omero_helper/gui.py index ecc0d5a..c9fff01 100644 --- a/active_plugins/omero_helper/gui.py +++ b/active_plugins/omero_helper/gui.py @@ -44,7 +44,7 @@ def show_login_dlg(e=None, server=None): # Show the login GUI app = wx.GetApp() frame = app.GetTopWindow() - with OmeroLoginDlg(frame, title="Log into Omero", server=server) as dlg: + with OmeroLoginDlg(frame, title="Login to OMERO", server=server) as dlg: dlg.ShowModal() if CREDENTIALS.client is not None: CREDENTIALS.create_temp_token() From 424841b65971f1ec8f8cd88cb54659d135ff1262 Mon Sep 17 00:00:00 2001 From: David Stirling Date: Wed, 6 Sep 2023 15:46:54 +0100 Subject: [PATCH 21/32] Explicitly set browse dlg choices in browse dialog --- active_plugins/omero_helper/gui.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/active_plugins/omero_helper/gui.py b/active_plugins/omero_helper/gui.py index c9fff01..82c9efb 100644 --- a/active_plugins/omero_helper/gui.py +++ b/active_plugins/omero_helper/gui.py @@ -294,9 +294,11 @@ def __init__(self, *args, **kwargs): b = wx.BoxSizer(wx.VERTICAL) self.groups_box = wx.Choice(self.browse_controls, choices=self.group_names) + self.groups_box.SetSelection(0) self.groups_box.Bind(wx.EVT_CHOICE, self.switch_group) self.members_box = wx.Choice(self.browse_controls, choices=list(self.users_in_group.keys())) + self.members_box.SetSelection(0) self.members_box.Bind(wx.EVT_CHOICE, self.switch_member) b.Add(self.groups_box, 0, wx.EXPAND) @@ -440,6 +442,7 @@ def refresh_group_members(self): }) self.members_box.Clear() self.members_box.AppendItems(list(self.users_in_group.keys())) + self.members_box.SetSelection(0) def fetch_children(self, event): # Load the next level in the tree for a target object. From 2c65a840537998b06898a35393b3f70f93f1e053 Mon Sep 17 00:00:00 2001 From: David Stirling Date: Wed, 20 Sep 2023 14:25:16 +0100 Subject: [PATCH 22/32] Fix shutdown bug --- active_plugins/omero_helper/connect.py | 11 ++++++++++- active_plugins/omero_helper/gui.py | 9 ++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/active_plugins/omero_helper/connect.py b/active_plugins/omero_helper/connect.py index 61212b8..13c734c 100644 --- a/active_plugins/omero_helper/connect.py +++ b/active_plugins/omero_helper/connect.py @@ -34,7 +34,10 @@ def login(e=None, server=None, token_path=None): LOGGER.warning("Failed to connect, was user token invalid?") return connected else: - from .gui import login_gui + from .gui import login_gui, configure_for_safe_shutdown + if not CREDENTIALS.bound: + configure_for_safe_shutdown() + CREDENTIALS.bound = True login_gui(connected, server=None) @@ -86,6 +89,8 @@ def __init__(self): self.gateway = None self.container_service = None self.tokens = {} + # Whether we've already hooked into the main GUI frame + self.bound = False # Any OMERO browser GUI which is connected to this object self.browser_window = None atexit.register(self.shutdown) @@ -202,6 +207,10 @@ def login(self, server=None, port=None, user=None, passwd=None, session_key=None self.container_service = self.session.getContainerService() return True + def handle_exit(self, e): + self.shutdown() + e.Skip() + def shutdown(self): # Disconnect from the server if self.client is not None: diff --git a/active_plugins/omero_helper/gui.py b/active_plugins/omero_helper/gui.py index 82c9efb..a3e7785 100644 --- a/active_plugins/omero_helper/gui.py +++ b/active_plugins/omero_helper/gui.py @@ -74,6 +74,13 @@ def inject_plugin_menu_entries(): ]) +def configure_for_safe_shutdown(): + # When GUI is running we need to capture wx exit events and close the OMERO connection + app = wx.GetApp() + frame = app.GetTopWindow() + frame.Bind(wx.EVT_CLOSE, CREDENTIALS.handle_exit) + + class OmeroLoginDlg(wx.Dialog): """ A dialog pane to provide and use OMERO login credentials. @@ -257,7 +264,7 @@ def on_ok(self, event): self.EndModal(wx.OK) -class OmeroBrowseDlg(wx.Frame): +class OmeroBrowseDlg(wx.Dialog): """ An OMERO server browser intended for browsing images and adding them to the main file list """ From a1d73cf4463d097b6f86385ffb1548f7970c038f Mon Sep 17 00:00:00 2001 From: David Stirling Date: Wed, 20 Sep 2023 14:26:00 +0100 Subject: [PATCH 23/32] Add expanded docs --- .../CP-plugins-documentation/OMERO.md | 100 ++++++++++++++++++ .../supported_plugins.md | 38 +++---- 2 files changed, 119 insertions(+), 19 deletions(-) create mode 100644 documentation/CP-plugins-documentation/OMERO.md diff --git a/documentation/CP-plugins-documentation/OMERO.md b/documentation/CP-plugins-documentation/OMERO.md new file mode 100644 index 0000000..6b80d6f --- /dev/null +++ b/documentation/CP-plugins-documentation/OMERO.md @@ -0,0 +1,100 @@ +# OMERO Plugins + +[OMERO](https://www.openmicroscopy.org/omero/) is an image data management server developed by the Open Microscopy Environment. +It allows for the storage and retrieval of image datasets and associated metadata. + + +## Using OMERO with CellProfiler + +The OMERO plugins can be used to connect to an OMERO server and exchange data with CellProfiler. You'll need to supply the server address and login credentials to establish a connection. + +The current iteration of the plugins supports a single active connection at any given time. This means that pipelines should only be requesting data from a single OMERO server. + +Supplying credentials every run can be repetitive, so the plugins will also detect and make use of tokens set by the [omero-user-token](https://github.com/glencoesoftware/omero-user-token) package. This provides a long-lasting mechanism for +reconnecting to a previously established OMERO session. N.b. tokens which are set to never expire will only be valid until the OMERO server restarts. + +## The connection interface + +When installed correctly, OMERO-related entries will be available in the CellProfiler '**Plugins**' menu (this menu only appears when an installed plugin uses it). You'll find _Connect to OMERO_ menu entries allowing you +to connect to an OMERO server. If a user token already exists this will be used to establish a connection, otherwise you'll see a dialog where credentials can be supplied. You can also use the +_Connect to OMERO (no token)_ option to skip checking for a login token, this is mostly useful if you need to switch server. + +After entering credentials and establishing a connection, a _Set Token_ button is available to create an omero-user-token on your system. This will allow +you to reconnect to the same server automatically without entering credentials. Tokens are set per-user, and only a single token can be stored at a time. This means that you +**should not** set tokens if using a shared user account. + +An active OMERO connection is required for most functionality in these plugins. A single connection can be made at a time, and the +plugin will automatically sustain, manage and safely shut down this connection as needed. The OMERO plugin will disconnect automatically when +quitting CellProfiler. The connection dialog should automatically display if you try to run the plugin without a connection. + +When reconnecting via a token, the plugin will display a message clarifying which server was connected to. This message can be disabled by ticking the _do not show again_ box. This +can be re-enabled using the reader configuration interface in the _File->Configure Readers_ menu. Under the OMEROReader reader config +you'll also find an option to entirely disable token usage, in case you need to use different credentials elsewhere on your machine. + +## Loading image data + +The plugin suite includes the OMEROReader image reader, which can be used by both NamesAndTypes and LoadData. + +To load images with the OMERO plugin we need to supply an image's unique OMERO ID. In OMERO.web, this is visible in the right pane with an image selected. We can supply image IDs to the file list in two forms: + +- As a URL in the format `https://my.omero.server/webclient/?show=image-3654` +- Using the (legacy) format `omero:iid=3654` + +Supplying the full URL is recommended, since this provides CellProfiler with the actual server address too. + +In previous versions of the integration, you needed to create a text file with one image per line and then use _File->Import->File List_ to load them +into CellProfiler. As of CellProfiler 5 you should be able to simply copy/paste these image links into the file list in the Images module. + +There is also a _Browse OMERO for Images_ option in the Plugins menu. This provides a fully featured interface for browsing an OMERO server and adding images to your pipeline. +Images can be added by selecting them and using the _Add to file list_ button, or by dragging from the tree pane onto the main file list. + +As for CellProfiler 5, these plugins also now interpret image channels correctly. If the image on OMERO contains multiple channels, you should either set the +image types in NamesAndTypes to _Color_ mode instead of _Greyscale_. Alternatively you can use the _Extract image planes_ option in the Images module and enable +splitting by channel to generate a single greyscale entry for each channel in the image. + +It should go without saying that the OMERO account you login to the server with needs to have permissions to view/read the image. + +It may be advisable to use an [OMERO.script](https://omero.readthedocs.io/en/stable/developers/scripts/index.html) to generate any large file lists that +you want to load from OMERO. + +Some OMERO servers may have the [omero-ms-pixel-buffer](https://github.com/glencoesoftware/omero-ms-pixel-buffer) microservice installed. This provides a conventional +HTTP API for fetching image data using a specially formatted URL. Since these are seen as standard file downloads the OMERO plugin is not needed to load data from this microservice into CellProfiler. + +## Saving data to OMERO + +The OMERO integration includes two module plugins for sending data back to OMERO: SaveImagesToOMERO and ExportToOMEROTable. + +### SaveImagesToOMERO + +The SaveImagesToOMERO plugin functions similarly to SaveImages, but the exported image is instead uploaded to the OMERO server. + +Images on OMERO are generally contained within Datasets. Datasets also have a unique ID visible within the right panel of OMERO.web. +To use the module you'll need to supply the target dataset's ID (or other parent object type), and resulting images will be uploaded to that dataset. + +On OMERO image names do not need to be unique (only image IDs). You may want to use metadata fields to construct a distinguishable name for each uploaded image. + +At present uploaded images are not linked to any data previously read from OMERO, so make sure you have a means of identifying the image from it's name. + +### ExportToOMEROTable + +[OMERO tables](https://omero.readthedocs.io/en/stable/developers/Tables.html) are tabular data stores which can be viewed in OMERO.web. Like all OMERO objects, tables are +associated with a parent object (typically an image or dataset). You'll need to provide an ID for the parent object. + +The module functions similarly to ExportToDatabase, in that measurements are uploaded after each image set completes. One caveat is that +it is not possible to add columns to an OMERO.table after it's initial creation, therefore certain meta-measurements are not available in this module. + +To retrieve and analyse data from OMERO.tables in other software, you should be able to use the [omero2pandas](https://github.com/glencoesoftware/omero2pandas) package. This will retrieve +table data as Pandas dataframes, allowing for their use with the wider Python scientific stack. + +## Troubleshooting + +- OMERO's Python API currently depends on a very specific version of the `zeroc-ice` package, which can be difficult to build and install. +The setup.py dependency manager has been supplied with several prebuilt wheels which should cater to most systems. Please raise an issue if you encounter problems. + +- When a server connection is established, CellProfiler will ping the server periodically to keep the session going. +The server connections may time out if your machine enters sleep mode for a prolonged period. If you see errors after waking from sleep, try re-running the _Connect to OMERO_ dialog. + +- To upload data you'll need relevant permissions both for the OMERO group and for whichever object you're attaching data to. The plugin's test button will +verify that the object exists, but not that you have write permissions. + +- Uploaded images from SaveImagesToOMERO are treated as fresh data by OMERO. Any channel settings and other metadata will not be copied over from loaded image. diff --git a/documentation/CP-plugins-documentation/supported_plugins.md b/documentation/CP-plugins-documentation/supported_plugins.md index 3cdb4d3..d512997 100644 --- a/documentation/CP-plugins-documentation/supported_plugins.md +++ b/documentation/CP-plugins-documentation/supported_plugins.md @@ -9,22 +9,22 @@ See [using plugins](using_plugins.md) for how to set up CellProfiler for plugin Most plugin documentation can be found within the plugin itself and can be accessed through CellProfiler help. Those plugins that do have extra documentation contain links below. -| Plugin | Description | Requires installation of dependencies? | Install flag | -|-------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------|--------------| -| CalculateMoments | CalculateMoments extracts moments statistics from a given distribution of pixel values. | No | | -| CallBarcodes | CallBarcodes is used for assigning a barcode to an object based on the channel with the strongest intensity for a given number of cycles. It is used for optical sequencing by synthesis (SBS). | No | | -| CompensateColors | CompensateColors determines how much signal in any given channel is because of bleed-through from another channel and removes the bleed-through. It can be performed across an image or masked to objects and provides a number of preprocessing and rescaling options to allow for troubleshooting if input image intensities are not well matched. | No | | -| DistanceTransform | DistanceTransform computes the distance transform of a binary image. The distance of each foreground pixel is computed to the nearest background pixel and the resulting image is then scaled so that the largest distance is 1. | No | | -| EnhancedMeasureTexture | EnhancedMeasureTexture measures the degree and nature of textures within an image or objects in a more comprehensive/tuneable manner than the MeasureTexture module native to CellProfiler. | No | | -| HistogramEqualization | HistogramEqualization increases the global contrast of a low-contrast image or volume. Histogram equalization redistributes intensities to utilize the full range of intensities, such that the most common frequencies are more distinct. This module can perform either global or local histogram equalization. | No | | -| HistogramMatching | HistogramMatching manipulates the pixel intensity values an input image and matches them to the histogram of a reference image. It can be used as a way to normalize intensities across different 2D or 3D images or different frames of the same 3D image. It allows you to choose which frame to use as the reference. | No | | -| PixelShuffle | PixelShuffle takes the intensity of each pixel in an image and randomly shuffles its position. | No | | -| Predict | Predict allows you to use an ilastik pixel classifier to generate a probability image. CellProfiler supports two types of ilastik projects: Pixel Classification and Autocontext (2-stage). | No | | -| [RunCellpose](RunCellPose.md) | RunCellpose allows you to run Cellpose within CellProfiler. Cellpose is a generalist machine-learning algorithm for cellular segmentation and is a great starting point for segmenting non-round cells. You can use pre-trained Cellpose models or your custom model with this plugin. You can use a GPU with this module to dramatically increase your speed/efficiency. | Yes | `cellpose` | -| RunImageJScript | RunImageJScript allows you to run any supported ImageJ script directly within CellProfiler. It is significantly more performant than RunImageJMacro, and is also less likely to leave behind temporary files. | Yes | XXXXX | -| RunOmnipose | RunOmnipose allows you to run Omnipose within CellProfiler. Omnipose is a general image segmentation tool that builds on Cellpose. | Yes | `omnipose` | -| RunStarDist | RunStarDist allows you to run StarDist within CellProfiler. StarDist is a machine-learning algorithm for object detection with star-convex shapes making it best suited for nuclei or round-ish cells. You can use pre-trained StarDist models or your custom model with this plugin. You can use a GPU with this module to dramatically increase your speed/efficiency. RunStarDist is generally faster than RunCellpose. | Yes | `stardist` | -| VarianceTransform | This module allows you to calculate the variance of an image, using a determined window size. It also has the option to find the optimal window size from a predetermined range to obtain the maximum variance of an image. | No | | -| OMEROReader | This reader allows you to connect to and load images from OMERO servers. | Yes | `omero` | -| SaveImagesToOMERO | A module to upload resulting images directly onto OMERO servers. | Yes | `omero` | -| ExportToOMEROTable | A module to upload results tables directly onto OMERO servers. | Yes | `omero` | +| Plugin | Description | Requires installation of dependencies? | Install flag | +|--------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------|--------------| +| CalculateMoments | CalculateMoments extracts moments statistics from a given distribution of pixel values. | No | | +| CallBarcodes | CallBarcodes is used for assigning a barcode to an object based on the channel with the strongest intensity for a given number of cycles. It is used for optical sequencing by synthesis (SBS). | No | | +| CompensateColors | CompensateColors determines how much signal in any given channel is because of bleed-through from another channel and removes the bleed-through. It can be performed across an image or masked to objects and provides a number of preprocessing and rescaling options to allow for troubleshooting if input image intensities are not well matched. | No | | +| DistanceTransform | DistanceTransform computes the distance transform of a binary image. The distance of each foreground pixel is computed to the nearest background pixel and the resulting image is then scaled so that the largest distance is 1. | No | | +| EnhancedMeasureTexture | EnhancedMeasureTexture measures the degree and nature of textures within an image or objects in a more comprehensive/tuneable manner than the MeasureTexture module native to CellProfiler. | No | | +| HistogramEqualization | HistogramEqualization increases the global contrast of a low-contrast image or volume. Histogram equalization redistributes intensities to utilize the full range of intensities, such that the most common frequencies are more distinct. This module can perform either global or local histogram equalization. | No | | +| HistogramMatching | HistogramMatching manipulates the pixel intensity values an input image and matches them to the histogram of a reference image. It can be used as a way to normalize intensities across different 2D or 3D images or different frames of the same 3D image. It allows you to choose which frame to use as the reference. | No | | +| PixelShuffle | PixelShuffle takes the intensity of each pixel in an image and randomly shuffles its position. | No | | +| Predict | Predict allows you to use an ilastik pixel classifier to generate a probability image. CellProfiler supports two types of ilastik projects: Pixel Classification and Autocontext (2-stage). | No | | +| [RunCellpose](RunCellPose.md) | RunCellpose allows you to run Cellpose within CellProfiler. Cellpose is a generalist machine-learning algorithm for cellular segmentation and is a great starting point for segmenting non-round cells. You can use pre-trained Cellpose models or your custom model with this plugin. You can use a GPU with this module to dramatically increase your speed/efficiency. | Yes | `cellpose` | +| RunImageJScript | RunImageJScript allows you to run any supported ImageJ script directly within CellProfiler. It is significantly more performant than RunImageJMacro, and is also less likely to leave behind temporary files. | Yes | XXXXX | +| RunOmnipose | RunOmnipose allows you to run Omnipose within CellProfiler. Omnipose is a general image segmentation tool that builds on Cellpose. | Yes | `omnipose` | +| RunStarDist | RunStarDist allows you to run StarDist within CellProfiler. StarDist is a machine-learning algorithm for object detection with star-convex shapes making it best suited for nuclei or round-ish cells. You can use pre-trained StarDist models or your custom model with this plugin. You can use a GPU with this module to dramatically increase your speed/efficiency. RunStarDist is generally faster than RunCellpose. | Yes | `stardist` | +| VarianceTransform | This module allows you to calculate the variance of an image, using a determined window size. It also has the option to find the optimal window size from a predetermined range to obtain the maximum variance of an image. | No | | +| [OMEROReader](OMERO.md) | This reader allows you to connect to and load images from OMERO servers. | Yes | `omero` | +| [SaveImagesToOMERO](OMERO.md) | A module to upload resulting images directly onto OMERO servers. | Yes | `omero` | +| [ExportToOMEROTable](OMERO.md) | A module to upload results tables directly onto OMERO servers. | Yes | `omero` | From 8fedc5fab71863c9dbf35e8bc2fa401983a618cf Mon Sep 17 00:00:00 2001 From: Beth Cimini Date: Tue, 1 Aug 2023 09:02:53 -0400 Subject: [PATCH 24/32] Fix tests (#210) * Fix tests * Pin cython * Type gooder * Install cython earlier * back back baaaaack * Update setup.py * Install centrosome branch * Update test_cellpose.yml * Update test_stardist.yml --- .github/workflows/test.yml | 21 ++------------------- .github/workflows/test_cellpose.yml | 6 ++---- .github/workflows/test_stardist.yml | 6 ++---- 3 files changed, 6 insertions(+), 27 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9a7295d..7f48bfb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -46,10 +46,9 @@ jobs: pip install wxPython-4.1.0-cp38-cp38-linux_x86_64.whl pip install --upgrade pip setuptools wheel pip install numpy + pip install git+https://github.com/CellProfiler/centrosome.git@cython3 - test_basic_plugins: - needs: build_cellprofiler - - name: Install basic CellProfiler plugins (require ) + - name: Install basic CellProfiler plugins require run: | pip install -e . - name: Run basic plugin pipeline @@ -62,19 +61,3 @@ jobs: sh ./tests/headless_test/test_run.sh "module histogrammatching" logfile sh ./tests/headless_test/test_run.sh "module pixelshuffle" logfile - test_cellpose_plugin: - needs: build_cellprofiler - - name: Run cellpose plugin pipeline - run: | - pip install -e .[cellpose] - python -m cellprofiler -c -r -p ./tests/headless_test/4.2.5_plugins_test_pipeline_CELLPOSE.cppipe -i ./tests/headless_test/test_pipeline_img -o . --plugins-directory=./active_plugins --log-level=DEBUG 2>&1 | tee logfile - sh ./tests/headless_test/test_run.sh "module runcellpose" logfile - - test_stardist_plugin: - needs: build_cellprofiler - - name: Run stardist plugin pipeline - run: | - pip uninstall -y cellpose torch - pip install -e .[stardist] - python -m cellprofiler -c -r -p ./tests/headless_test/4.2.5_plugins_test_pipeline_STARDIST.cppipe -i ./tests/headless_test/test_pipeline_img -o . --plugins-directory=./active_plugins --log-level=DEBUG 2>&1 | tee logfile - sh ./tests/headless_test/test_run.sh "module runstardist" logfile diff --git a/.github/workflows/test_cellpose.yml b/.github/workflows/test_cellpose.yml index 1e7e76b..55cdb29 100644 --- a/.github/workflows/test_cellpose.yml +++ b/.github/workflows/test_cellpose.yml @@ -9,7 +9,7 @@ on: jobs: build_cellprofiler: - name: Build CellProfiler + name: Test CellProfiler-Cellpose runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -46,10 +46,8 @@ jobs: pip install wxPython-4.1.0-cp38-cp38-linux_x86_64.whl pip install --upgrade pip setuptools wheel pip install numpy + pip install git+https://github.com/CellProfiler/centrosome.git@cython3 - - test_cellpose_plugin: - needs: build_cellprofiler - name: Run cellpose plugin pipeline run: | pip install -e .[cellpose] diff --git a/.github/workflows/test_stardist.yml b/.github/workflows/test_stardist.yml index 5788411..da710f4 100644 --- a/.github/workflows/test_stardist.yml +++ b/.github/workflows/test_stardist.yml @@ -9,7 +9,7 @@ on: jobs: build_cellprofiler: - name: Build CellProfiler + name: Test CellProfiler-Stardist runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -46,12 +46,10 @@ jobs: pip install wxPython-4.1.0-cp38-cp38-linux_x86_64.whl pip install --upgrade pip setuptools wheel pip install numpy + pip install git+https://github.com/CellProfiler/centrosome.git@cython3 - test_stardist_plugin: - needs: build_cellprofiler - name: Run stardist plugin pipeline run: | - pip uninstall -y cellpose torch pip install -e .[stardist] python -m cellprofiler -c -r -p ./tests/headless_test/4.2.5_plugins_test_pipeline_STARDIST.cppipe -i ./tests/headless_test/test_pipeline_img -o . --plugins-directory=./active_plugins --log-level=DEBUG 2>&1 | tee logfile sh ./tests/headless_test/test_run.sh "module runstardist" logfile From 267662d15d2406936c8e9f5ef16d6222b5f7f666 Mon Sep 17 00:00:00 2001 From: callum-jpg Date: Fri, 4 Aug 2023 21:49:00 +0100 Subject: [PATCH 25/32] RunCellpose via docker (#203) * adding docker subprocess option to RunCellpose * Dockerfile and associated script for building RunCellpose docker * add docker documentation * better docs since docker image is automatically downloaded when requested * upgrade module version * fix upgrade settings * docs for using docker * Add model to upgrade_settings * Typo * Fix upgrade_settings fixier --------- Co-authored-by: Erin Weisbart <54687786+ErinWeisbart@users.noreply.github.com> Co-authored-by: Beth Cimini Co-authored-by: bethac07 --- active_plugins/runcellpose.py | 319 ++++++++++++------ dockerfiles/RunCellpose/Dockerfile | 7 + .../RunCellpose/download_cellpose_models.py | 8 + .../CP-plugins-documentation/RunCellPose.md | 8 +- .../CP-plugins-documentation/_toc.yml | 2 + .../CP-plugins-documentation/using_plugins.md | 26 +- 6 files changed, 256 insertions(+), 114 deletions(-) create mode 100644 dockerfiles/RunCellpose/Dockerfile create mode 100644 dockerfiles/RunCellpose/download_cellpose_models.py diff --git a/active_plugins/runcellpose.py b/active_plugins/runcellpose.py index e838af9..1b7da41 100644 --- a/active_plugins/runcellpose.py +++ b/active_plugins/runcellpose.py @@ -6,9 +6,13 @@ import numpy import os -from cellpose import models, io, core, utils -from skimage.transform import resize +import skimage import importlib.metadata +import subprocess +import uuid +import shutil +import logging +import sys ################################# # @@ -19,10 +23,11 @@ from cellprofiler_core.image import Image from cellprofiler_core.module.image_segmentation import ImageSegmentation from cellprofiler_core.object import Objects -from cellprofiler_core.setting import Binary +from cellprofiler_core.setting import Binary, ValidationError from cellprofiler_core.setting.choice import Choice from cellprofiler_core.setting.do_something import DoSomething from cellprofiler_core.setting.subscriber import ImageSubscriber +from cellprofiler_core.preferences import get_default_output_directory from cellprofiler_core.setting.text import ( Integer, ImageName, @@ -34,7 +39,7 @@ CUDA_LINK = "https://pytorch.org/get-started/locally/" Cellpose_link = " https://doi.org/10.1038/s41592-020-01018-x" Omnipose_link = "https://doi.org/10.1101/2021.11.03.467199" -cellpose_ver = importlib.metadata.version("cellpose") +LOGGER = logging.getLogger(__name__) __doc__ = f"""\ RunCellpose @@ -78,8 +83,11 @@ """ -model_dic = models.MODEL_NAMES -model_dic.append("custom") +CELLPOSE_DOCKER_NO_PRETRAINED = "cellprofiler/runcellpose_no_pretrained:0.1" +CELLPOSE_DOCKER_IMAGE_WITH_PRETRAINED = "cellprofiler/runcellpose_with_pretrained:0.1" + +MODEL_NAMES = ['cyto','nuclei','tissuenet','livecell', 'cyto2', 'general', + 'CP', 'CPx', 'TN1', 'TN2', 'TN3', 'LC1', 'LC2', 'LC3', 'LC4', 'custom'] class RunCellpose(ImageSegmentation): @@ -87,7 +95,7 @@ class RunCellpose(ImageSegmentation): module_name = "RunCellpose" - variable_revision_number = 3 + variable_revision_number = 4 doi = { "Please cite the following when using RunCellPose:": "https://doi.org/10.1038/s41592-020-01018-x", @@ -97,9 +105,42 @@ class RunCellpose(ImageSegmentation): def create_settings(self): super(RunCellpose, self).create_settings() + self.docker_or_python = Choice( + text="Run CellPose in docker or local python environment", + choices=["Docker", "Python"], + value="Docker", + doc="""\ +If Docker is selected, ensure that Docker Desktop is open and running on your +computer. On first run of the RunCellpose plugin, the Docker container will be +downloaded. However, this slow downloading process will only have to happen +once. + +If Python is selected, the Python environment in which CellProfiler and Cellpose +are installed will be used. +""", + ) + + self.docker_image = Choice( + text="Select Cellpose docker image", + choices=[CELLPOSE_DOCKER_IMAGE_WITH_PRETRAINED, CELLPOSE_DOCKER_NO_PRETRAINED], + value=CELLPOSE_DOCKER_IMAGE_WITH_PRETRAINED, + doc="""\ +Select which Docker image to use for running Cellpose. + +If you are not using a custom model, you can select +**"{CELLPOSE_DOCKER_IMAGE_WITH_PRETRAINED}"**. If you are using a custom model, +you can use either **"{CELLPOSE_DOCKER_NO_PRETRAINED}"** or +**"{CELLPOSE_DOCKER_IMAGE_WITH_PRETRAINED}"**, but the latter will be slightly +larger (~500 MB) due to including all of the pretrained models. +""".format( + **{"CELLPOSE_DOCKER_NO_PRETRAINED": CELLPOSE_DOCKER_NO_PRETRAINED, + "CELLPOSE_DOCKER_IMAGE_WITH_PRETRAINED": CELLPOSE_DOCKER_IMAGE_WITH_PRETRAINED} +), + ) + self.expected_diameter = Integer( text="Expected object diameter", - value=15, + value=30, minval=0, doc="""\ The average diameter of the objects to be detected. Setting this to 0 will attempt to automatically detect object size. @@ -113,8 +154,8 @@ def create_settings(self): self.mode = Choice( text="Detection mode", - choices=model_dic, - value="cyto2", + choices=MODEL_NAMES, + value=MODEL_NAMES[0], doc="""\ CellPose comes with models for detecting nuclei or cells. Alternatively, you can supply a custom-trained model generated using the command line or Cellpose GUI. Custom models can be useful if working with unusual cell types. @@ -154,7 +195,7 @@ def create_settings(self): self.use_averaging = Binary( text="Use averaging", - value=True, + value=False, doc="""\ If enabled, CellPose will run it's 4 inbuilt models and take a consensus to determine the results. If disabled, only a single model will be called to produce results. Disabling averaging is faster to run but less accurate.""", @@ -299,6 +340,8 @@ def set_directory_fn(path): def settings(self): return [ self.x_name, + self.docker_or_python, + self.docker_image, self.expected_diameter, self.mode, self.y_name, @@ -322,10 +365,15 @@ def settings(self): ] def visible_settings(self): - if float(cellpose_ver[0:3]) >= 0.6 and int(cellpose_ver[0]) < 2: - vis_settings = [self.mode, self.omni, self.x_name] - else: - vis_settings = [self.mode, self.x_name] + vis_settings = [self.docker_or_python] + + if self.docker_or_python.value == "Docker": + vis_settings += [self.docker_image] + + vis_settings += [self.mode, self.x_name] + + if self.docker_or_python.value == "Python": + vis_settings += [self.omni] if self.mode.value != "nuclei": vis_settings += [self.supply_nuclei] @@ -357,8 +405,9 @@ def visible_settings(self): vis_settings += [self.use_averaging, self.use_gpu] - if self.use_gpu.value: - vis_settings += [self.gpu_test, self.manual_GPU_memory_share] + if self.docker_or_python.value == 'Python': + if self.use_gpu.value: + vis_settings += [self.gpu_test, self.manual_GPU_memory_share] return vis_settings @@ -375,48 +424,16 @@ def validate_module(self, pipeline): "Failed to load custom file: %s " % model_path, self.model_file_name, ) - try: - model = models.CellposeModel( - pretrained_model=model_path, gpu=self.use_gpu.value - ) - except: - raise ValidationError( - "Failed to load custom model: %s " % model_path, - self.model_file_name, - ) + if self.docker_or_python.value == "Python": + try: + model = models.CellposeModel(pretrained_model=model_path, gpu=self.use_gpu.value) + except: + raise ValidationError( + "Failed to load custom model: %s " + % model_path, self.model_file_name, + ) def run(self, workspace): - if float(cellpose_ver[0:3]) >= 0.6 and int(cellpose_ver[0]) < 2: - if self.mode.value != "custom": - model = models.Cellpose( - model_type=self.mode.value, gpu=self.use_gpu.value - ) - else: - model_file = self.model_file_name.value - model_directory = self.model_directory.get_absolute_path() - model_path = os.path.join(model_directory, model_file) - model = models.CellposeModel( - pretrained_model=model_path, gpu=self.use_gpu.value - ) - - else: - if self.mode.value != "custom": - model = models.CellposeModel( - model_type=self.mode.value, gpu=self.use_gpu.value - ) - else: - model_file = self.model_file_name.value - model_directory = self.model_directory.get_absolute_path() - model_path = os.path.join(model_directory, model_file) - model = models.CellposeModel( - pretrained_model=model_path, gpu=self.use_gpu.value - ) - - if self.use_gpu.value and model.torch: - from torch import cuda - - cuda.set_per_process_memory_fraction(self.manual_GPU_memory_share.value) - x_name = self.x_name.value y_name = self.y_name.value images = workspace.image_set @@ -424,10 +441,11 @@ def run(self, workspace): dimensions = x.dimensions x_data = x.pixel_data anisotropy = 0.0 - if self.do_3D.value: anisotropy = x.spacing[0] / x.spacing[1] + diam = self.expected_diameter.value if self.expected_diameter.value > 0 else None + if x.multichannel: raise ValueError( "Color images are not currently supported. Please provide greyscale images." @@ -450,59 +468,142 @@ def run(self, workspace): else: channels = [0, 0] - diam = ( - self.expected_diameter.value if self.expected_diameter.value > 0 else None - ) + if self.docker_or_python.value == "Python": + from cellpose import models, io, core, utils + self.cellpose_ver = importlib.metadata.version('cellpose') + if float(self.cellpose_ver[0:3]) >= 0.6 and int(self.cellpose_ver[0])<2: + if self.mode.value != 'custom': + model = models.Cellpose(model_type= self.mode.value, + gpu=self.use_gpu.value) + else: + model_file = self.model_file_name.value + model_directory = self.model_directory.get_absolute_path() + model_path = os.path.join(model_directory, model_file) + model = models.CellposeModel(pretrained_model=model_path, gpu=self.use_gpu.value) - try: - if float(cellpose_ver[0:3]) >= 0.7 and int(cellpose_ver[0]) < 2: - y_data, flows, *_ = model.eval( - x_data, - channels=channels, - diameter=diam, - net_avg=self.use_averaging.value, - do_3D=self.do_3D.value, - anisotropy=anisotropy, - flow_threshold=self.flow_threshold.value, - cellprob_threshold=self.cellprob_threshold.value, - stitch_threshold=self.stitch_threshold.value, - min_size=self.min_size.value, - omni=self.omni.value, - invert=self.invert.value, - ) else: - y_data, flows, *_ = model.eval( - x_data, - channels=channels, - diameter=diam, - net_avg=self.use_averaging.value, - do_3D=self.do_3D.value, - anisotropy=anisotropy, - flow_threshold=self.flow_threshold.value, - cellprob_threshold=self.cellprob_threshold.value, - stitch_threshold=self.stitch_threshold.value, - min_size=self.min_size.value, - invert=self.invert.value, + if self.mode.value != 'custom': + model = models.CellposeModel(model_type= self.mode.value, + gpu=self.use_gpu.value) + else: + model_file = self.model_file_name.value + model_directory = self.model_directory.get_absolute_path() + model_path = os.path.join(model_directory, model_file) + model = models.CellposeModel(pretrained_model=model_path, gpu=self.use_gpu.value) + + if self.use_gpu.value and model.torch: + from torch import cuda + cuda.set_per_process_memory_fraction(self.manual_GPU_memory_share.value) + + try: + if float(self.cellpose_ver[0:3]) >= 0.7 and int(self.cellpose_ver[0])<2: + y_data, flows, *_ = model.eval( + x_data, + channels=channels, + diameter=diam, + net_avg=self.use_averaging.value, + do_3D=self.do_3D.value, + anisotropy=anisotropy, + flow_threshold=self.flow_threshold.value, + cellprob_threshold=self.cellprob_threshold.value, + stitch_threshold=self.stitch_threshold.value, + min_size=self.min_size.value, + omni=self.omni.value, + invert=self.invert.value, + ) + else: + y_data, flows, *_ = model.eval( + x_data, + channels=channels, + diameter=diam, + net_avg=self.use_averaging.value, + do_3D=self.do_3D.value, + anisotropy=anisotropy, + flow_threshold=self.flow_threshold.value, + cellprob_threshold=self.cellprob_threshold.value, + stitch_threshold=self.stitch_threshold.value, + min_size=self.min_size.value, + invert=self.invert.value, ) - if self.remove_edge_masks: - y_data = utils.remove_edge_masks(y_data) + if self.remove_edge_masks: + y_data = utils.remove_edge_masks(y_data) + + except Exception as a: + print(f"Unable to create masks. Check your module settings. {a}") + finally: + if self.use_gpu.value and model.torch: + # Try to clear some GPU memory for other worker processes. + try: + cuda.empty_cache() + except Exception as e: + print(f"Unable to clear GPU memory. You may need to restart CellProfiler to change models. {e}") + + elif self.docker_or_python.value == "Docker": + # Define how to call docker + docker_path = "docker" if sys.platform.lower().startswith("win") else "/usr/local/bin/docker" + # Create a UUID for this run + unique_name = str(uuid.uuid4()) + # Directory that will be used to pass images to the docker container + temp_dir = os.path.join(get_default_output_directory(), ".cellprofiler_temp", unique_name) + temp_img_dir = os.path.join(temp_dir, "img") + + os.makedirs(temp_dir, exist_ok=True) + os.makedirs(temp_img_dir, exist_ok=True) + + temp_img_path = os.path.join(temp_img_dir, unique_name+".tiff") + if self.mode.value == "custom": + model_file = self.model_file_name.value + model_directory = self.model_directory.get_absolute_path() + model_path = os.path.join(model_directory, model_file) + temp_model_dir = os.path.join(temp_dir, "model") + + os.makedirs(temp_model_dir, exist_ok=True) + # Copy the model + shutil.copy(model_path, os.path.join(temp_model_dir, model_file)) + + # Save the image to the Docker mounted directory + skimage.io.imsave(temp_img_path, x_data) + + cmd = f""" + {docker_path} run --rm -v {temp_dir}:/data + {self.docker_image.value} + {'--gpus all' if self.use_gpu.value else ''} + cellpose + --dir /data/img + {'--pretrained_model ' + self.mode.value if self.mode.value != 'custom' else '--pretrained_model /data/model/' + model_file} + --chan {channels[0]} + --chan2 {channels[1]} + --diameter {diam} + {'--net_avg' if self.use_averaging.value else ''} + {'--do_3D' if self.do_3D.value else ''} + --anisotropy {anisotropy} + --flow_threshold {self.flow_threshold.value} + --cellprob_threshold {self.cellprob_threshold.value} + --stitch_threshold {self.stitch_threshold.value} + --min_size {self.min_size.value} + {'--invert' if self.invert.value else ''} + {'--exclude_on_edges' if self.remove_edge_masks.value else ''} + --verbose + """ - y = Objects() - y.segmented = y_data + try: + subprocess.run(cmd.split(), text=True) + cellpose_output = numpy.load(os.path.join(temp_img_dir, unique_name + "_seg.npy"), allow_pickle=True).item() - except Exception as a: - print(f"Unable to create masks. Check your module settings. {a}") - finally: - if self.use_gpu.value and model.torch: - # Try to clear some GPU memory for other worker processes. + y_data = cellpose_output["masks"] + flows = cellpose_output["flows"] + finally: + # Delete the temporary files try: - cuda.empty_cache() - except Exception as e: - print( - f"Unable to clear GPU memory. You may need to restart CellProfiler to change models. {e}" - ) + shutil.rmtree(temp_dir) + except: + LOGGER.error("Unable to delete temporary directory, files may be in use by another program.") + LOGGER.error("Temp folder is subfolder {tempdir} in your Default Output Folder.\nYou may need to remove it manually.") + + y = Objects() + y.segmented = y_data y.parent_image = x.parent_image objects = workspace.object_set objects.add_objects(y, y_name) @@ -510,7 +611,7 @@ def run(self, workspace): if self.save_probabilities.value: # Flows come out sized relative to CellPose's inbuilt model size. # We need to slightly resize to match the original image. - size_corrected = resize(flows[2], y_data.shape) + size_corrected = skimage.transform.resize(flows[2], y_data.shape) prob_image = Image( size_corrected, parent_image=x.parent_image, @@ -571,10 +672,9 @@ def display(self, workspace, figure): def do_check_gpu(self): import importlib.util - - torch_installed = importlib.util.find_spec("torch") is not None - # if the old version of cellpose <2.0, then use istorch kwarg - if float(cellpose_ver[0:3]) >= 0.7 and int(cellpose_ver[0]) < 2: + torch_installed = importlib.util.find_spec('torch') is not None + #if the old version of cellpose <2.0, then use istorch kwarg + if float(self.cellpose_ver[0:3]) >= 0.7 and int(self.cellpose_ver[0])<2: GPU_works = core.use_gpu(istorch=torch_installed) else: # if new version of cellpose, use use_torch kwarg GPU_works = core.use_gpu(use_torch=torch_installed) @@ -595,4 +695,7 @@ def upgrade_settings(self, setting_values, variable_revision_number, module_name if variable_revision_number == 2: setting_values = setting_values + ["0.0", False, "15", "1.0", False, False] variable_revision_number = 3 + if variable_revision_number == 3: + setting_values = [setting_values[0]] + ["Python",CELLPOSE_DOCKER_IMAGE_WITH_PRETRAINED] + setting_values[1:] + variable_revision_number = 4 return setting_values, variable_revision_number diff --git a/dockerfiles/RunCellpose/Dockerfile b/dockerfiles/RunCellpose/Dockerfile new file mode 100644 index 0000000..c46a0ab --- /dev/null +++ b/dockerfiles/RunCellpose/Dockerfile @@ -0,0 +1,7 @@ +FROM pytorch/pytorch:1.13.0-cuda11.6-cudnn8-runtime + +RUN pip install cellpose==2.2 + +# Include if you wish the image to contain Cellpose pretrained models +COPY download_cellpose_models.py / +RUN python /download_cellpose_models.py diff --git a/dockerfiles/RunCellpose/download_cellpose_models.py b/dockerfiles/RunCellpose/download_cellpose_models.py new file mode 100644 index 0000000..249f5c8 --- /dev/null +++ b/dockerfiles/RunCellpose/download_cellpose_models.py @@ -0,0 +1,8 @@ +import cellpose +from cellpose.models import MODEL_NAMES + +for model in MODEL_NAMES: + for model_index in range(4): + model_name = cellpose.models.model_path(model, model_index) + if model in ("cyto", "nuclei", "cyto2"): + size_model_name = cellpose.models.size_model_path(model) \ No newline at end of file diff --git a/documentation/CP-plugins-documentation/RunCellPose.md b/documentation/CP-plugins-documentation/RunCellPose.md index 168fa7c..744e667 100644 --- a/documentation/CP-plugins-documentation/RunCellPose.md +++ b/documentation/CP-plugins-documentation/RunCellPose.md @@ -1,6 +1,10 @@ -# RunCellPose +# RunCellpose -## Using RunCellPose with a GPU +RunCellpose is one of the modules that has additional dependencies that are not packaged with the built CellProfiler. +Therefore, you must additionally download RunCellpose's dependencies. +See [Using Plugins](using_plugins.md) for more information. + +## Using RunCellpose with a GPU If you want to use a GPU to run the model (this is recommended for speed), you'll need a compatible version of PyTorch and a supported GPU. General instructions are available at this [link](https://pytorch.org/get-started/locally/). diff --git a/documentation/CP-plugins-documentation/_toc.yml b/documentation/CP-plugins-documentation/_toc.yml index 1ecd291..64f99d2 100644 --- a/documentation/CP-plugins-documentation/_toc.yml +++ b/documentation/CP-plugins-documentation/_toc.yml @@ -6,6 +6,8 @@ parts: - caption: Overview chapters: - file: using_plugins + sections: RunCellpose + - file: config_examples - file: supported_plugins - file: unsupported_plugins - file: contributing_plugins diff --git a/documentation/CP-plugins-documentation/using_plugins.md b/documentation/CP-plugins-documentation/using_plugins.md index 9c05ecd..d2fbfa6 100644 --- a/documentation/CP-plugins-documentation/using_plugins.md +++ b/documentation/CP-plugins-documentation/using_plugins.md @@ -14,11 +14,15 @@ Please report any installation issues or bugs related to plugins in the [CellPro If the plugin you would like to use does not have any additional dependencies outside of those required for running CellProfiler (this is most plugins), using plugins is very simple. See [Installing plugins without dependencies](#installing-plugins-without-dependencies). -If the plugin you would like to use has dependencies, you have two separate options for installation. -The first option requires building CellProfiler from source, but plugin installation is simpler. +If the plugin you would like to use has dependencies, you have three separate options for installation. +- The first option requires building CellProfiler from source, but plugin installation is simpler. See [Installing plugins with dependencies, using CellProfiler from source](#installing-plugins-with-dependencies-using-cellprofiler-from-source). -The second option allows you to use pre-built CellProfiler, but plugin installation is more complex. +- The second option allows you to use pre-built CellProfiler, but plugin installation is more complex. See [Installing plugins with dependencies, using pre-built CellProfiler](#installing-plugins-with-dependencies-using-pre-built-cellprofiler). +- The third option uses Docker to bypass installation requirements. +It is the simplest option that only requires download of Docker Desktop; the module that has dependencies will automatically download a Docker that has all of the dependencies upon run and access that Docker while running the plugin. +It is currently only supported for the RunCellpose plugin but will be available in other plugins soon. +See [Using Docker to Bypass Installation Requirements](#using-docker-to-bypass-installation-requirements). ### Installing plugins without dependencies @@ -157,4 +161,18 @@ These are all the folders you need to copy over: 7. **Open and use CellProfiler.** When you try to run your plugin in your pipeline, if you have missed copying over any specific requirements, it will give you an error message that will tell you what dependency is missing in the terminal window that opens with CellProfiler on Windows machines. -This information is not available in Mac machines. \ No newline at end of file +This information is not available in Mac machines. + +### Using Docker to bypass installation requirements + +1. **Download Docker** +Download Docker Desktop from [Docker.com](https://www.docker.com/products/docker-desktop/). + +2. **Run Docker Desktop** +Open Docker Desktop. +Docker Desktop will need to be open every time you use a plugin with Docker. + +3. **Select "Run with Docker"** +In your plugin, select `Docker` for "Run module in docker or local python environment" setting. +On the first run of the plugin, the Docker container will be downloaded, however, this slow downloading process will only have to happen +once. \ No newline at end of file From a92bf84a657987326819f33c3d5753221902eab6 Mon Sep 17 00:00:00 2001 From: Beth Cimini Date: Fri, 4 Aug 2023 16:57:59 -0400 Subject: [PATCH 26/32] Update _toc.yml (#212) --- documentation/CP-plugins-documentation/_toc.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/CP-plugins-documentation/_toc.yml b/documentation/CP-plugins-documentation/_toc.yml index 64f99d2..9c508c4 100644 --- a/documentation/CP-plugins-documentation/_toc.yml +++ b/documentation/CP-plugins-documentation/_toc.yml @@ -6,8 +6,8 @@ parts: - caption: Overview chapters: - file: using_plugins - sections: RunCellpose - - file: config_examples + sections: + - file: RunCellpose - file: supported_plugins - file: unsupported_plugins - file: contributing_plugins From 836a73ae8cbc7ff7929a2bdcde5b42e180530f88 Mon Sep 17 00:00:00 2001 From: Beth Cimini Date: Tue, 8 Aug 2023 11:56:08 -0400 Subject: [PATCH 27/32] Doc updates (#211) * Update _config.yml * Add issue for zsh in macs * Update _toc.yml * runcellpose capitalization --------- Co-authored-by: Erin Weisbart <54687786+ErinWeisbart@users.noreply.github.com> --- documentation/CP-plugins-documentation/_config.yml | 4 ++-- documentation/CP-plugins-documentation/_toc.yml | 6 ++++-- .../{RunCellPose.md => runcellpose.md} | 0 documentation/CP-plugins-documentation/using_plugins.md | 6 +++++- 4 files changed, 11 insertions(+), 5 deletions(-) rename documentation/CP-plugins-documentation/{RunCellPose.md => runcellpose.md} (100%) diff --git a/documentation/CP-plugins-documentation/_config.yml b/documentation/CP-plugins-documentation/_config.yml index 702fc3a..1c0723e 100644 --- a/documentation/CP-plugins-documentation/_config.yml +++ b/documentation/CP-plugins-documentation/_config.yml @@ -17,8 +17,8 @@ execute: # Information about where the book exists on the web repository: url: https://github.com/cellprofiler/cellprofiler-plugins - branch: main # Which branch of the repository should be used when creating links (optional) - path_to_book: CP-plugins-documentation + branch: master # Which branch of the repository should be used when creating links (optional) + path_to_book: documentation/CP-plugins-documentation html: baseurl: cellprofiler.github.io diff --git a/documentation/CP-plugins-documentation/_toc.yml b/documentation/CP-plugins-documentation/_toc.yml index 9c508c4..130d632 100644 --- a/documentation/CP-plugins-documentation/_toc.yml +++ b/documentation/CP-plugins-documentation/_toc.yml @@ -6,11 +6,13 @@ parts: - caption: Overview chapters: - file: using_plugins - sections: - - file: RunCellpose - file: supported_plugins - file: unsupported_plugins - file: contributing_plugins - file: troubleshooting - file: citing - file: versions + +- caption: Extra information about specific plugins + chapters: + - file: runcellpose diff --git a/documentation/CP-plugins-documentation/RunCellPose.md b/documentation/CP-plugins-documentation/runcellpose.md similarity index 100% rename from documentation/CP-plugins-documentation/RunCellPose.md rename to documentation/CP-plugins-documentation/runcellpose.md diff --git a/documentation/CP-plugins-documentation/using_plugins.md b/documentation/CP-plugins-documentation/using_plugins.md index d2fbfa6..d823ec5 100644 --- a/documentation/CP-plugins-documentation/using_plugins.md +++ b/documentation/CP-plugins-documentation/using_plugins.md @@ -75,6 +75,8 @@ pip install -e .[FLAG] ``` e.g. To install Cellpose the pip install command would be `pip install -e .[cellpose]` +If using Mac and getting an error saying `zsh: no matches found: .[somepackage]`, put the dot and square brackets in single quotes, ie `pip install -e '.[cellpose]'` + 5. **Open and use CellProfiler.** Please note that plugins that have separate install flags may have conflicting dependencies so we recommend making a separate python environment in which to run separate installations. @@ -163,6 +165,7 @@ These are all the folders you need to copy over: When you try to run your plugin in your pipeline, if you have missed copying over any specific requirements, it will give you an error message that will tell you what dependency is missing in the terminal window that opens with CellProfiler on Windows machines. This information is not available in Mac machines. + ### Using Docker to bypass installation requirements 1. **Download Docker** @@ -175,4 +178,5 @@ Docker Desktop will need to be open every time you use a plugin with Docker. 3. **Select "Run with Docker"** In your plugin, select `Docker` for "Run module in docker or local python environment" setting. On the first run of the plugin, the Docker container will be downloaded, however, this slow downloading process will only have to happen -once. \ No newline at end of file +once. + From eca8fae0e9d37ac3e01c9787b3aec516888cc559 Mon Sep 17 00:00:00 2001 From: mccruz07 <52753134+mccruz07@users.noreply.github.com> Date: Tue, 15 Aug 2023 11:29:43 -0300 Subject: [PATCH 28/32] Update runstardist.py (#205) * Update runstardist.py Potentially resolves the issue#196 stardist plugin does not free gpu memory. Now you can runStardist and runCellpose on the same pipeline. Tested on windows, need to test on Macs * Only run if cuda is available * Add optional GPU memory control code --------- Co-authored-by: Beth Cimini --- active_plugins/runstardist.py | 53 +++++++++++++++++++++++++++++++---- 1 file changed, 48 insertions(+), 5 deletions(-) diff --git a/active_plugins/runstardist.py b/active_plugins/runstardist.py index e3015d4..b416507 100644 --- a/active_plugins/runstardist.py +++ b/active_plugins/runstardist.py @@ -8,6 +8,8 @@ from skimage.transform import resize from stardist.models import StarDist2D, StarDist3D from csbdeep.utils import normalize +from numba import cuda +import tensorflow as tf ################################# # @@ -206,6 +208,25 @@ def create_settings(self): maxval=1.0, doc=f"""\ Prevent overlapping +""", + ) + + self.manage_gpu = Binary( + text="Manually set how much GPU memory each worker can use?", + value=False, + doc=""" +If enabled, you can manually set how much of the GPU memory each worker can use. +This is likely to provide the most benefit on Macs. Do not use in a multi-GPU setup.""", + ) + + self.manual_GPU_memory_GB = Float( + text="GPU memory (in GB) for each worker", + value=0.5, + minval=0.0000001, + maxval=30, + doc="""\ +GPU memory in GB available to each worker. Value should be set such that this number times the number +of workers in each copy of CellProfiler times the number of copies of CellProfiler running (if applicable) is <1 """, ) @@ -224,6 +245,8 @@ def settings(self): self.model_choice3D, self.prob_thresh, self.nms_thresh, + self.manage_gpu, + self.manual_GPU_memory_GB, ] def visible_settings(self): @@ -250,7 +273,10 @@ def visible_settings(self): if self.tile_image.value: vis_settings += [self.n_tiles_x, self.n_tiles_y] - vis_settings += [self.prob_thresh, self.nms_thresh, self.gpu_test] + vis_settings += [self.prob_thresh, self.nms_thresh, self.gpu_test, self.manage_gpu] + + if self.manage_gpu.value: + vis_settings += [self.manual_GPU_memory_GB] return vis_settings @@ -271,6 +297,23 @@ def run(self, workspace): raise ValueError( "Greyscale images are not supported by this model. Please provide a color overlay." ) + + # Stolen nearly wholesale from https://wiki.ncsa.illinois.edu/display/ISL20/Managing+GPU+memory+when+using+Tensorflow+and+Pytorch + if self.manage_gpu.value: + # First, Get a list of GPU devices + gpus = tf.config.list_physical_devices('GPU') + if len(gpus) > 0: + # Restrict to only the first GPU. + tf.config.set_visible_devices(gpus[:1], device_type='GPU') + # Create a LogicalDevice with the appropriate memory limit + log_dev_conf = tf.config.LogicalDeviceConfiguration( + memory_limit=self.manual_GPU_memory_GB.value*1024 # 2 GB + ) + # Apply the logical device configuration to the first GPU + tf.config.set_logical_device_configuration( + gpus[0], + [log_dev_conf]) + if self.model.value == CUSTOM_MODEL: model_directory, model_name = os.path.split( @@ -347,6 +390,8 @@ def run(self, workspace): objects.add_objects(y, self.y_name.value) self.add_measurements(workspace) + if cuda.is_available(): + cuda.current_context().memory_manager.deallocations.clear() if self.show_window: workspace.display_data.x_data = x_data @@ -389,11 +434,9 @@ def display(self, workspace, figure): ) def do_check_gpu(self): - import tensorflow - - if len(tensorflow.config.list_physical_devices("GPU")) > 0: + if len(tf.config.list_physical_devices("GPU")) > 0: message = "GPU appears to be working correctly!" - print("GPUs:", tensorflow.config.list_physical_devices("GPU")) + print("GPUs:", tf.config.list_physical_devices("GPU")) else: message = ( "GPU test failed. There may be something wrong with your configuration." From 2bbb329c94a03ea67ad8cb5d62c17649a78dbce0 Mon Sep 17 00:00:00 2001 From: Beth Cimini Date: Tue, 22 Aug 2023 16:27:06 -0400 Subject: [PATCH 29/32] Update runstardist.py (#214) --- active_plugins/runstardist.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/active_plugins/runstardist.py b/active_plugins/runstardist.py index b416507..cea418b 100644 --- a/active_plugins/runstardist.py +++ b/active_plugins/runstardist.py @@ -225,8 +225,10 @@ def create_settings(self): minval=0.0000001, maxval=30, doc="""\ -GPU memory in GB available to each worker. Value should be set such that this number times the number -of workers in each copy of CellProfiler times the number of copies of CellProfiler running (if applicable) is <1 +Gigabytes of GPU memory available to each worker. Value should be set such that this number times the number +of workers in each copy of CellProfiler times the number of copies of CellProfiler running (if applicable) is <1. +The "correct" value will depend on your system's GPU, the number of workers you want to run in parallel, and +the size of the model that you want to run; some experimentation may be needed. """, ) From b710fc407ac0e730bcc01cf813f94c57e02614c3 Mon Sep 17 00:00:00 2001 From: Beth Cimini Date: Thu, 7 Sep 2023 15:04:20 -0400 Subject: [PATCH 30/32] Update supported_plugins.md (#215) --- documentation/CP-plugins-documentation/supported_plugins.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/documentation/CP-plugins-documentation/supported_plugins.md b/documentation/CP-plugins-documentation/supported_plugins.md index d512997..1ab649e 100644 --- a/documentation/CP-plugins-documentation/supported_plugins.md +++ b/documentation/CP-plugins-documentation/supported_plugins.md @@ -9,7 +9,8 @@ See [using plugins](using_plugins.md) for how to set up CellProfiler for plugin Most plugin documentation can be found within the plugin itself and can be accessed through CellProfiler help. Those plugins that do have extra documentation contain links below. -| Plugin | Description | Requires installation of dependencies? | Install flag | + +| Plugin | Description | Requires installation of dependencies? | Install flag | |--------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------|--------------| | CalculateMoments | CalculateMoments extracts moments statistics from a given distribution of pixel values. | No | | | CallBarcodes | CallBarcodes is used for assigning a barcode to an object based on the channel with the strongest intensity for a given number of cycles. It is used for optical sequencing by synthesis (SBS). | No | | @@ -21,7 +22,7 @@ Those plugins that do have extra documentation contain links below. | PixelShuffle | PixelShuffle takes the intensity of each pixel in an image and randomly shuffles its position. | No | | | Predict | Predict allows you to use an ilastik pixel classifier to generate a probability image. CellProfiler supports two types of ilastik projects: Pixel Classification and Autocontext (2-stage). | No | | | [RunCellpose](RunCellPose.md) | RunCellpose allows you to run Cellpose within CellProfiler. Cellpose is a generalist machine-learning algorithm for cellular segmentation and is a great starting point for segmenting non-round cells. You can use pre-trained Cellpose models or your custom model with this plugin. You can use a GPU with this module to dramatically increase your speed/efficiency. | Yes | `cellpose` | -| RunImageJScript | RunImageJScript allows you to run any supported ImageJ script directly within CellProfiler. It is significantly more performant than RunImageJMacro, and is also less likely to leave behind temporary files. | Yes | XXXXX | +| RunImageJScript | RunImageJScript allows you to run any supported ImageJ script directly within CellProfiler. It is significantly more performant than RunImageJMacro, and is also less likely to leave behind temporary files. | Yes | `imagejscript` , though note that conda installation may be preferred, see [this link](https://py.imagej.net/en/latest/Install.html#installing-via-pip) for more information | | RunOmnipose | RunOmnipose allows you to run Omnipose within CellProfiler. Omnipose is a general image segmentation tool that builds on Cellpose. | Yes | `omnipose` | | RunStarDist | RunStarDist allows you to run StarDist within CellProfiler. StarDist is a machine-learning algorithm for object detection with star-convex shapes making it best suited for nuclei or round-ish cells. You can use pre-trained StarDist models or your custom model with this plugin. You can use a GPU with this module to dramatically increase your speed/efficiency. RunStarDist is generally faster than RunCellpose. | Yes | `stardist` | | VarianceTransform | This module allows you to calculate the variance of an image, using a determined window size. It also has the option to find the optimal window size from a predetermined range to obtain the maximum variance of an image. | No | | From b7578ccc2b2c6d7e9c9fc333179f57ea91782dfc Mon Sep 17 00:00:00 2001 From: Mark Hiner Date: Thu, 21 Sep 2023 17:16:41 -0500 Subject: [PATCH 31/32] CP-IJ: add workaround for Java names (#219) Required for newer PyImageJ releases (1.4.1) until we have a release with https://github.com/imagej/pyimagej/commit/a1861b6c1658d6751fa314650b13411f956549ab merged. Closes #218 --- active_plugins/cpij/server.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/active_plugins/cpij/server.py b/active_plugins/cpij/server.py index 02928e2..a40fc10 100644 --- a/active_plugins/cpij/server.py +++ b/active_plugins/cpij/server.py @@ -196,7 +196,13 @@ def _convert_java_to_python_type(ij, return_value): # TODO actualize changes in a virtual ImagePlus. Remove this when pyimagej does this innately if issubclass(return_class, jpype.JClass("ij.ImagePlus")): ij.py.synchronize_ij1_to_ij2(return_value) - return ij.py.from_java(return_value) + py_img = ij.py.from_java(return_value) + + # HACK + # Workaround for DataArrays potentially coming back with Java names. Fixed upstream in: + # https://github.com/imagej/pyimagej/commit/a1861b6c1658d6751fa314650b13411f956549ab + py_img.name = ij.py.from_java(py_img.name) + return py_img # Not a supported type return None From 587dd0601bc3f0a9990b719b00d0b95318b38f4d Mon Sep 17 00:00:00 2001 From: David Stirling Date: Mon, 25 Sep 2023 09:36:29 +0100 Subject: [PATCH 32/32] Fix table markdown --- .../supported_plugins.md | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/documentation/CP-plugins-documentation/supported_plugins.md b/documentation/CP-plugins-documentation/supported_plugins.md index 1ab649e..3dce741 100644 --- a/documentation/CP-plugins-documentation/supported_plugins.md +++ b/documentation/CP-plugins-documentation/supported_plugins.md @@ -10,22 +10,22 @@ Most plugin documentation can be found within the plugin itself and can be acces Those plugins that do have extra documentation contain links below. -| Plugin | Description | Requires installation of dependencies? | Install flag | -|--------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------|--------------| -| CalculateMoments | CalculateMoments extracts moments statistics from a given distribution of pixel values. | No | | -| CallBarcodes | CallBarcodes is used for assigning a barcode to an object based on the channel with the strongest intensity for a given number of cycles. It is used for optical sequencing by synthesis (SBS). | No | | -| CompensateColors | CompensateColors determines how much signal in any given channel is because of bleed-through from another channel and removes the bleed-through. It can be performed across an image or masked to objects and provides a number of preprocessing and rescaling options to allow for troubleshooting if input image intensities are not well matched. | No | | -| DistanceTransform | DistanceTransform computes the distance transform of a binary image. The distance of each foreground pixel is computed to the nearest background pixel and the resulting image is then scaled so that the largest distance is 1. | No | | -| EnhancedMeasureTexture | EnhancedMeasureTexture measures the degree and nature of textures within an image or objects in a more comprehensive/tuneable manner than the MeasureTexture module native to CellProfiler. | No | | -| HistogramEqualization | HistogramEqualization increases the global contrast of a low-contrast image or volume. Histogram equalization redistributes intensities to utilize the full range of intensities, such that the most common frequencies are more distinct. This module can perform either global or local histogram equalization. | No | | -| HistogramMatching | HistogramMatching manipulates the pixel intensity values an input image and matches them to the histogram of a reference image. It can be used as a way to normalize intensities across different 2D or 3D images or different frames of the same 3D image. It allows you to choose which frame to use as the reference. | No | | -| PixelShuffle | PixelShuffle takes the intensity of each pixel in an image and randomly shuffles its position. | No | | -| Predict | Predict allows you to use an ilastik pixel classifier to generate a probability image. CellProfiler supports two types of ilastik projects: Pixel Classification and Autocontext (2-stage). | No | | -| [RunCellpose](RunCellPose.md) | RunCellpose allows you to run Cellpose within CellProfiler. Cellpose is a generalist machine-learning algorithm for cellular segmentation and is a great starting point for segmenting non-round cells. You can use pre-trained Cellpose models or your custom model with this plugin. You can use a GPU with this module to dramatically increase your speed/efficiency. | Yes | `cellpose` | -| RunImageJScript | RunImageJScript allows you to run any supported ImageJ script directly within CellProfiler. It is significantly more performant than RunImageJMacro, and is also less likely to leave behind temporary files. | Yes | `imagejscript` , though note that conda installation may be preferred, see [this link](https://py.imagej.net/en/latest/Install.html#installing-via-pip) for more information | -| RunOmnipose | RunOmnipose allows you to run Omnipose within CellProfiler. Omnipose is a general image segmentation tool that builds on Cellpose. | Yes | `omnipose` | -| RunStarDist | RunStarDist allows you to run StarDist within CellProfiler. StarDist is a machine-learning algorithm for object detection with star-convex shapes making it best suited for nuclei or round-ish cells. You can use pre-trained StarDist models or your custom model with this plugin. You can use a GPU with this module to dramatically increase your speed/efficiency. RunStarDist is generally faster than RunCellpose. | Yes | `stardist` | -| VarianceTransform | This module allows you to calculate the variance of an image, using a determined window size. It also has the option to find the optimal window size from a predetermined range to obtain the maximum variance of an image. | No | | -| [OMEROReader](OMERO.md) | This reader allows you to connect to and load images from OMERO servers. | Yes | `omero` | -| [SaveImagesToOMERO](OMERO.md) | A module to upload resulting images directly onto OMERO servers. | Yes | `omero` | -| [ExportToOMEROTable](OMERO.md) | A module to upload results tables directly onto OMERO servers. | Yes | `omero` | +| Plugin | Description | Requires installation of dependencies? | Install flag | Docker version currently available? | +|--------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------|--------|-------------------------------------| +| CalculateMoments | CalculateMoments extracts moments statistics from a given distribution of pixel values. | No | | N/A | +| CallBarcodes | CallBarcodes is used for assigning a barcode to an object based on the channel with the strongest intensity for a given number of cycles. It is used for optical sequencing by synthesis (SBS). | No | | N/A | +| CompensateColors | CompensateColors determines how much signal in any given channel is because of bleed-through from another channel and removes the bleed-through. It can be performed across an image or masked to objects and provides a number of preprocessing and rescaling options to allow for troubleshooting if input image intensities are not well matched. | No | | N/A | +| DistanceTransform | DistanceTransform computes the distance transform of a binary image. The distance of each foreground pixel is computed to the nearest background pixel and the resulting image is then scaled so that the largest distance is 1. | No | | N/A | +| EnhancedMeasureTexture | EnhancedMeasureTexture measures the degree and nature of textures within an image or objects in a more comprehensive/tuneable manner than the MeasureTexture module native to CellProfiler. | No | | N/A | +| HistogramEqualization | HistogramEqualization increases the global contrast of a low-contrast image or volume. Histogram equalization redistributes intensities to utilize the full range of intensities, such that the most common frequencies are more distinct. This module can perform either global or local histogram equalization. | No | | N/A | +| HistogramMatching | HistogramMatching manipulates the pixel intensity values an input image and matches them to the histogram of a reference image. It can be used as a way to normalize intensities across different 2D or 3D images or different frames of the same 3D image. It allows you to choose which frame to use as the reference. | No | | N/A | +| PixelShuffle | PixelShuffle takes the intensity of each pixel in an image and randomly shuffles its position. | No | | N/A | +| Predict | Predict allows you to use an ilastik pixel classifier to generate a probability image. CellProfiler supports two types of ilastik projects: Pixel Classification and Autocontext (2-stage). | No | | N/A | +| [RunCellpose](RunCellPose.md) | RunCellpose allows you to run Cellpose within CellProfiler. Cellpose is a generalist machine-learning algorithm for cellular segmentation and is a great starting point for segmenting non-round cells. You can use pre-trained Cellpose models or your custom model with this plugin. You can use a GPU with this module to dramatically increase your speed/efficiency. | Yes | `cellpose` | Yes | +| RunImageJScript | RunImageJScript allows you to run any supported ImageJ script directly within CellProfiler. It is significantly more performant than RunImageJMacro, and is also less likely to leave behind temporary files. | Yes | `imagejscript` , though note that conda installation may be preferred, see [this link](https://py.imagej.net/en/latest/Install.html#installing-via-pip) for more information | No | +| RunOmnipose | RunOmnipose allows you to run Omnipose within CellProfiler. Omnipose is a general image segmentation tool that builds on Cellpose. | Yes | `omnipose` | No | +| RunStarDist | RunStarDist allows you to run StarDist within CellProfiler. StarDist is a machine-learning algorithm for object detection with star-convex shapes making it best suited for nuclei or round-ish cells. You can use pre-trained StarDist models or your custom model with this plugin. You can use a GPU with this module to dramatically increase your speed/efficiency. RunStarDist is generally faster than RunCellpose. | Yes | `stardist` | No | +| VarianceTransform | This module allows you to calculate the variance of an image, using a determined window size. It also has the option to find the optimal window size from a predetermined range to obtain the maximum variance of an image. | No | | N/A | +| [OMEROReader](OMERO.md) | This reader allows you to connect to and load images from OMERO servers. | Yes | `omero` | No | +| [SaveImagesToOMERO](OMERO.md) | A module to upload resulting images directly onto OMERO servers. | Yes | `omero` | No | +| [ExportToOMEROTable](OMERO.md) | A module to upload results tables directly onto OMERO servers. | Yes | `omero` | No |