From 29ec8eca459a4d2b0ff2d071739a1001d0de00aa Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Mon, 26 Jun 2023 08:57:44 -0400 Subject: [PATCH 01/25] run `poetry lock --no-update` --- poetry.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index f94e4f657..1107652fa 100644 --- a/poetry.lock +++ b/poetry.lock @@ -5919,4 +5919,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "3.10.x" -content-hash = "615276728918e478f74e65b71c3c27378d320fe5611c4c0dc4395d1835814fdc" \ No newline at end of file +content-hash = "dee9d1c731617db5df7919ad57d2126eccd9ce3fd7a6ca0688adaeb93086daec" From 2f299341d3915ed12ae9f7eca342600797258b8c Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Mon, 26 Jun 2023 11:38:18 -0400 Subject: [PATCH 02/25] add alive-progress via poetry and in code --- openadapt/record.py | 21 +++++++++++++-------- poetry.lock | 41 ++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + 3 files changed, 54 insertions(+), 9 deletions(-) diff --git a/openadapt/record.py b/openadapt/record.py index 48dc069ba..4c521b187 100644 --- a/openadapt/record.py +++ b/openadapt/record.py @@ -17,6 +17,7 @@ import threading import time +from alive_progress import alive_bar from loguru import logger from pynput import keyboard, mouse import fire @@ -247,14 +248,18 @@ def write_events( utils.set_start_time(recording_timestamp) logger.info(f"{event_type=} starting") signal.signal(signal.SIGINT, signal.SIG_IGN) - while not terminate_event.is_set() or not write_q.empty(): - try: - event = write_q.get_nowait() - except queue.Empty: - continue - assert event.type == event_type, (event_type, event) - write_fn(recording_timestamp, event, perf_q) - logger.debug(f"{event_type=} written") + + with alive_bar(total=write_q.qsize(), title=f"Writing {event_type} events") as progress: + while not terminate_event.is_set() or not write_q.empty(): + try: + event = write_q.get_nowait() + except queue.Empty: + continue + assert event.type == event_type, (event_type, event) + write_fn(recording_timestamp, event, perf_q) + logger.debug(f"{event_type=} written") + progress() + logger.info(f"{event_type=} done") diff --git a/poetry.lock b/poetry.lock index 1107652fa..7c4343aaa 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,5 +1,16 @@ # This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +[[package]] +name = "about-time" +version = "4.2.1" +description = "Easily measure timing and throughput of code blocks, with beautiful human friendly representations." +optional = false +python-versions = ">=3.7, <4" +files = [ + {file = "about-time-4.2.1.tar.gz", hash = "sha256:6a538862d33ce67d997429d14998310e1dbfda6cb7d9bbfbf799c4709847fece"}, + {file = "about_time-4.2.1-py3-none-any.whl", hash = "sha256:8bbf4c75fe13cbd3d72f49a03b02c5c7dca32169b6d49117c257e7eb3eaee341"}, +] + [[package]] name = "aiohttp" version = "3.8.4" @@ -151,6 +162,21 @@ SQLAlchemy = ">=1.3.0" [package.extras] tz = ["python-dateutil"] +[[package]] +name = "alive-progress" +version = "3.1.4" +description = "A new kind of Progress Bar, with real-time throughput, ETA, and very cool animations!" +optional = false +python-versions = ">=3.7, <4" +files = [ + {file = "alive-progress-3.1.4.tar.gz", hash = "sha256:74a95d8d0d42bc99d3a3725dbd06ebb852245f1b64e301a7c375b92b22663f7b"}, + {file = "alive_progress-3.1.4-py3-none-any.whl", hash = "sha256:c80ad87ce9c1054b01135a87fae69ecebbfc2107497ae87cbe6aec7e534903db"}, +] + +[package.dependencies] +about-time = "4.2.1" +grapheme = "0.6.0" + [[package]] name = "altgraph" version = "0.17.3" @@ -1277,6 +1303,19 @@ files = [ [package.extras] speedup = ["python-levenshtein (>=0.12)"] +[[package]] +name = "grapheme" +version = "0.6.0" +description = "Unicode grapheme helpers" +optional = false +python-versions = "*" +files = [ + {file = "grapheme-0.6.0.tar.gz", hash = "sha256:44c2b9f21bbe77cfb05835fec230bd435954275267fea1858013b102f8603cca"}, +] + +[package.extras] +test = ["pytest", "sphinx", "sphinx-autobuild", "twine", "wheel"] + [[package]] name = "greenlet" version = "2.0.2" @@ -5919,4 +5958,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "3.10.x" -content-hash = "dee9d1c731617db5df7919ad57d2126eccd9ce3fd7a6ca0688adaeb93086daec" +content-hash = "4e17975198b5cdac01ab69ce731c2db4979584a5580b42bc7748f081b5ebf64d" diff --git a/pyproject.toml b/pyproject.toml index 0cdf09a61..64d554caa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ nltk = "3.8.1" pywinauto = {version = "^0.6.8", markers = "sys_platform == 'win32'"} moviepy = "1.0.3" python-levenshtein = "^0.21.1" +alive-progress = "^3.1.4" [tool.poetry.dependencies.en_core_web_trf] url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_trf-3.5.0/en_core_web_trf-3.5.0.tar.gz" From 1ff5e9d0bbf69851bbd46c05afeca4be1aa96866 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Mon, 26 Jun 2023 11:47:15 -0400 Subject: [PATCH 03/25] add progress bar in visualization --- openadapt/visualize.py | 131 +++++++++++++++++++++-------------------- 1 file changed, 68 insertions(+), 63 deletions(-) diff --git a/openadapt/visualize.py b/openadapt/visualize.py index 1219f7f5d..358d1e3cc 100644 --- a/openadapt/visualize.py +++ b/openadapt/visualize.py @@ -4,6 +4,7 @@ import os import string +from alive_progress import alive_bar from bokeh.io import output_file, show from bokeh.layouts import layout, row from bokeh.models.widgets import Div @@ -188,70 +189,74 @@ def main(): ), ] logger.info(f"{len(action_events)=}") - for idx, action_event in enumerate(action_events): - if idx == MAX_EVENTS: - break - image = display_event(action_event) - diff = display_event(action_event, diff=True) - mask = action_event.screenshot.diff_mask - - if SCRUB: - image = scrub.scrub_image(image) - diff = scrub.scrub_image(diff) - mask = scrub.scrub_image(mask) - - image_utf8 = image2utf8(image) - diff_utf8 = image2utf8(diff) - mask_utf8 = image2utf8(mask) - width, height = image.size - - action_event_dict = row2dict(action_event) - window_event_dict = row2dict(action_event.window_event) - - if SCRUB: - action_event_dict = scrub.scrub_dict(action_event_dict) - window_event_dict = scrub.scrub_dict(window_event_dict) - - rows.append( - [ - row( - Div( - text=f""" -
- - - -
- - {dict2html(window_event_dict , None)} -
- """, - ), - Div( - text=f""" - - {dict2html(action_event_dict)} -
- """ + + with alive_bar(total=min(MAX_EVENTS, len(action_events)), title="Processing Action Events") as progress: + for idx, action_event in enumerate(action_events): + if idx == MAX_EVENTS: + break + image = display_event(action_event) + diff = display_event(action_event, diff=True) + mask = action_event.screenshot.diff_mask + + if SCRUB: + image = scrub.scrub_image(image) + diff = scrub.scrub_image(diff) + mask = scrub.scrub_image(mask) + + image_utf8 = image2utf8(image) + diff_utf8 = image2utf8(diff) + mask_utf8 = image2utf8(mask) + width, height = image.size + + action_event_dict = row2dict(action_event) + window_event_dict = row2dict(action_event.window_event) + + if SCRUB: + action_event_dict = scrub.scrub_dict(action_event_dict) + window_event_dict = scrub.scrub_dict(window_event_dict) + + rows.append( + [ + row( + Div( + text=f""" +
+ + + +
+ + {dict2html(window_event_dict , None)} +
+ """, + ), + Div( + text=f""" + + {dict2html(action_event_dict)} +
+ """ + ), ), - ), - ] - ) + ] + ) + + progress() title = f"recording-{recording.id}" fname_out = f"recording-{recording.id}.html" From 790a17e3e3f8aacec1dc7aacd77583917dd43738 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Mon, 26 Jun 2023 11:52:11 -0400 Subject: [PATCH 04/25] add a check for MAX_EVENT = None --- openadapt/visualize.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/openadapt/visualize.py b/openadapt/visualize.py index 358d1e3cc..e9a67d54f 100644 --- a/openadapt/visualize.py +++ b/openadapt/visualize.py @@ -190,7 +190,9 @@ def main(): ] logger.info(f"{len(action_events)=}") - with alive_bar(total=min(MAX_EVENTS, len(action_events)), title="Processing Action Events") as progress: + rows = [] + num_events = min(MAX_EVENTS, len(action_events)) if MAX_EVENTS is not None else len(action_events) + with alive_bar(total=num_events, title="Processing Action Events") as progress: for idx, action_event in enumerate(action_events): if idx == MAX_EVENTS: break From 37f22c09e005a67e0c7946abae0a62a2b896813b Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Mon, 26 Jun 2023 12:06:29 -0400 Subject: [PATCH 05/25] update the title for the Progress bAr (better for USer pov) --- openadapt/visualize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openadapt/visualize.py b/openadapt/visualize.py index e9a67d54f..dd2cbd4f3 100644 --- a/openadapt/visualize.py +++ b/openadapt/visualize.py @@ -192,7 +192,7 @@ def main(): rows = [] num_events = min(MAX_EVENTS, len(action_events)) if MAX_EVENTS is not None else len(action_events) - with alive_bar(total=num_events, title="Processing Action Events") as progress: + with alive_bar(total=num_events, title="Processing HTML for visualization:") as progress: for idx, action_event in enumerate(action_events): if idx == MAX_EVENTS: break From afbaa2c7747b73ea57667116ddf378a9ae02b7a0 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Mon, 26 Jun 2023 12:24:30 -0400 Subject: [PATCH 06/25] update the requirement.txt --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index ca5b3150e..06baf3932 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ alembic==1.8.1 +alive-progress==3.1.4 ascii_magic==2.3.0 black==23.3.0 atomacos @ git+https://github.com/abrichr/atomacos.git#egg=atomacos ; sys_platform == 'darwin' From 7227c5cf7f8da8868d1d3a663e7055ae9b164293 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Mon, 26 Jun 2023 12:27:07 -0400 Subject: [PATCH 07/25] ran ` black --line-length 80 ` on record.py and visualize.py --- openadapt/record.py | 17 ++++++++++++----- openadapt/visualize.py | 14 ++++++++++---- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/openadapt/record.py b/openadapt/record.py index 4c521b187..8c7d177f9 100644 --- a/openadapt/record.py +++ b/openadapt/record.py @@ -60,7 +60,9 @@ def wrapper_logging(*args, **kwargs): func_kwargs = kwargs_to_str(**kwargs) if func_kwargs != "": - logger.info(f" -> Enter: {func_name}({func_args}, {func_kwargs})") + logger.info( + f" -> Enter: {func_name}({func_args}, {func_kwargs})" + ) else: logger.info(f" -> Enter: {func_name}({func_args})") @@ -249,7 +251,9 @@ def write_events( logger.info(f"{event_type=} starting") signal.signal(signal.SIGINT, signal.SIG_IGN) - with alive_bar(total=write_q.qsize(), title=f"Writing {event_type} events") as progress: + with alive_bar( + total=write_q.qsize(), title=f"Writing {event_type} events" + ) as progress: while not terminate_event.is_set() or not write_q.empty(): try: event = write_q.get_nowait() @@ -352,7 +356,8 @@ def handle_key( "vk", ] attrs = { - f"key_{attr_name}": getattr(key, attr_name, None) for attr_name in attr_names + f"key_{attr_name}": getattr(key, attr_name, None) + for attr_name in attr_names } logger.debug(f"{attrs=}") canonical_attrs = { @@ -360,7 +365,9 @@ def handle_key( for attr_name in attr_names } logger.debug(f"{canonical_attrs=}") - trigger_action_event(event_q, {"name": event_name, **attrs, **canonical_attrs}) + trigger_action_event( + event_q, {"name": event_name, **attrs, **canonical_attrs} + ) def read_screen_events( @@ -438,7 +445,7 @@ def read_window_events( @trace(logger) -def performance_stats_writer ( +def performance_stats_writer( perf_q: multiprocessing.Queue, recording_timestamp: float, terminate_event: multiprocessing.Event, diff --git a/openadapt/visualize.py b/openadapt/visualize.py index dd2cbd4f3..d36e1b783 100644 --- a/openadapt/visualize.py +++ b/openadapt/visualize.py @@ -189,10 +189,16 @@ def main(): ), ] logger.info(f"{len(action_events)=}") - + rows = [] - num_events = min(MAX_EVENTS, len(action_events)) if MAX_EVENTS is not None else len(action_events) - with alive_bar(total=num_events, title="Processing HTML for visualization:") as progress: + num_events = ( + min(MAX_EVENTS, len(action_events)) + if MAX_EVENTS is not None + else len(action_events) + ) + with alive_bar( + total=num_events, title="Processing HTML for visualization:" + ) as progress: for idx, action_event in enumerate(action_events): if idx == MAX_EVENTS: break @@ -257,7 +263,7 @@ def main(): ), ] ) - + progress() title = f"recording-{recording.id}" From 0c5f4e9fc50a9c8f5b7d79dd67f749a693add721 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Mon, 26 Jun 2023 18:14:55 -0400 Subject: [PATCH 08/25] remove all progress bar from record --- openadapt/record.py | 22 ++++++++-------------- 1 file changed, 8 insertions(+), 14 deletions(-) diff --git a/openadapt/record.py b/openadapt/record.py index 8c7d177f9..d27f1c77c 100644 --- a/openadapt/record.py +++ b/openadapt/record.py @@ -250,20 +250,14 @@ def write_events( utils.set_start_time(recording_timestamp) logger.info(f"{event_type=} starting") signal.signal(signal.SIGINT, signal.SIG_IGN) - - with alive_bar( - total=write_q.qsize(), title=f"Writing {event_type} events" - ) as progress: - while not terminate_event.is_set() or not write_q.empty(): - try: - event = write_q.get_nowait() - except queue.Empty: - continue - assert event.type == event_type, (event_type, event) - write_fn(recording_timestamp, event, perf_q) - logger.debug(f"{event_type=} written") - progress() - + while not terminate_event.is_set() or not write_q.empty(): + try: + event = write_q.get_nowait() + except queue.Empty: + continue + assert event.type == event_type, (event_type, event) + write_fn(recording_timestamp, event, perf_q) + logger.debug(f"{event_type=} written") logger.info(f"{event_type=} done") From 9e8ee4d35ac7731ba7c9122007c622626fe9382c Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Tue, 27 Jun 2023 01:29:52 -0400 Subject: [PATCH 09/25] add tqdm progress bar in recrod.py --- openadapt/record.py | 46 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/openadapt/record.py b/openadapt/record.py index d27f1c77c..f738811c4 100644 --- a/openadapt/record.py +++ b/openadapt/record.py @@ -17,9 +17,9 @@ import threading import time -from alive_progress import alive_bar from loguru import logger from pynput import keyboard, mouse +from tqdm import tqdm import fire import mss.tools @@ -233,6 +233,7 @@ def write_events( perf_q: multiprocessing.Queue, recording_timestamp: float, terminate_event: multiprocessing.Event, + term_pipe: multiprocessing.Pipe, ): """ Write events of a specific type to the db using the provided write function. @@ -244,13 +245,36 @@ def write_events( perf_q: A queue for collecting performance data. recording_timestamp: The timestamp of the recording. terminate_event: An event to signal the termination of the process. + term_pipe: A pipe for communicating \ + the number of events left to be written. """ utils.configure_logging(logger, LOG_LEVEL) utils.set_start_time(recording_timestamp) logger.info(f"{event_type=} starting") signal.signal(signal.SIGINT, signal.SIG_IGN) - while not terminate_event.is_set() or not write_q.empty(): + + num_left = 0 + progress = None + while ( + not terminate_event.is_set() or + not write_q.empty() + ): + if term_pipe.poll(): + num_left = term_pipe.recv() + if num_left != 0 and progress is None: + progress = tqdm( + total=num_left, + desc="Writing to Database", + unit="event", colour="green", + dynamic_ncols=True, + ) + if ( + terminate_event.is_set() and + num_left != 0 and + progress is not None + ): + progress.update() try: event = write_q.get_nowait() except queue.Empty: @@ -258,6 +282,10 @@ def write_events( assert event.type == event_type, (event_type, event) write_fn(recording_timestamp, event, perf_q) logger.debug(f"{event_type=} written") + + if progress is not None: + progress.close() + logger.info(f"{event_type=} done") @@ -574,7 +602,11 @@ def record( # TODO: save write times to DB; display performance plot in visualize.py perf_q = multiprocessing.Queue() terminate_event = multiprocessing.Event() - + + term_pipe_parent_window, term_pipe_child_window = multiprocessing.Pipe() + term_pipe_parent_screen, term_pipe_child_screen = multiprocessing.Pipe() + term_pipe_parent_action, term_pipe_child_action = multiprocessing.Pipe() + window_event_reader = threading.Thread( target=read_window_events, args=(event_q, terminate_event, recording_timestamp), @@ -622,6 +654,7 @@ def record( perf_q, recording_timestamp, terminate_event, + term_pipe_child_screen, ), ) screen_event_writer.start() @@ -635,6 +668,7 @@ def record( perf_q, recording_timestamp, terminate_event, + term_pipe_child_action, ), ) action_event_writer.start() @@ -648,6 +682,7 @@ def record( perf_q, recording_timestamp, terminate_event, + term_pipe_child_window, ), ) window_event_writer.start() @@ -671,6 +706,10 @@ def record( except KeyboardInterrupt: terminate_event.set() + term_pipe_parent_window.send(window_write_q.qsize()) + term_pipe_parent_action.send(action_write_q.qsize()) + term_pipe_parent_screen.send(screen_write_q.qsize()) + logger.info(f"joining...") keyboard_event_reader.join() mouse_event_reader.join() @@ -680,7 +719,6 @@ def record( screen_event_writer.join() action_event_writer.join() window_event_writer.join() - terminate_perf_event.set() if PLOT_PERFORMANCE: From 3de913f3942d42ffc398734cbd16a2e18da1ae1d Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Tue, 27 Jun 2023 01:33:27 -0400 Subject: [PATCH 10/25] add tqdm for visualiztion --- openadapt/visualize.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/openadapt/visualize.py b/openadapt/visualize.py index d36e1b783..ae4dc589a 100644 --- a/openadapt/visualize.py +++ b/openadapt/visualize.py @@ -4,11 +4,11 @@ import os import string -from alive_progress import alive_bar from bokeh.io import output_file, show from bokeh.layouts import layout, row from bokeh.models.widgets import Div from loguru import logger +from tqdm import tqdm from openadapt.crud import ( get_latest_recording, @@ -196,8 +196,12 @@ def main(): if MAX_EVENTS is not None else len(action_events) ) - with alive_bar( - total=num_events, title="Processing HTML for visualization:" + with tqdm( + total=num_events, + desc="Processing HTML for Visualization", + colour="green", + dynamic_ncols=True, + ) as progress: for idx, action_event in enumerate(action_events): if idx == MAX_EVENTS: @@ -264,7 +268,7 @@ def main(): ] ) - progress() + progress.update() title = f"recording-{recording.id}" fname_out = f"recording-{recording.id}.html" From e5b1601448940521a92c8fc730acca0f4d3cb09c Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Tue, 27 Jun 2023 01:35:07 -0400 Subject: [PATCH 11/25] remove alive-progress --- poetry.lock | 41 +---------------------------------------- pyproject.toml | 1 - 2 files changed, 1 insertion(+), 41 deletions(-) diff --git a/poetry.lock b/poetry.lock index 7c4343aaa..1107652fa 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,16 +1,5 @@ # This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. -[[package]] -name = "about-time" -version = "4.2.1" -description = "Easily measure timing and throughput of code blocks, with beautiful human friendly representations." -optional = false -python-versions = ">=3.7, <4" -files = [ - {file = "about-time-4.2.1.tar.gz", hash = "sha256:6a538862d33ce67d997429d14998310e1dbfda6cb7d9bbfbf799c4709847fece"}, - {file = "about_time-4.2.1-py3-none-any.whl", hash = "sha256:8bbf4c75fe13cbd3d72f49a03b02c5c7dca32169b6d49117c257e7eb3eaee341"}, -] - [[package]] name = "aiohttp" version = "3.8.4" @@ -162,21 +151,6 @@ SQLAlchemy = ">=1.3.0" [package.extras] tz = ["python-dateutil"] -[[package]] -name = "alive-progress" -version = "3.1.4" -description = "A new kind of Progress Bar, with real-time throughput, ETA, and very cool animations!" -optional = false -python-versions = ">=3.7, <4" -files = [ - {file = "alive-progress-3.1.4.tar.gz", hash = "sha256:74a95d8d0d42bc99d3a3725dbd06ebb852245f1b64e301a7c375b92b22663f7b"}, - {file = "alive_progress-3.1.4-py3-none-any.whl", hash = "sha256:c80ad87ce9c1054b01135a87fae69ecebbfc2107497ae87cbe6aec7e534903db"}, -] - -[package.dependencies] -about-time = "4.2.1" -grapheme = "0.6.0" - [[package]] name = "altgraph" version = "0.17.3" @@ -1303,19 +1277,6 @@ files = [ [package.extras] speedup = ["python-levenshtein (>=0.12)"] -[[package]] -name = "grapheme" -version = "0.6.0" -description = "Unicode grapheme helpers" -optional = false -python-versions = "*" -files = [ - {file = "grapheme-0.6.0.tar.gz", hash = "sha256:44c2b9f21bbe77cfb05835fec230bd435954275267fea1858013b102f8603cca"}, -] - -[package.extras] -test = ["pytest", "sphinx", "sphinx-autobuild", "twine", "wheel"] - [[package]] name = "greenlet" version = "2.0.2" @@ -5958,4 +5919,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "3.10.x" -content-hash = "4e17975198b5cdac01ab69ce731c2db4979584a5580b42bc7748f081b5ebf64d" +content-hash = "dee9d1c731617db5df7919ad57d2126eccd9ce3fd7a6ca0688adaeb93086daec" diff --git a/pyproject.toml b/pyproject.toml index b7c11acea..97192f6e0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -75,7 +75,6 @@ nltk = "3.8.1" pywinauto = {version = "^0.6.8", markers = "sys_platform == 'win32'"} moviepy = "1.0.3" python-levenshtein = "^0.21.1" -alive-progress = "^3.1.4" [tool.poetry.dependencies.en_core_web_trf] url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_trf-3.5.0/en_core_web_trf-3.5.0.tar.gz" From f7c42d1442a192480536c3e1a705b69279f96e08 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Tue, 27 Jun 2023 01:54:16 -0400 Subject: [PATCH 12/25] consistent tqdm api --add dynamic_cols: to enable adjustments when window is resized Order: --total -description --unit --Optional[bar_format] --colour --dynamic_ncols --- openadapt/record.py | 3 ++- openadapt/scripts/scrub.py | 1 + openadapt/visualize.py | 12 ++++++------ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/openadapt/record.py b/openadapt/record.py index f738811c4..2582eec97 100644 --- a/openadapt/record.py +++ b/openadapt/record.py @@ -266,7 +266,8 @@ def write_events( progress = tqdm( total=num_left, desc="Writing to Database", - unit="event", colour="green", + unit="event", + colour="green", dynamic_ncols=True, ) if ( diff --git a/openadapt/scripts/scrub.py b/openadapt/scripts/scrub.py index 486f28725..b4058f28c 100644 --- a/openadapt/scripts/scrub.py +++ b/openadapt/scripts/scrub.py @@ -113,6 +113,7 @@ def scrub_mp4( unit="frame", bar_format=progress_bar_format, colour="green", + dynamic_ncols=True, ) progress_interval = 0.1 # Print progress every 10% of frames progress_threshold = math.floor(frame_count * progress_interval) diff --git a/openadapt/visualize.py b/openadapt/visualize.py index ae4dc589a..353aad37d 100644 --- a/openadapt/visualize.py +++ b/openadapt/visualize.py @@ -197,12 +197,12 @@ def main(): else len(action_events) ) with tqdm( - total=num_events, - desc="Processing HTML for Visualization", - colour="green", - dynamic_ncols=True, - - ) as progress: + total=num_left, + desc="Writing to Database", + unit="event", + colour="green", + dynamic_ncols=True, + ) as progress: for idx, action_event in enumerate(action_events): if idx == MAX_EVENTS: break From 03dc2237d5dd7fe51ed35db631bd8cde8b908f23 Mon Sep 17 00:00:00 2001 From: Krish Patel <65433817+KrishPatel13@users.noreply.github.com> Date: Wed, 28 Jun 2023 10:30:06 -0400 Subject: [PATCH 13/25] Update requirements.txt Co-authored-by: Aaron <57018940+0dm@users.noreply.github.com> --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 06baf3932..ca5b3150e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,4 @@ alembic==1.8.1 -alive-progress==3.1.4 ascii_magic==2.3.0 black==23.3.0 atomacos @ git+https://github.com/abrichr/atomacos.git#egg=atomacos ; sys_platform == 'darwin' From 2c26a459e8103ea64580dce2ce7ef5ca544eee34 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Wed, 28 Jun 2023 10:35:35 -0400 Subject: [PATCH 14/25] Address comemnt: https://github.com/MLDSAI/OpenAdapt/pull/318#discussion_r1244598202 --- openadapt/visualize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openadapt/visualize.py b/openadapt/visualize.py index 353aad37d..b6ff510d6 100644 --- a/openadapt/visualize.py +++ b/openadapt/visualize.py @@ -197,7 +197,7 @@ def main(): else len(action_events) ) with tqdm( - total=num_left, + total=num_events, desc="Writing to Database", unit="event", colour="green", From d41d088a292859e646171c70782984c06d8b515a Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Wed, 28 Jun 2023 10:41:30 -0400 Subject: [PATCH 15/25] remove incorrect indent --- openadapt/visualize.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/openadapt/visualize.py b/openadapt/visualize.py index b6ff510d6..c5f95852e 100644 --- a/openadapt/visualize.py +++ b/openadapt/visualize.py @@ -270,23 +270,23 @@ def main(): progress.update() - title = f"recording-{recording.id}" - fname_out = f"recording-{recording.id}.html" - logger.info(f"{fname_out=}") - output_file(fname_out, title=title) - - result = show( - layout( - rows, + title = f"recording-{recording.id}" + fname_out = f"recording-{recording.id}.html" + logger.info(f"{fname_out=}") + output_file(fname_out, title=title) + + result = show( + layout( + rows, + ) ) - ) - def cleanup(): - os.remove(fname_out) - removed = not os.path.exists(fname_out) - logger.info(f"{removed=}") + def cleanup(): + os.remove(fname_out) + removed = not os.path.exists(fname_out) + logger.info(f"{removed=}") - Timer(1, cleanup).start() + Timer(1, cleanup).start() if __name__ == "__main__": From 2234eb998f4938fd3a284f906d8102e4b9673988 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Wed, 28 Jun 2023 10:43:42 -0400 Subject: [PATCH 16/25] remove rows --- openadapt/visualize.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openadapt/visualize.py b/openadapt/visualize.py index c5f95852e..13150e045 100644 --- a/openadapt/visualize.py +++ b/openadapt/visualize.py @@ -190,7 +190,6 @@ def main(): ] logger.info(f"{len(action_events)=}") - rows = [] num_events = ( min(MAX_EVENTS, len(action_events)) if MAX_EVENTS is not None From 8f6d02b3a621add41ebef1de6d4f3c52b9c1d3bc Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Wed, 28 Jun 2023 11:13:58 -0400 Subject: [PATCH 17/25] try to fix distorted table in html --- openadapt/visualize.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/openadapt/visualize.py b/openadapt/visualize.py index 13150e045..9a2e96f3d 100644 --- a/openadapt/visualize.py +++ b/openadapt/visualize.py @@ -197,7 +197,7 @@ def main(): ) with tqdm( total=num_events, - desc="Writing to Database", + desc="Preparing HTML", unit="event", colour="green", dynamic_ncols=True, @@ -269,23 +269,25 @@ def main(): progress.update() - title = f"recording-{recording.id}" - fname_out = f"recording-{recording.id}.html" - logger.info(f"{fname_out=}") - output_file(fname_out, title=title) + progress.close() - result = show( - layout( - rows, - ) + title = f"recording-{recording.id}" + fname_out = f"recording-{recording.id}.html" + logger.info(f"{fname_out=}") + output_file(fname_out, title=title) + + result = show( + layout( + rows, ) + ) - def cleanup(): - os.remove(fname_out) - removed = not os.path.exists(fname_out) - logger.info(f"{removed=}") + def cleanup(): + os.remove(fname_out) + removed = not os.path.exists(fname_out) + logger.info(f"{removed=}") - Timer(1, cleanup).start() + Timer(1, cleanup).start() if __name__ == "__main__": From 9c063ef14768d59eab2f73a3dc05a6478bdb8d9d Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Wed, 28 Jun 2023 20:02:52 -0400 Subject: [PATCH 18/25] add custom queue class --- openadapt/custom/my_queue.py | 92 ++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 openadapt/custom/my_queue.py diff --git a/openadapt/custom/my_queue.py b/openadapt/custom/my_queue.py new file mode 100644 index 000000000..df2571a1f --- /dev/null +++ b/openadapt/custom/my_queue.py @@ -0,0 +1,92 @@ +from multiprocessing.queues import Queue +import multiprocessing + +# The following implementation of custom MyQueue to avoid NotImplementedError +# when calling queue.qsize() in MacOS X comes almost entirely from this github +# discussion: https://github.com/keras-team/autokeras/issues/368 +# Necessary modification is made to make the code compatible with Python3. + + +class SharedCounter(object): + """ A synchronized shared counter. + The locking done by multiprocessing.Value ensures that only a single + process or thread may read or write the in-memory ctypes object. However, + in order to do n += 1, Python performs a read followed by a write, so a + second process may read the old value before the new one is written by the + first process. The solution is to use a multiprocessing.Lock to guarantee + the atomicity of the modifications to Value. + This class comes almost entirely from Eli Bendersky's blog: + http://eli.thegreenplace.net/2012/01/04/ + shared-counter-with-pythons-multiprocessing/ + """ + + def __init__(self, n=0): + self.count = multiprocessing.Value('i', n) + + def increment(self, n=1): + """ Increment the counter by n (default = 1) """ + with self.count.get_lock(): + self.count.value += n + + @property + def value(self): + """ Return the value of the counter """ + return self.count.value + + +class MyQueue(Queue): + """ A portable implementation of multiprocessing.Queue. + Because of multithreading / multiprocessing semantics, Queue.qsize() may + raise the NotImplementedError exception on Unix platforms like Mac OS X + where sem_getvalue() is not implemented. This subclass addresses this + problem by using a synchronized shared counter (initialized to zero) and + increasing / decreasing its value every time the put() and get() methods + are called, respectively. This not only prevents NotImplementedError from + being raised, but also allows us to implement a reliable version of both + qsize() and empty(). + Note the implementation of __getstate__ and __setstate__ which help to + serialize MyQueue when it is passed between processes. If these functions + are not defined, MyQueue cannot be serialized, which will lead to the error + of "AttributeError: 'MyQueue' object has no attribute 'size'". + See the answer provided here: https://stackoverflow.com/a/65513291/9723036 + + For documentation of using __getstate__ and __setstate__ + to serialize objects, refer to here: + https://docs.python.org/3/library/pickle.html#pickling-class-instances + """ + + def __init__(self): + super().__init__(ctx=multiprocessing.get_context()) + self.size = SharedCounter(0) + + def __getstate__(self): + """Help to make MyQueue instance serializable. + Note that we record the parent class state, which is the state of the + actual queue, and the size of the queue, which is the state of MyQueue. + self.size is a SharedCounter instance. It is itself serializable. + """ + return { + 'parent_state': super().__getstate__(), + 'size': self.size, + } + + def __setstate__(self, state): + super().__setstate__(state['parent_state']) + self.size = state['size'] + + def put(self, *args, **kwargs): + super().put(*args, **kwargs) + self.size.increment(1) + + def get(self, *args, **kwargs): + item = super().get(*args, **kwargs) + self.size.increment(-1) + return item + + def qsize(self): + """ Reliable implementation of multiprocessing.Queue.qsize() """ + return self.size.value + + def empty(self): + """ Reliable implementation of multiprocessing.Queue.empty() """ + return not self.qsize() \ No newline at end of file From 857382ac6d87fa3040dd67a085c0df53202877b3 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Wed, 28 Jun 2023 20:37:32 -0400 Subject: [PATCH 19/25] lint --line-length 80 --- openadapt/custom/my_queue.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openadapt/custom/my_queue.py b/openadapt/custom/my_queue.py index df2571a1f..2d7abed71 100644 --- a/openadapt/custom/my_queue.py +++ b/openadapt/custom/my_queue.py @@ -89,4 +89,4 @@ def qsize(self): def empty(self): """ Reliable implementation of multiprocessing.Queue.empty() """ - return not self.qsize() \ No newline at end of file + return not self.qsize() From c6fbf3a1b84081f25822e2f44269498a16819cff Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Wed, 28 Jun 2023 22:37:39 -0400 Subject: [PATCH 20/25] fix `NotImplementedError` for MacOs -- using custom MyQueue class --- openadapt/record.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/openadapt/record.py b/openadapt/record.py index 2582eec97..290e69d22 100644 --- a/openadapt/record.py +++ b/openadapt/record.py @@ -24,6 +24,7 @@ import mss.tools from openadapt import config, crud, utils, window +from openadapt.custom.my_queue import MyQueue import functools @@ -86,10 +87,10 @@ def process_event(event, write_q, write_fn, recording_timestamp, perf_q): @trace(logger) def process_events( event_q: queue.Queue, - screen_write_q: multiprocessing.Queue, - action_write_q: multiprocessing.Queue, - window_write_q: multiprocessing.Queue, - perf_q: multiprocessing.Queue, + screen_write_q: MyQueue, + action_write_q: MyQueue, + window_write_q: MyQueue, + perf_q: MyQueue, recording_timestamp: float, terminate_event: multiprocessing.Event, ): @@ -168,7 +169,7 @@ def process_events( def write_action_event( recording_timestamp: float, event: Event, - perf_q: multiprocessing.Queue, + perf_q: MyQueue, ): """ Write an action event to the database and update the performance queue. @@ -187,7 +188,7 @@ def write_action_event( def write_screen_event( recording_timestamp: float, event: Event, - perf_q: multiprocessing.Queue, + perf_q: MyQueue, ): """ Write a screen event to the database and update the performance queue. @@ -209,7 +210,7 @@ def write_screen_event( def write_window_event( recording_timestamp: float, event: Event, - perf_q: multiprocessing.Queue, + perf_q: MyQueue, ): """ Write a window event to the database and update the performance queue. @@ -229,8 +230,8 @@ def write_window_event( def write_events( event_type: str, write_fn: Callable, - write_q: multiprocessing.Queue, - perf_q: multiprocessing.Queue, + write_q: MyQueue, + perf_q: MyQueue, recording_timestamp: float, terminate_event: multiprocessing.Event, term_pipe: multiprocessing.Pipe, @@ -469,7 +470,7 @@ def read_window_events( @trace(logger) def performance_stats_writer( - perf_q: multiprocessing.Queue, + perf_q: MyQueue, recording_timestamp: float, terminate_event: multiprocessing.Event, ): @@ -597,11 +598,11 @@ def record( recording_timestamp = recording.timestamp event_q = queue.Queue() - screen_write_q = multiprocessing.Queue() - action_write_q = multiprocessing.Queue() - window_write_q = multiprocessing.Queue() + screen_write_q = MyQueue() + action_write_q = MyQueue() + window_write_q = MyQueue() # TODO: save write times to DB; display performance plot in visualize.py - perf_q = multiprocessing.Queue() + perf_q = MyQueue() terminate_event = multiprocessing.Event() term_pipe_parent_window, term_pipe_child_window = multiprocessing.Pipe() From 416ce80190bf9c25891be2d71a4d47a2f8284cb1 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Wed, 28 Jun 2023 22:43:35 -0400 Subject: [PATCH 21/25] rename custom -> thirdparty_customization --- openadapt/{custom => thirdparty_customization}/my_queue.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename openadapt/{custom => thirdparty_customization}/my_queue.py (100%) diff --git a/openadapt/custom/my_queue.py b/openadapt/thirdparty_customization/my_queue.py similarity index 100% rename from openadapt/custom/my_queue.py rename to openadapt/thirdparty_customization/my_queue.py From 00fdcc3f6e372eb82dbbf8991934cea4229554a2 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Wed, 28 Jun 2023 22:45:49 -0400 Subject: [PATCH 22/25] rename to something useful --- openadapt/record.py | 30 +++++++++---------- .../thirdparty_customization/my_queue.py | 14 ++++----- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/openadapt/record.py b/openadapt/record.py index 290e69d22..ba5a8f531 100644 --- a/openadapt/record.py +++ b/openadapt/record.py @@ -24,7 +24,7 @@ import mss.tools from openadapt import config, crud, utils, window -from openadapt.custom.my_queue import MyQueue +from openadapt.thirdparty_customization.my_queue import SynchronizedQueue import functools @@ -87,10 +87,10 @@ def process_event(event, write_q, write_fn, recording_timestamp, perf_q): @trace(logger) def process_events( event_q: queue.Queue, - screen_write_q: MyQueue, - action_write_q: MyQueue, - window_write_q: MyQueue, - perf_q: MyQueue, + screen_write_q: SynchronizedQueue, + action_write_q: SynchronizedQueue, + window_write_q: SynchronizedQueue, + perf_q: SynchronizedQueue, recording_timestamp: float, terminate_event: multiprocessing.Event, ): @@ -169,7 +169,7 @@ def process_events( def write_action_event( recording_timestamp: float, event: Event, - perf_q: MyQueue, + perf_q: SynchronizedQueue, ): """ Write an action event to the database and update the performance queue. @@ -188,7 +188,7 @@ def write_action_event( def write_screen_event( recording_timestamp: float, event: Event, - perf_q: MyQueue, + perf_q: SynchronizedQueue, ): """ Write a screen event to the database and update the performance queue. @@ -210,7 +210,7 @@ def write_screen_event( def write_window_event( recording_timestamp: float, event: Event, - perf_q: MyQueue, + perf_q: SynchronizedQueue, ): """ Write a window event to the database and update the performance queue. @@ -230,8 +230,8 @@ def write_window_event( def write_events( event_type: str, write_fn: Callable, - write_q: MyQueue, - perf_q: MyQueue, + write_q: SynchronizedQueue, + perf_q: SynchronizedQueue, recording_timestamp: float, terminate_event: multiprocessing.Event, term_pipe: multiprocessing.Pipe, @@ -470,7 +470,7 @@ def read_window_events( @trace(logger) def performance_stats_writer( - perf_q: MyQueue, + perf_q: SynchronizedQueue, recording_timestamp: float, terminate_event: multiprocessing.Event, ): @@ -598,11 +598,11 @@ def record( recording_timestamp = recording.timestamp event_q = queue.Queue() - screen_write_q = MyQueue() - action_write_q = MyQueue() - window_write_q = MyQueue() + screen_write_q = SynchronizedQueue() + action_write_q = SynchronizedQueue() + window_write_q = SynchronizedQueue() # TODO: save write times to DB; display performance plot in visualize.py - perf_q = MyQueue() + perf_q = SynchronizedQueue() terminate_event = multiprocessing.Event() term_pipe_parent_window, term_pipe_child_window = multiprocessing.Pipe() diff --git a/openadapt/thirdparty_customization/my_queue.py b/openadapt/thirdparty_customization/my_queue.py index 2d7abed71..70aa6db35 100644 --- a/openadapt/thirdparty_customization/my_queue.py +++ b/openadapt/thirdparty_customization/my_queue.py @@ -1,7 +1,7 @@ from multiprocessing.queues import Queue import multiprocessing -# The following implementation of custom MyQueue to avoid NotImplementedError +# The following implementation of custom SynchronizedQueue to avoid NotImplementedError # when calling queue.qsize() in MacOS X comes almost entirely from this github # discussion: https://github.com/keras-team/autokeras/issues/368 # Necessary modification is made to make the code compatible with Python3. @@ -34,7 +34,7 @@ def value(self): return self.count.value -class MyQueue(Queue): +class SynchronizedQueue(Queue): """ A portable implementation of multiprocessing.Queue. Because of multithreading / multiprocessing semantics, Queue.qsize() may raise the NotImplementedError exception on Unix platforms like Mac OS X @@ -45,9 +45,9 @@ class MyQueue(Queue): being raised, but also allows us to implement a reliable version of both qsize() and empty(). Note the implementation of __getstate__ and __setstate__ which help to - serialize MyQueue when it is passed between processes. If these functions - are not defined, MyQueue cannot be serialized, which will lead to the error - of "AttributeError: 'MyQueue' object has no attribute 'size'". + serialize SynchronizedQueue when it is passed between processes. If these functions + are not defined, SynchronizedQueue cannot be serialized, which will lead to the error + of "AttributeError: 'SynchronizedQueue' object has no attribute 'size'". See the answer provided here: https://stackoverflow.com/a/65513291/9723036 For documentation of using __getstate__ and __setstate__ @@ -60,9 +60,9 @@ def __init__(self): self.size = SharedCounter(0) def __getstate__(self): - """Help to make MyQueue instance serializable. + """Help to make SynchronizedQueue instance serializable. Note that we record the parent class state, which is the state of the - actual queue, and the size of the queue, which is the state of MyQueue. + actual queue, and the size of the queue, which is the state of SynchronizedQueue. self.size is a SharedCounter instance. It is itself serializable. """ return { From a612535aea4f281c431f94f555dadfecfdf3b941 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Wed, 28 Jun 2023 23:56:20 -0400 Subject: [PATCH 23/25] address comments --- openadapt/record.py | 30 +++++++++---------- .../{my_queue.py => synchronized_queue.py} | 7 +++++ 2 files changed, 22 insertions(+), 15 deletions(-) rename openadapt/thirdparty_customization/{my_queue.py => synchronized_queue.py} (95%) diff --git a/openadapt/record.py b/openadapt/record.py index ba5a8f531..080984c04 100644 --- a/openadapt/record.py +++ b/openadapt/record.py @@ -24,7 +24,7 @@ import mss.tools from openadapt import config, crud, utils, window -from openadapt.thirdparty_customization.my_queue import SynchronizedQueue +from openadapt.thirdparty_customization import synchronized_queue as sq import functools @@ -87,10 +87,10 @@ def process_event(event, write_q, write_fn, recording_timestamp, perf_q): @trace(logger) def process_events( event_q: queue.Queue, - screen_write_q: SynchronizedQueue, - action_write_q: SynchronizedQueue, - window_write_q: SynchronizedQueue, - perf_q: SynchronizedQueue, + screen_write_q: sq.SynchronizedQueue, + action_write_q: sq.SynchronizedQueue, + window_write_q: sq.SynchronizedQueue, + perf_q: sq.SynchronizedQueue, recording_timestamp: float, terminate_event: multiprocessing.Event, ): @@ -169,7 +169,7 @@ def process_events( def write_action_event( recording_timestamp: float, event: Event, - perf_q: SynchronizedQueue, + perf_q: sq.SynchronizedQueue, ): """ Write an action event to the database and update the performance queue. @@ -188,7 +188,7 @@ def write_action_event( def write_screen_event( recording_timestamp: float, event: Event, - perf_q: SynchronizedQueue, + perf_q: sq.SynchronizedQueue, ): """ Write a screen event to the database and update the performance queue. @@ -210,7 +210,7 @@ def write_screen_event( def write_window_event( recording_timestamp: float, event: Event, - perf_q: SynchronizedQueue, + perf_q: sq.SynchronizedQueue, ): """ Write a window event to the database and update the performance queue. @@ -230,8 +230,8 @@ def write_window_event( def write_events( event_type: str, write_fn: Callable, - write_q: SynchronizedQueue, - perf_q: SynchronizedQueue, + write_q: sq.SynchronizedQueue, + perf_q: sq.SynchronizedQueue, recording_timestamp: float, terminate_event: multiprocessing.Event, term_pipe: multiprocessing.Pipe, @@ -470,7 +470,7 @@ def read_window_events( @trace(logger) def performance_stats_writer( - perf_q: SynchronizedQueue, + perf_q: sq.SynchronizedQueue, recording_timestamp: float, terminate_event: multiprocessing.Event, ): @@ -598,11 +598,11 @@ def record( recording_timestamp = recording.timestamp event_q = queue.Queue() - screen_write_q = SynchronizedQueue() - action_write_q = SynchronizedQueue() - window_write_q = SynchronizedQueue() + screen_write_q = sq.SynchronizedQueue() + action_write_q = sq.SynchronizedQueue() + window_write_q = sq.SynchronizedQueue() # TODO: save write times to DB; display performance plot in visualize.py - perf_q = SynchronizedQueue() + perf_q = sq.SynchronizedQueue() terminate_event = multiprocessing.Event() term_pipe_parent_window, term_pipe_child_window = multiprocessing.Pipe() diff --git a/openadapt/thirdparty_customization/my_queue.py b/openadapt/thirdparty_customization/synchronized_queue.py similarity index 95% rename from openadapt/thirdparty_customization/my_queue.py rename to openadapt/thirdparty_customization/synchronized_queue.py index 70aa6db35..0c5e0da66 100644 --- a/openadapt/thirdparty_customization/my_queue.py +++ b/openadapt/thirdparty_customization/synchronized_queue.py @@ -1,6 +1,13 @@ +""" + Module for customizing multiprocessing.Queue to avoid NotImplementedError in MacOS +""" + + from multiprocessing.queues import Queue import multiprocessing +# Credit: https://gist.github.com/FanchenBao/d8577599c46eab1238a81857bb7277c9 + # The following implementation of custom SynchronizedQueue to avoid NotImplementedError # when calling queue.qsize() in MacOS X comes almost entirely from this github # discussion: https://github.com/keras-team/autokeras/issues/368 From 520414819379b8bf3946754307e976abfa822d9e Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Thu, 29 Jun 2023 00:02:02 -0400 Subject: [PATCH 24/25] rename dir to customized_imports --- .../synchronized_queue.py | 0 openadapt/record.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename openadapt/{thirdparty_customization => customized_imports}/synchronized_queue.py (100%) diff --git a/openadapt/thirdparty_customization/synchronized_queue.py b/openadapt/customized_imports/synchronized_queue.py similarity index 100% rename from openadapt/thirdparty_customization/synchronized_queue.py rename to openadapt/customized_imports/synchronized_queue.py diff --git a/openadapt/record.py b/openadapt/record.py index 080984c04..541ce79d3 100644 --- a/openadapt/record.py +++ b/openadapt/record.py @@ -24,7 +24,7 @@ import mss.tools from openadapt import config, crud, utils, window -from openadapt.thirdparty_customization import synchronized_queue as sq +from openadapt.customized_imports import synchronized_queue as sq import functools From e2419cc6e60910e0968d90cce75c2c47149f9b08 Mon Sep 17 00:00:00 2001 From: Krish Patel Date: Thu, 29 Jun 2023 00:09:19 -0400 Subject: [PATCH 25/25] rename to extensions https://github.com/MLDSAI/OpenAdapt/pull/318#discussion_r1246073970 --- .../{customized_imports => extensions}/synchronized_queue.py | 0 openadapt/record.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename openadapt/{customized_imports => extensions}/synchronized_queue.py (100%) diff --git a/openadapt/customized_imports/synchronized_queue.py b/openadapt/extensions/synchronized_queue.py similarity index 100% rename from openadapt/customized_imports/synchronized_queue.py rename to openadapt/extensions/synchronized_queue.py diff --git a/openadapt/record.py b/openadapt/record.py index 541ce79d3..4e6bf5ece 100644 --- a/openadapt/record.py +++ b/openadapt/record.py @@ -24,7 +24,7 @@ import mss.tools from openadapt import config, crud, utils, window -from openadapt.customized_imports import synchronized_queue as sq +from openadapt.extensions import synchronized_queue as sq import functools