From a79ae70eeccf1ab8bdd28370cd28f9546bd4f657 Mon Sep 17 00:00:00 2001 From: "Maarten A. Breddels" Date: Wed, 27 May 2020 11:30:34 +0200 Subject: [PATCH] fix: always run a coroutine and work with tornado --- binder/run_nbclient.ipynb | 2 +- docs/changelog.md | 1 + nbclient/client.py | 15 --------- nbclient/tests/util.py | 57 +++++++++++++++++++++++++++++++++ nbclient/util.py | 66 +++++++++++++++++++++++++++------------ 5 files changed, 105 insertions(+), 36 deletions(-) create mode 100644 nbclient/tests/util.py diff --git a/binder/run_nbclient.ipynb b/binder/run_nbclient.ipynb index 9f6f0584..d8994b48 100644 --- a/binder/run_nbclient.ipynb +++ b/binder/run_nbclient.ipynb @@ -46,7 +46,7 @@ "nb = nbf.read('./empty_notebook.ipynb', nbf.NO_CONVERT)\n", "\n", "# Execute our in-memory notebook, which will now have outputs\n", - "nb = nbclient.execute(nb, nest_asyncio=True)" + "nb = nbclient.execute(nb)" ] }, { diff --git a/docs/changelog.md b/docs/changelog.md index 875159bb..8ef633c4 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -5,6 +5,7 @@ ### Major Changes - Mimic an Output widget at the frontend so that the Output widget behaves correctly [#68](https://github.com/jupyter/nbclient/pull/68) +- Nested asyncio is automatic, and works with Tornado [#71](https://github.com/jupyter/nbclient/pull/71) ## 0.3.1 diff --git a/nbclient/client.py b/nbclient/client.py index 304d701a..6d2caf9b 100644 --- a/nbclient/client.py +++ b/nbclient/client.py @@ -103,21 +103,6 @@ class NotebookClient(LoggingConfigurable): ), ).tag(config=True) - nest_asyncio = Bool( - False, - help=dedent( - """ - If False (default), then blocking functions such as `execute` - assume that no event loop is already running. These functions - run their async counterparts (e.g. `async_execute`) in an event - loop with `asyncio.run_until_complete`, which will fail if an - event loop is already running. This can be the case if nbclient - is used e.g. in a Jupyter Notebook. In that case, `nest_asyncio` - should be set to True. - """ - ), - ).tag(config=True) - force_raise_errors = Bool( False, help=dedent( diff --git a/nbclient/tests/util.py b/nbclient/tests/util.py new file mode 100644 index 00000000..2e864b85 --- /dev/null +++ b/nbclient/tests/util.py @@ -0,0 +1,57 @@ +import asyncio + +import tornado + +from nbclient.util import run_sync + + +@run_sync +async def some_async_function(): + await asyncio.sleep(0.01) + return 42 + + +def test_nested_asyncio_with_existing_ioloop(): + ioloop = asyncio.new_event_loop() + try: + asyncio.set_event_loop(ioloop) + assert some_async_function() == 42 + assert asyncio.get_event_loop() is ioloop + finally: + asyncio._set_running_loop(None) # it seems nest_asyncio doesn't reset this + + +def test_nested_asyncio_with_no_ioloop(): + asyncio.set_event_loop(None) + try: + assert some_async_function() == 42 + finally: + asyncio._set_running_loop(None) # it seems nest_asyncio doesn't reset this + + +def test_nested_asyncio_with_tornado(): + # This tests if tornado accepts the pure-Python Futures, see + # https://github.com/tornadoweb/tornado/issues/2753 + # https://github.com/erdewit/nest_asyncio/issues/23 + asyncio.set_event_loop(asyncio.new_event_loop()) + ioloop = tornado.ioloop.IOLoop.current() + + async def some_async_function(): + future = asyncio.ensure_future(asyncio.sleep(0.1)) + # this future is a different future after nested-asyncio has patched + # the asyncio module, check if tornado likes it: + ioloop.add_future(future, lambda f: f.result()) + await future + return 42 + + def some_sync_function(): + return run_sync(some_async_function)() + + async def run(): + # calling some_async_function directly should work + assert await some_async_function() == 42 + # but via a sync function (using nested-asyncio) can lead to issues: + # https://github.com/tornadoweb/tornado/issues/2753 + assert some_sync_function() == 42 + + ioloop.run_sync(run) diff --git a/nbclient/util.py b/nbclient/util.py index a57b6e5e..eeaa6199 100644 --- a/nbclient/util.py +++ b/nbclient/util.py @@ -4,9 +4,53 @@ # Distributed under the terms of the Modified BSD License. import asyncio +import sys import inspect +def check_ipython(): + # original from vaex/asyncio.py + IPython = sys.modules.get('IPython') + if IPython: + IPython_version = tuple(map(int, IPython.__version__.split('.'))) + if IPython_version < (7, 0, 0): + raise RuntimeError(f'You are using IPython {IPython.__version__} while we require' + '7.0.0+, please update IPython') + + +def check_patch_tornado(): + """If tornado is imported, add the patched asyncio.Future to its tuple of acceptable Futures""" + # original from vaex/asyncio.py + if 'tornado' in sys.modules: + import tornado.concurrent + if asyncio.Future not in tornado.concurrent.FUTURES: + tornado.concurrent.FUTURES = tornado.concurrent.FUTURES + (asyncio.Future, ) + + +def just_run(coro): + """Make the coroutine run, even if there is an event loop running (using nest_asyncio)""" + # original from vaex/asyncio.py + loop = asyncio._get_running_loop() + if loop is None: + had_running_loop = False + try: + loop = asyncio.get_event_loop() + except RuntimeError: + # we can still get 'There is no current event loop in ...' + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + else: + had_running_loop = True + if had_running_loop: + # if there is a running loop, we patch using nest_asyncio + # to have reentrant event loops + check_ipython() + import nest_asyncio + nest_asyncio.apply() + check_patch_tornado() + return loop.run_until_complete(coro) + + def run_sync(coro): """Runs a coroutine and blocks until it has executed. @@ -24,26 +68,8 @@ def run_sync(coro): result : Whatever the coroutine returns. """ - def wrapped(self, *args, **kwargs): - try: - loop = asyncio.get_event_loop() - except RuntimeError: - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - if self.nest_asyncio: - import nest_asyncio - nest_asyncio.apply(loop) - try: - result = loop.run_until_complete(coro(self, *args, **kwargs)) - except RuntimeError as e: - if str(e) == 'This event loop is already running': - raise RuntimeError( - 'You are trying to run nbclient in an environment where an ' - 'event loop is already running. Please pass `nest_asyncio=True` in ' - '`NotebookClient.execute` and such methods.' - ) from e - raise - return result + def wrapped(*args, **kwargs): + return just_run(coro(*args, **kwargs)) wrapped.__doc__ = coro.__doc__ return wrapped