diff --git a/ibis/backends/__init__.py b/ibis/backends/__init__.py index bbbcb36d9bab3..4ec58a23f6de7 100644 --- a/ibis/backends/__init__.py +++ b/ibis/backends/__init__.py @@ -7,7 +7,6 @@ import keyword import re import urllib.parse -import weakref from pathlib import Path from typing import TYPE_CHECKING, Any, ClassVar @@ -35,12 +34,6 @@ class TablesAccessor(collections.abc.Mapping): """A mapping-like object for accessing tables off a backend. - ::: {.callout-note} - ## The `tables` accessor is tied to the lifetime of the backend. - - If the backend goes out of scope, the `tables` accessor is no longer valid. - ::: - Tables may be accessed by name using either index or attribute access: Examples @@ -53,6 +46,42 @@ class TablesAccessor(collections.abc.Mapping): def __init__(self, backend: BaseBackend) -> None: self._backend = backend + def _execute_if_exists( + self, method_name: str, database=None, like=None + ) -> list[str]: + """Executes method if it exists and it doesn't raise a NotImplementedError, else returns an empty list.""" + method = getattr(self._backend.ddl, method_name) + if callable(method): + try: + return method(database=database, like=like) + except NotImplementedError: + pass + return [] + + def _gather_tables(self, database=None, like=None) -> list[str]: + """Gathers table names using the list_* methods available on the backend.""" + # TODO: break this down into views/tables to be more explicit in repr (see #9859) + # list_* methods that might exist on a given backends. + list_methods = [ + "list_tables", + "list_temp_tables", + "list_views", + "list_temp_views", + ] + tables = [] + for method_name in list_methods: + tables.extend( + self._execute_if_exists(method_name, database=database, like=like) + ) + return list(set(tables)) + + def __call__(self, database=None, like=None): + return self._gather_tables(database, like) + + @property + def _tables(self) -> list[str]: + return self._gather_tables() + def __getitem__(self, name) -> ir.Table: try: return self._backend.table(name) @@ -68,29 +97,70 @@ def __getattr__(self, name) -> ir.Table: raise AttributeError(name) from exc def __iter__(self) -> Iterator[str]: - return iter(sorted(self._backend.list_tables())) + return iter(sorted(self._tables)) def __len__(self) -> int: - return len(self._backend.list_tables()) + return len(self._tables) def __dir__(self) -> list[str]: o = set() o.update(dir(type(self))) o.update( name - for name in self._backend.list_tables() + for name in self._tables if name.isidentifier() and not keyword.iskeyword(name) ) return list(o) def __repr__(self) -> str: - tables = self._backend.list_tables() rows = ["Tables", "------"] - rows.extend(f"- {name}" for name in sorted(tables)) + rows.extend(f"- {name}" for name in sorted(self._tables)) return "\n".join(rows) def _ipython_key_completions_(self) -> list[str]: - return self._backend.list_tables() + return self._tables + + +class DDLAccessor: + """ddl accessor list views.""" + + def __init__(self, backend: BaseBackend) -> None: + self._backend = backend + + def _raise_if_not_implemented(self, method_name: str): + method = getattr(self._backend, method_name) + if not callable(method): + raise NotImplementedError( + f"The method {method_name} is not implemented for the {self._backend.name} backend" + ) + + def list_tables( + self, like: str | None = None, database: tuple[str, str] | str | None = None + ) -> list[str]: + """Return the list of table names via the backend's implementation.""" + self._raise_if_not_implemented("_list_tables") + return self._backend._list_tables(like=like, database=database) + + def list_temp_tables( + self, like: str | None = None, database: tuple[str, str] | str | None = None + ) -> list[str]: + """Return the list of temporary table names via the backend's implementation.""" + self._raise_if_not_implemented("_list_temp_tables") + return self._backend._list_temp_tables(like=like, database=database) + + def list_views( + self, like: str | None = None, database: tuple[str, str] | str | None = None + ) -> list[str]: + """Return the list of view names via the backend's implementation.""" + self._raise_if_not_implemented("_list_views") + return self._backend._list_views(like=like, database=database) + + def list_temp_views( + self, like: str | None = None, database: tuple[str, str] | str | None = None + ) -> list[str]: + """Return the list of temp view names via the backend's implementation.""" + self._raise_if_not_implemented("_list_temp_views") + return self._backend._list_temp_views(like=like, database=database) class _FileIOHandler: @@ -811,7 +881,12 @@ def __init__(self, *args, **kwargs): self._con_args: tuple[Any] = args self._con_kwargs: dict[str, Any] = kwargs self._can_reconnect: bool = True - self._query_cache = RefCountedCache(weakref.proxy(self)) + # expression cache + self._query_cache = RefCountedCache( + populate=self._load_into_cache, + lookup=lambda name: self.table(name).op(), + finalize=self._clean_up_cached_table, + ) @property @abc.abstractmethod @@ -933,44 +1008,6 @@ def _filter_with_like(values: Iterable[str], like: str | None = None) -> list[st pattern = re.compile(like) return sorted(filter(pattern.findall, values)) - @abc.abstractmethod - def list_tables( - self, like: str | None = None, database: tuple[str, str] | str | None = None - ) -> list[str]: - """Return the list of table names in the current database. - - For some backends, the tables may be files in a directory, - or other equivalent entities in a SQL database. - - ::: {.callout-note} - ## Ibis does not use the word `schema` to refer to database hierarchy. - - A collection of tables is referred to as a `database`. - A collection of `database` is referred to as a `catalog`. - - These terms are mapped onto the corresponding features in each - backend (where available), regardless of whether the backend itself - uses the same terminology. - ::: - - Parameters - ---------- - like - A pattern in Python's regex format. - database - The database from which to list tables. - If not provided, the current database is used. - For backends that support multi-level table hierarchies, you can - pass in a dotted string path like `"catalog.database"` or a tuple of - strings like `("catalog", "database")`. - - Returns - ------- - list[str] - The list of the table names that match the pattern `like`. - - """ - @abc.abstractmethod def table( self, name: str, database: tuple[str, str] | str | None = None @@ -1019,7 +1056,12 @@ def tables(self): >>> people = con.tables.people # access via attribute """ - return TablesAccessor(weakref.proxy(self)) + return TablesAccessor(self) + + @property + def ddl(self): + """A ddl accessor.""" + return DDLAccessor(self) @property @abc.abstractmethod diff --git a/ibis/backends/duckdb/__init__.py b/ibis/backends/duckdb/__init__.py index 13890d1b5d01c..f14c414d94ba8 100644 --- a/ibis/backends/duckdb/__init__.py +++ b/ibis/backends/duckdb/__init__.py @@ -977,86 +977,137 @@ def read_delta( self.con.register(table_name, delta_table.to_pyarrow_dataset()) return self.table(table_name) - def list_tables( + def _list_query_constructor(self, col: str, where_predicates: list) -> str: + """Helper function to construct sqlglot queries for _list_* methods.""" + + sg_query = ( + sg.select(col) + .from_(sg.table("tables", db="information_schema")) + .where(*where_predicates) + ).sql(self.name) + + return sg_query + + def _list_tables( self, like: str | None = None, database: tuple[str, str] | str | None = None, - schema: str | None = None, ) -> list[str]: - """List tables and views. + """List physical tables.""" - ::: {.callout-note} - ## Ibis does not use the word `schema` to refer to database hierarchy. + table_loc = self._warn_and_create_table_loc(database) - A collection of tables is referred to as a `database`. - A collection of `database` is referred to as a `catalog`. + catalog = table_loc.catalog or self.current_catalog + database = table_loc.db or self.current_database - These terms are mapped onto the corresponding features in each - backend (where available), regardless of whether the backend itself - uses the same terminology. - ::: + col = "table_name" + where_predicates = [ + C.table_catalog.eq(sge.convert(catalog)), + C.table_schema.eq(sge.convert(database)), + C.table_type.eq("BASE TABLE"), + ] - Parameters - ---------- - like - Regex to filter by table/view name. - database - Database location. If not passed, uses the current database. + sql = self._list_query_constructor(col, where_predicates) + out = self.con.execute(sql).fetch_arrow_table() - By default uses the current `database` (`self.current_database`) and - `catalog` (`self.current_catalog`). + return self._filter_with_like(out[col].to_pylist(), like) - To specify a table in a separate catalog, you can pass in the - catalog and database as a string `"catalog.database"`, or as a tuple of - strings `("catalog", "database")`. - schema - [deprecated] Schema name. If not passed, uses the current schema. + def _list_temp_tables( + self, + like: str | None = None, + database: tuple[str, str] | str | None = None, + ) -> list[str]: + """List temporary tables.""" - Returns - ------- - list[str] - List of table and view names. + table_loc = self._warn_and_create_table_loc(database) - Examples - -------- - >>> import ibis - >>> con = ibis.duckdb.connect() - >>> foo = con.create_table("foo", schema=ibis.schema(dict(a="int"))) - >>> con.list_tables() - ['foo'] - >>> bar = con.create_view("bar", foo) - >>> con.list_tables() - ['bar', 'foo'] - >>> con.create_database("my_database") - >>> con.list_tables(database="my_database") - [] - >>> with con.begin() as c: - ... c.exec_driver_sql("CREATE TABLE my_database.baz (a INTEGER)") # doctest: +ELLIPSIS - <...> - >>> con.list_tables(database="my_database") - ['baz'] + catalog = table_loc.catalog or "temp" + database = table_loc.db or self.current_database - """ - table_loc = self._warn_and_create_table_loc(database, schema) + col = "table_name" + where_predicates = [ + C.table_type.eq("LOCAL TEMPORARY"), + C.table_catalog.eq(sge.convert(catalog)), + C.table_schema.eq(sge.convert(database)), + ] + + sql = self._list_query_constructor(col, where_predicates) + out = self.con.execute(sql).fetch_arrow_table() + + return self._filter_with_like(out[col].to_pylist(), like) + + def _list_views( + self, + like: str | None = None, + database: tuple[str, str] | str | None = None, + ) -> list[str]: + """List views.""" + + table_loc = self._warn_and_create_table_loc(database) catalog = table_loc.catalog or self.current_catalog database = table_loc.db or self.current_database col = "table_name" - sql = ( - sg.select(col) - .from_(sg.table("tables", db="information_schema")) - .distinct() - .where( - C.table_catalog.isin(sge.convert(catalog), sge.convert("temp")), - C.table_schema.eq(sge.convert(database)), - ) - .sql(self.dialect) - ) + where_predicates = [ + C.table_catalog.eq(sge.convert(catalog)), + C.table_schema.eq(sge.convert(database)), + C.table_type.eq("VIEW"), + ] + + sql = self._list_query_constructor(col, where_predicates) out = self.con.execute(sql).fetch_arrow_table() return self._filter_with_like(out[col].to_pylist(), like) + def _list_temp_views( + self, + like: str | None = None, + database: tuple[str, str] | str | None = None, + ) -> list[str]: + """List views.""" + + table_loc = self._warn_and_create_table_loc(database) + + catalog = table_loc.catalog or "temp" + database = table_loc.db or self.current_database + + col = "table_name" + where_predicates = [ + C.table_catalog.eq(sge.convert(catalog)), + C.table_schema.eq(sge.convert(database)), + C.table_type.eq("VIEW"), + ] + + sql = self._list_query_constructor(col, where_predicates) + out = self.con.execute(sql).fetch_arrow_table() + + return self._filter_with_like(out[col].to_pylist(), like) + + @deprecated(as_of="10.0", instead="use the con.tables") + def list_tables( + self, + like: str | None = None, + database: tuple[str, str] | str | None = None, + schema: str | None = None, + ) -> list[str]: + """List tables and views.""" + + table_loc = self._warn_and_create_table_loc(database, schema) + + database = self.current_database + if table_loc is not None: + database = table_loc.db or database + + tables_and_views = list( + set(self._list_tables(like=like, database=database)) + | set(self._list_temp_tables(like=like, database=database)) + | set(self._list_views(like=like, database=database)) + | set(self._list_temp_views(like=like, database=database)) + ) + + return tables_and_views + def read_postgres( self, uri: str, *, table_name: str | None = None, database: str = "public" ) -> ir.Table: diff --git a/ibis/backends/duckdb/tests/test_catalog.py b/ibis/backends/duckdb/tests/test_catalog.py index a59c04ee37437..a5a96d99b81a4 100644 --- a/ibis/backends/duckdb/tests/test_catalog.py +++ b/ibis/backends/duckdb/tests/test_catalog.py @@ -36,8 +36,8 @@ def test_read_write_external_catalog(con, external_duckdb_file, monkeypatch): assert "ext" in con.list_catalogs() assert "main" in con.list_databases(catalog="ext") - assert "starwars" in con.list_tables(database="ext.main") - assert "starwars" not in con.list_tables() + assert "starwars" in con.ddl.list_tables(database="ext.main") + assert "starwars" not in con.ddl.list_tables() starwars = con.table("starwars", database="ext.main") tm.assert_frame_equal(starwars.to_pandas(), starwars_df) @@ -47,8 +47,8 @@ def test_read_write_external_catalog(con, external_duckdb_file, monkeypatch): _ = con.create_table("t2", obj=t, database="ext.main") - assert "t2" in con.list_tables(database="ext.main") - assert "t2" not in con.list_tables() + assert "t2" in con.ddl.list_tables(database="ext.main") + assert "t2" not in con.ddl.list_tables() table = con.table("t2", database="ext.main") @@ -60,8 +60,8 @@ def test_read_write_external_catalog(con, external_duckdb_file, monkeypatch): _ = con.create_table("t2", obj=t_overwrite, database="ext.main", overwrite=True) - assert "t2" in con.list_tables(database="ext.main") - assert "t2" not in con.list_tables() + assert "t2" in con.ddl.list_tables(database="ext.main") + assert "t2" not in con.ddl.list_tables() table = con.table("t2", database="ext.main") diff --git a/ibis/backends/duckdb/tests/test_client.py b/ibis/backends/duckdb/tests/test_client.py index 94a0c028a302f..55dc6296cdbc2 100644 --- a/ibis/backends/duckdb/tests/test_client.py +++ b/ibis/backends/duckdb/tests/test_client.py @@ -1,6 +1,5 @@ from __future__ import annotations -import gc import os import subprocess import sys @@ -293,15 +292,15 @@ def test_list_tables_schema_warning_refactor(con): "diamonds", "functional_alltypes", "win", - }.issubset(con.list_tables()) + }.issubset(con.tables) icecream_table = ["ice_cream"] with pytest.warns(FutureWarning): assert con.list_tables(schema="shops") == icecream_table - assert con.list_tables(database="shops") == icecream_table - assert con.list_tables(database=("shops",)) == icecream_table + assert con.ddl.list_tables(database="shops") == icecream_table + assert con.ddl.list_tables(database=("shops",)) == icecream_table def test_settings_repr(): @@ -315,16 +314,16 @@ def test_connect_named_in_memory_db(): con_named_db = ibis.duckdb.connect(":memory:mydb") con_named_db.create_table("ork", schema=ibis.schema(dict(bork="int32"))) - assert "ork" in con_named_db.list_tables() + assert "ork" in con_named_db.tables con_named_db_2 = ibis.duckdb.connect(":memory:mydb") - assert "ork" in con_named_db_2.list_tables() + assert "ork" in con_named_db_2.tables unnamed_memory_db = ibis.duckdb.connect(":memory:") - assert "ork" not in unnamed_memory_db.list_tables() + assert "ork" not in unnamed_memory_db.tables default_memory_db = ibis.duckdb.connect() - assert "ork" not in default_memory_db.list_tables() + assert "ork" not in default_memory_db.tables @pytest.mark.parametrize( @@ -404,23 +403,3 @@ def test_read_csv_with_types(tmp_path, input, all_varchar): path.write_bytes(data) t = con.read_csv(path, all_varchar=all_varchar, **input) assert t.schema()["geom"].is_geospatial() - - -def test_tables_accessor_no_reference_cycle(): - """Test that a single reference to a connection has the desired lifetime semantics.""" - con = ibis.duckdb.connect() - - before = len(gc.get_referrers(con)) - tables = con.tables - after = len(gc.get_referrers(con)) - - assert after == before - - # valid call, and there are no tables in the database - assert not list(tables) - - del con - - # no longer valid because the backend has been manually decref'd - with pytest.raises(ReferenceError): - list(tables) diff --git a/ibis/backends/duckdb/tests/test_geospatial.py b/ibis/backends/duckdb/tests/test_geospatial.py index 2c748c1d9f48f..d62004751c618 100644 --- a/ibis/backends/duckdb/tests/test_geospatial.py +++ b/ibis/backends/duckdb/tests/test_geospatial.py @@ -263,12 +263,9 @@ def test_geospatial_flip_coordinates(geotable): def test_create_table_geospatial_types(geotable, con): name = ibis.util.gen_name("geotable") - - # con = ibis.get_backend(geotable) - t = con.create_table(name, geotable, temp=True) - assert t.op().name in con.list_tables() + assert t.op().name in con.tables assert any(map(methodcaller("is_geospatial"), t.schema().values())) diff --git a/ibis/backends/duckdb/tests/test_register.py b/ibis/backends/duckdb/tests/test_register.py index 5f4a79564bfa1..93e008d554c07 100644 --- a/ibis/backends/duckdb/tests/test_register.py +++ b/ibis/backends/duckdb/tests/test_register.py @@ -292,7 +292,7 @@ def test_attach_sqlite(data_dir, tmp_path): con = ibis.duckdb.connect() con.attach_sqlite(test_db_path) - assert set(con.list_tables()) >= { + assert set(con.tables) >= { "functional_alltypes", "awards_players", "batting", @@ -304,7 +304,7 @@ def test_attach_sqlite(data_dir, tmp_path): # overwrite existing sqlite_db and force schema to all strings con.attach_sqlite(test_db_path, overwrite=True, all_varchar=True) - assert set(con.list_tables()) >= { + assert set(con.tables) >= { "functional_alltypes", "awards_players", "batting", diff --git a/ibis/backends/tests/test_api.py b/ibis/backends/tests/test_api.py index 074c288933da8..17dfc967308c2 100644 --- a/ibis/backends/tests/test_api.py +++ b/ibis/backends/tests/test_api.py @@ -1,7 +1,5 @@ from __future__ import annotations -import gc - import pytest from pytest import param @@ -61,9 +59,9 @@ def test_catalog_consistency(backend, con): assert current_catalog in catalogs -def test_list_tables(con): - tables = con.list_tables() - assert isinstance(tables, list) +def test_list_all_tables_and_views(con): + tables = con.tables + # only table that is guaranteed to be in all backends key = "functional_alltypes" assert key in tables or key.upper() in tables @@ -84,7 +82,7 @@ def test_tables_accessor_mapping(con): # temporary might pop into existence in parallel test runs, in between the # first `list_tables` call and the second, so we check that there's a # non-empty intersection - assert TEST_TABLES.keys() & set(map(str.lower, con.list_tables())) + assert TEST_TABLES.keys() & set(map(str.lower, con.ddl.list_tables())) assert TEST_TABLES.keys() & set(map(str.lower, con.tables)) @@ -117,16 +115,6 @@ def test_tables_accessor_repr(con): assert f"- {name}" in result -def test_tables_accessor_no_reference_cycle(con): - before = len(gc.get_referrers(con)) - _ = con.tables - after = len(gc.get_referrers(con)) - - # assert that creating a `tables` accessor object doesn't increase the - # number of strong references - assert after == before - - @pytest.mark.parametrize( "expr_fn", [ @@ -154,3 +142,58 @@ def test_unbind(alltypes, expr_fn): assert "Unbound" not in repr(expr) assert "Unbound" in repr(expr.unbind()) + + +## works on duckdb only for now +def test_list_tables(ddl_con): + # should check only physical tables + table_name = "functional_alltypes" + tables = ddl_con.ddl.list_tables() + assert isinstance(tables, list) + assert table_name in tables + + assert table_name not in ddl_con.ddl.list_views() + assert table_name not in ddl_con.ddl.list_temp_tables() + assert table_name not in ddl_con.ddl.list_temp_views() + + +def test_list_views(ddl_con, temp_view): + # temp_view: view name + expr = ddl_con.table("functional_alltypes") + ddl_con.create_view(temp_view, expr) + + views = ddl_con.ddl.list_views() + + assert isinstance(views, list) + assert temp_view in views + assert temp_view not in ddl_con.ddl.list_tables() + assert temp_view not in ddl_con.ddl.list_temp_tables() + assert temp_view not in ddl_con.ddl.list_temp_views() + + +def test_list_temp_tables(ddl_con): + expr = ddl_con.table("functional_alltypes") + temp_table_name = "all_types_temp" + ddl_con.create_table(temp_table_name, expr, temp=True) + temp_tables = ddl_con.ddl.list_temp_tables() + + assert isinstance(temp_tables, list) + assert temp_table_name in temp_tables + assert temp_table_name not in ddl_con.ddl.list_views() + assert temp_table_name not in ddl_con.ddl.list_temp_views() + assert temp_table_name not in ddl_con.ddl.list_tables() + + +def test_list_temp_views(ddl_con): + # TODO: replace raw_sql with create_temp + ddl_con.raw_sql(""" + CREATE TEMPORARY VIEW temp_view_example AS SELECT * FROM functional_alltypes + """) + + temporary_views = ddl_con.ddl.list_temp_views() + + assert isinstance(temporary_views, list) + assert "temp_view_example" in temporary_views + assert "temp_view_example" not in ddl_con.ddl.list_tables() + assert "temp_view_example" not in ddl_con.ddl.list_views() + assert "temp_view_example" not in ddl_con.ddl.list_temp_tables() diff --git a/ibis/backends/tests/test_client.py b/ibis/backends/tests/test_client.py index 432f0be2836b3..a56629cbb5f5a 100644 --- a/ibis/backends/tests/test_client.py +++ b/ibis/backends/tests/test_client.py @@ -329,7 +329,7 @@ def test_create_temporary_table_from_schema(con_no_data, new_schema): con_no_data.reconnect() # verify table no longer exist after reconnect - assert temp_table not in con_no_data.list_tables() + assert temp_table not in con_no_data.tables @mark.notimpl( @@ -397,7 +397,7 @@ def test_nullable_input_output(con, temp_table): def test_create_drop_view(ddl_con, temp_view): # setup table_name = "functional_alltypes" - tables = ddl_con.list_tables() + tables = ddl_con.tables if table_name in tables or (table_name := table_name.upper()) in tables: expr = ddl_con.table(table_name) @@ -409,7 +409,7 @@ def test_create_drop_view(ddl_con, temp_view): # create a new view ddl_con.create_view(temp_view, expr) # check if the view was created - assert temp_view in ddl_con.list_tables() + assert temp_view in ddl_con.ddl.list_views() t_expr = ddl_con.table(table_name) v_expr = ddl_con.table(temp_view) @@ -452,7 +452,7 @@ def employee_data_1_temp_table(backend, con, test_employee_schema): _create_temp_table_with_schema( backend, con, temp_table_name, test_employee_schema, data=test_employee_data_1 ) - assert temp_table_name in con.list_tables() + assert temp_table_name in con.tables yield temp_table_name con.drop_table(temp_table_name, force=True) @@ -725,7 +725,7 @@ def test_unsigned_integer_type(con, temp_table): schema=ibis.schema(dict(a="uint8", b="uint16", c="uint32", d="uint64")), overwrite=True, ) - assert temp_table in con.list_tables() + assert temp_table in con.tables @pytest.mark.backend @@ -1047,7 +1047,7 @@ def test_create_table_in_memory(con, obj, table_name, monkeypatch): t = con.create_table(table_name, obj) result = pa.table({"a": ["a"], "b": [1]}) - assert table_name in con.list_tables() + assert table_name in con.tables assert result.equals(t.to_pyarrow()) diff --git a/ibis/backends/tests/test_expr_caching.py b/ibis/backends/tests/test_expr_caching.py index d2623b0a593dd..84fb196180912 100644 --- a/ibis/backends/tests/test_expr_caching.py +++ b/ibis/backends/tests/test_expr_caching.py @@ -100,7 +100,7 @@ def test_persist_expression_multiple_refs(backend, con, alltypes): assert op not in con._query_cache.cache # assert that table has been dropped - assert name not in con.list_tables() + assert name not in con.tables @mark.notimpl(["datafusion", "flink", "impala", "trino", "druid"]) @@ -126,7 +126,7 @@ def test_persist_expression_repeated_cache(alltypes, con): del nested_cached_table, cached_table - assert name not in con.list_tables() + assert name not in con.tables @mark.notimpl(["datafusion", "flink", "impala", "trino", "druid"]) diff --git a/ibis/backends/tests/test_register.py b/ibis/backends/tests/test_register.py index cdfa1683743f2..7b8801bc8485f 100644 --- a/ibis/backends/tests/test_register.py +++ b/ibis/backends/tests/test_register.py @@ -103,7 +103,7 @@ def test_register_csv(con, data_dir, fname, in_table_name, out_table_name): with pytest.warns(FutureWarning, match="v9.1"): table = con.register(fname, table_name=in_table_name) - assert any(out_table_name in t for t in con.list_tables()) + assert any(out_table_name in t for t in con.tables) if con.name != "datafusion": table.count().execute() @@ -226,7 +226,7 @@ def test_register_parquet( with pytest.warns(FutureWarning, match="v9.1"): table = con.register(f"parquet://{fname.name}", table_name=in_table_name) - assert any(out_table_name in t for t in con.list_tables()) + assert any(out_table_name in t for t in con.tables) if con.name != "datafusion": table.count().execute() @@ -273,7 +273,7 @@ def test_register_iterator_parquet( table_name=None, ) - assert any("ibis_read_parquet" in t for t in con.list_tables()) + assert any("ibis_read_parquet" in t for t in con.tables) assert table.count().execute() diff --git a/ibis/common/caching.py b/ibis/common/caching.py index 14e1e569b50df..006f572310597 100644 --- a/ibis/common/caching.py +++ b/ibis/common/caching.py @@ -2,9 +2,9 @@ import functools import sys -import weakref from collections import namedtuple from typing import TYPE_CHECKING, Any +from weakref import finalize, ref if TYPE_CHECKING: from collections.abc import Callable @@ -39,8 +39,17 @@ class RefCountedCache: We can implement that interface if and when we need to. """ - def __init__(self, backend: weakref.proxy) -> None: - self.backend = backend + def __init__( + self, + *, + populate: Callable[[str, Any], None], + lookup: Callable[[str], Any], + finalize: Callable[[Any], None], + ) -> None: + self.populate = populate + self.lookup = lookup + self.finalize = finalize + self.cache: dict[Any, CacheEntry] = dict() def get(self, key, default=None): @@ -61,13 +70,11 @@ def store(self, input): key = input.op() name = gen_name("cache") + self.populate(name, input) + cached = self.lookup(name) + finalizer = finalize(cached, self._release, key) - self.backend._load_into_cache(name, input) - - cached = self.backend.table(name).op() - finalizer = weakref.finalize(cached, self._release, key) - - self.cache[key] = CacheEntry(name, weakref.ref(cached), finalizer) + self.cache[key] = CacheEntry(name, ref(cached), finalizer) return cached @@ -81,7 +88,7 @@ def release(self, name: str) -> None: def _release(self, key) -> None: entry = self.cache.pop(key) try: - self.backend._clean_up_cached_table(entry.name) + self.finalize(entry.name) except Exception: # suppress exceptions during interpreter shutdown if not sys.is_finalizing(): diff --git a/ibis/examples/tests/test_examples.py b/ibis/examples/tests/test_examples.py index 40d1e2bf4afed..a2eeee4affd0e 100644 --- a/ibis/examples/tests/test_examples.py +++ b/ibis/examples/tests/test_examples.py @@ -84,7 +84,7 @@ def test_non_example(): def test_backend_arg(): con = ibis.duckdb.connect() t = ibis.examples.penguins.fetch(backend=con) - assert t.get_name() in con.list_tables() + assert t.get_name() in con.tables @pytest.mark.duckdb