From cd0faf9a0a557b2a79cb9dab40dd2d1d068bb26f Mon Sep 17 00:00:00 2001 From: universalmind303 Date: Tue, 5 Mar 2024 14:52:40 -0600 Subject: [PATCH] chore: use a lance fork without the duckdb submodule (#2742) lance uses duckdb as a submodule, and it's a pretty massive repo, `555.9 MB` on a fresh clone. It currently takes absolutely forever to build glaredb the first time _(or subsequent times if you nuke your cargo cache)_. `git clone https://github.com/duckdb/duckdb.git` alone takes almost 4 minutes locally. this just forks lance and removes the submodule to avoid the problem alltogether. --- Cargo.lock | 46 ++++++++++++++++++++--------------- crates/datasources/Cargo.toml | 2 +- crates/sqlbuiltins/Cargo.toml | 2 +- 3 files changed, 28 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c51c071ef..b6986dd52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4094,8 +4094,8 @@ dependencies = [ [[package]] name = "lance" -version = "0.9.12" -source = "git+https://github.com/lancedb/lance?rev=310d79eccf93f3c6a48c162c575918cdba13faec#310d79eccf93f3c6a48c162c575918cdba13faec" +version = "0.10.2" +source = "git+https://github.com/GlareDB/lance?rev=de6df70d9c5d95a4818b8799c23e3d1ad649bc1d#de6df70d9c5d95a4818b8799c23e3d1ad649bc1d" dependencies = [ "arrow", "arrow-arith", @@ -4151,11 +4151,12 @@ dependencies = [ [[package]] name = "lance-arrow" -version = "0.9.12" -source = "git+https://github.com/lancedb/lance?rev=310d79eccf93f3c6a48c162c575918cdba13faec#310d79eccf93f3c6a48c162c575918cdba13faec" +version = "0.10.2" +source = "git+https://github.com/GlareDB/lance?rev=de6df70d9c5d95a4818b8799c23e3d1ad649bc1d#de6df70d9c5d95a4818b8799c23e3d1ad649bc1d" dependencies = [ "arrow-array", "arrow-buffer", + "arrow-cast", "arrow-data", "arrow-schema", "arrow-select", @@ -4168,8 +4169,8 @@ dependencies = [ [[package]] name = "lance-core" -version = "0.9.12" -source = "git+https://github.com/lancedb/lance?rev=310d79eccf93f3c6a48c162c575918cdba13faec#310d79eccf93f3c6a48c162c575918cdba13faec" +version = "0.10.2" +source = "git+https://github.com/GlareDB/lance?rev=de6df70d9c5d95a4818b8799c23e3d1ad649bc1d#de6df70d9c5d95a4818b8799c23e3d1ad649bc1d" dependencies = [ "arrow-array", "arrow-buffer", @@ -4183,6 +4184,7 @@ dependencies = [ "futures", "lance-arrow", "lazy_static", + "libc", "mock_instant", "moka 0.11.3", "object_store", @@ -4199,8 +4201,8 @@ dependencies = [ [[package]] name = "lance-datafusion" -version = "0.9.12" -source = "git+https://github.com/lancedb/lance?rev=310d79eccf93f3c6a48c162c575918cdba13faec#310d79eccf93f3c6a48c162c575918cdba13faec" +version = "0.10.2" +source = "git+https://github.com/GlareDB/lance?rev=de6df70d9c5d95a4818b8799c23e3d1ad649bc1d#de6df70d9c5d95a4818b8799c23e3d1ad649bc1d" dependencies = [ "arrow", "arrow-array", @@ -4222,8 +4224,8 @@ dependencies = [ [[package]] name = "lance-datagen" -version = "0.9.12" -source = "git+https://github.com/lancedb/lance?rev=310d79eccf93f3c6a48c162c575918cdba13faec#310d79eccf93f3c6a48c162c575918cdba13faec" +version = "0.10.2" +source = "git+https://github.com/GlareDB/lance?rev=de6df70d9c5d95a4818b8799c23e3d1ad649bc1d#de6df70d9c5d95a4818b8799c23e3d1ad649bc1d" dependencies = [ "arrow", "arrow-array", @@ -4236,8 +4238,8 @@ dependencies = [ [[package]] name = "lance-file" -version = "0.9.12" -source = "git+https://github.com/lancedb/lance?rev=310d79eccf93f3c6a48c162c575918cdba13faec#310d79eccf93f3c6a48c162c575918cdba13faec" +version = "0.10.2" +source = "git+https://github.com/GlareDB/lance?rev=de6df70d9c5d95a4818b8799c23e3d1ad649bc1d#de6df70d9c5d95a4818b8799c23e3d1ad649bc1d" dependencies = [ "arrow-arith", "arrow-array", @@ -4264,8 +4266,8 @@ dependencies = [ [[package]] name = "lance-index" -version = "0.9.12" -source = "git+https://github.com/lancedb/lance?rev=310d79eccf93f3c6a48c162c575918cdba13faec#310d79eccf93f3c6a48c162c575918cdba13faec" +version = "0.10.2" +source = "git+https://github.com/GlareDB/lance?rev=de6df70d9c5d95a4818b8799c23e3d1ad649bc1d#de6df70d9c5d95a4818b8799c23e3d1ad649bc1d" dependencies = [ "arrow", "arrow-array", @@ -4281,6 +4283,7 @@ dependencies = [ "datafusion-sql", "futures", "half", + "itertools 0.12.0", "lance-arrow", "lance-core", "lance-datafusion", @@ -4288,6 +4291,7 @@ dependencies = [ "lance-io", "lance-linalg", "lance-table", + "lazy_static", "log", "num-traits", "num_cpus", @@ -4307,8 +4311,8 @@ dependencies = [ [[package]] name = "lance-io" -version = "0.9.12" -source = "git+https://github.com/lancedb/lance?rev=310d79eccf93f3c6a48c162c575918cdba13faec#310d79eccf93f3c6a48c162c575918cdba13faec" +version = "0.10.2" +source = "git+https://github.com/GlareDB/lance?rev=de6df70d9c5d95a4818b8799c23e3d1ad649bc1d#de6df70d9c5d95a4818b8799c23e3d1ad649bc1d" dependencies = [ "arrow-arith", "arrow-array", @@ -4336,14 +4340,15 @@ dependencies = [ "shellexpand", "snafu", "tokio", + "tokio-stream", "tracing", "url", ] [[package]] name = "lance-linalg" -version = "0.9.12" -source = "git+https://github.com/lancedb/lance?rev=310d79eccf93f3c6a48c162c575918cdba13faec#310d79eccf93f3c6a48c162c575918cdba13faec" +version = "0.10.2" +source = "git+https://github.com/GlareDB/lance?rev=de6df70d9c5d95a4818b8799c23e3d1ad649bc1d#de6df70d9c5d95a4818b8799c23e3d1ad649bc1d" dependencies = [ "arrow-array", "arrow-ord", @@ -4353,6 +4358,7 @@ dependencies = [ "half", "lance-arrow", "lance-core", + "lazy_static", "log", "num-traits", "num_cpus", @@ -4363,8 +4369,8 @@ dependencies = [ [[package]] name = "lance-table" -version = "0.9.12" -source = "git+https://github.com/lancedb/lance?rev=310d79eccf93f3c6a48c162c575918cdba13faec#310d79eccf93f3c6a48c162c575918cdba13faec" +version = "0.10.2" +source = "git+https://github.com/GlareDB/lance?rev=de6df70d9c5d95a4818b8799c23e3d1ad649bc1d#de6df70d9c5d95a4818b8799c23e3d1ad649bc1d" dependencies = [ "arrow-array", "arrow-buffer", diff --git a/crates/datasources/Cargo.toml b/crates/datasources/Cargo.toml index 6f22b75e6..dcbecfbbe 100644 --- a/crates/datasources/Cargo.toml +++ b/crates/datasources/Cargo.toml @@ -69,7 +69,7 @@ tiberius = { version = "0.12.2", default-features = false, features = [ "rustls", "chrono", ] } -lance = { git = "https://github.com/lancedb/lance", rev = "310d79eccf93f3c6a48c162c575918cdba13faec" } +lance = { git = "https://github.com/GlareDB/lance", rev = "de6df70d9c5d95a4818b8799c23e3d1ad649bc1d" } bson = "2.9.0" scylla = { version = "0.12.0" } glob = "0.3.1" diff --git a/crates/sqlbuiltins/Cargo.toml b/crates/sqlbuiltins/Cargo.toml index 247599813..33372d1b1 100644 --- a/crates/sqlbuiltins/Cargo.toml +++ b/crates/sqlbuiltins/Cargo.toml @@ -36,4 +36,4 @@ reqwest.workspace = true # Important to keep this in sync with the datafusion arrow-cast version arrow-cast = { version = "50.0.0" } -lance-linalg = { git = "https://github.com/lancedb/lance", rev = "310d79eccf93f3c6a48c162c575918cdba13faec" } +lance-linalg = { git = "https://github.com/GlareDB/lance", rev = "de6df70d9c5d95a4818b8799c23e3d1ad649bc1d" }