Skip to content

Commit

Permalink
feat: [ODIN-895] LIST and STRUCT types (#99)
Browse files Browse the repository at this point in the history
  • Loading branch information
@rprovodenko
rprovodenko committed Jun 2, 2021
1 parent 1c97fbc commit 6effe12
Show file tree
Hide file tree
Showing 6 changed files with 139 additions and 55 deletions.
71 changes: 47 additions & 24 deletions addon/result_iterator.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,80 +141,103 @@ Napi::Value ResultIterator::getRowObject(Napi::Env env) {
}

Napi::Value ResultIterator::getCellValue(Napi::Env env, duckdb::idx_t col_idx) {
auto val = current_chunk->data[col_idx].GetValue(chunk_offset);
auto value = current_chunk->data[col_idx].GetValue(chunk_offset);
return getMappedValue(env, value);
}

if (val.is_null) {
Napi::Value ResultIterator::getMappedValue(Napi::Env env, duckdb::Value value) {
if (value.is_null) {
return env.Null();
}

switch (result->types[col_idx].id()) {
switch (value.type().id()) {
case duckdb::LogicalTypeId::BOOLEAN:
return Napi::Boolean::New(env, val.GetValue<bool>());
return Napi::Boolean::New(env, value.GetValue<bool>());
case duckdb::LogicalTypeId::TINYINT:
return Napi::Number::New(env, val.GetValue<int8_t>());
return Napi::Number::New(env, value.GetValue<int8_t>());
case duckdb::LogicalTypeId::SMALLINT:
return Napi::Number::New(env, val.GetValue<int16_t>());
return Napi::Number::New(env, value.GetValue<int16_t>());
case duckdb::LogicalTypeId::INTEGER:
return Napi::Number::New(env, val.GetValue<int32_t>());
return Napi::Number::New(env, value.GetValue<int32_t>());
case duckdb::LogicalTypeId::BIGINT:
return Napi::BigInt::New(env, val.GetValue<int64_t>());
return Napi::BigInt::New(env, value.GetValue<int64_t>());
case duckdb::LogicalTypeId::HUGEINT: {
// hugeint_t represents a signed 128 bit integer in two's complement
// notation napi's BigInt is basically a regular signed integer (MSB) so we
// want to make sure we pass the absolute value of the huge int into napi
// plus the sign bit
auto huge_int = val.GetValue<duckdb::hugeint_t>();
auto huge_int = value.GetValue<duckdb::hugeint_t>();
int is_negative = huge_int.upper < 0;
duckdb::hugeint_t positive_huge_int =
is_negative ? huge_int * duckdb::hugeint_t(-1) : huge_int;
uint64_t arr[2]{positive_huge_int.lower, (uint64_t)positive_huge_int.upper};
return Napi::BigInt::New(env, is_negative, 2, &arr[0]);
}
case duckdb::LogicalTypeId::FLOAT:
return Napi::Number::New(env, val.GetValue<float>());
return Napi::Number::New(env, value.GetValue<float>());
case duckdb::LogicalTypeId::DOUBLE:
return Napi::Number::New(env, val.GetValue<double>());
return Napi::Number::New(env, value.GetValue<double>());
case duckdb::LogicalTypeId::DECIMAL:
return Napi::Number::New(
env, val.CastAs(duckdb::LogicalType::DOUBLE).GetValue<double>());
env, value.CastAs(duckdb::LogicalType::DOUBLE).GetValue<double>());
case duckdb::LogicalTypeId::VARCHAR:
return Napi::String::New(env, val.GetValue<string>());
return Napi::String::New(env, value.GetValue<string>());
case duckdb::LogicalTypeId::BLOB: {
int array_length = val.str_value.length();
int array_length = value.str_value.length();
char char_array[array_length + 1];
// TODO: multiple copies, improve
strcpy(char_array, val.str_value.c_str());
strcpy(char_array, value.str_value.c_str());
return Napi::Buffer<char>::Copy(env, char_array, array_length);
}
case duckdb::LogicalTypeId::TIMESTAMP: {
if (result->types[col_idx].InternalType() != duckdb::PhysicalType::INT64) {
if (value.type().InternalType() != duckdb::PhysicalType::INT64) {
throw runtime_error("expected int64 for timestamp");
}
int64_t tval = val.GetValue<int64_t>();
int64_t tval = value.GetValue<int64_t>();
return Napi::Number::New(env, tval / 1000);
}
case duckdb::LogicalTypeId::TIME: {
if (result->types[col_idx].InternalType() != duckdb::PhysicalType::INT64) {
if (value.type().InternalType() != duckdb::PhysicalType::INT64) {
throw runtime_error("expected int64 for time");
}
int64_t tval = val.GetValue<int64_t>();
int64_t tval = value.GetValue<int64_t>();
return Napi::Number::New(env, GetTime(tval));
}
case duckdb::LogicalTypeId::INTERVAL: {
return Napi::String::New(env, val.ToString());
return Napi::String::New(env, value.ToString());
}
case duckdb::LogicalTypeId::UTINYINT:
return Napi::Number::New(env, val.GetValue<uint8_t>());
return Napi::Number::New(env, value.GetValue<uint8_t>());
case duckdb::LogicalTypeId::USMALLINT:
return Napi::Number::New(env, val.GetValue<uint16_t>());
return Napi::Number::New(env, value.GetValue<uint16_t>());
case duckdb::LogicalTypeId::UINTEGER:
// GetValue is not supported for uint32_t, so using the wider type
return Napi::Number::New(env, val.GetValue<int64_t>());
return Napi::Number::New(env, value.GetValue<int64_t>());
case duckdb::LogicalTypeId::LIST: {
auto array = Napi::Array::New(env);
for (size_t i = 0; i < value.list_value.size(); i++) {
auto &element = value.list_value[i];
auto mapped_value = getMappedValue(env, element);
array.Set(i, mapped_value);
}
return array;
}
case duckdb::LogicalTypeId::STRUCT: {
auto object = Napi::Object::New(env);
for (size_t i = 0; i < value.struct_value.size(); i++) {
auto &key = value.type().child_types()[i].first;
auto &element = value.struct_value[i];
auto child_value = getMappedValue(env, element);
object.Set(key, child_value);
}
return object;
}
default:
// default to getting string representation
return Napi::String::New(env, val.ToString());
return Napi::String::New(env, value.ToString());
}
}

Napi::Value ResultIterator::Close(const Napi::CallbackInfo &info) {
result.reset();
return info.Env().Undefined();
Expand Down
1 change: 1 addition & 0 deletions addon/result_iterator.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ class ResultIterator : public Napi::ObjectWrap<ResultIterator> {
duckdb::unique_ptr<duckdb::DataChunk> current_chunk;
uint64_t chunk_offset = 0;
Napi::Value getCellValue(Napi::Env env, duckdb::idx_t col_idx);
Napi::Value getMappedValue(Napi::Env env, duckdb::Value duckdb_value);
Napi::Value getRowArray(Napi::Env env);
Napi::Value getRowObject(Napi::Env env);
};
Expand Down
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "node-duckdb",
"version": "0.0.70",
"version": "0.0.71",
"private": false,
"description": "DuckDB for Node.JS",
"keywords": [
Expand Down
65 changes: 65 additions & 0 deletions src/tests/data-types.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -220,4 +220,69 @@ describe("Data type mapping", () => {

expect(result.fetchRow()).toMatchObject([1]);
});

it("supports LIST - integers", async () => {
const result = await connection.executeIterator<any[]>(`SELECT array_slice([1,2,3], 1, NULL)`);
expect(result.fetchRow()).toMatchObject({ "array_slice(list_value(1, 2, 3), 1, NULL)": [2, 3] });
});

it("supports LIST - strings", async () => {
const result = await connection.executeIterator<any[]>(`SELECT array_slice(['a','b','c'], 1, NULL)`);
expect(result.fetchRow()).toEqual({ "array_slice(list_value(a, b, c), 1, NULL)": ["b", "c"] });
});

it("supports LIST - STRUCTs", async () => {
const result = await connection.executeIterator<any[]>(
`SELECT raw_header from parquet_scan('src/tests/test-fixtures/crawl_urls.parquet')`,
);
expect(result.fetchRow()).toEqual({
raw_header: [
{
key: "Content-Encoding",
value: "gzip",
},
{
key: "X-Frame-Options",
value: "SAMEORIGIN",
},
{
key: "Connection",
value: "keep-alive",
},
{
key: "X-Xss-Protection",
value: "1; mode=block",
},
{
key: "Content-Type",
value: "text/html;charset=utf-8",
},
{
key: "Date",
value: "Tue, 18 Aug 2020 13:46:36 GMT",
},
{
key: "Vary",
value: "User-agent,Accept-Encoding",
},
{
key: "Server",
value: "nginx/1.10.3",
},
{
key: "X-Content-Type-Options",
value: "nosniff",
},
{
key: "Content-Length",
value: "1180",
},
],
});
});

it("supports STRUCT", async () => {
const result = await connection.executeIterator<any[]>(`SELECT struct_pack(i := 4, s := 'string')`);
expect(result.fetchRow()).toEqual({ "struct_pack(4, string)": { i: 4, s: "string" } });
});
});
Binary file added src/tests/test-fixtures/crawl_urls.parquet
View file
Binary file not shown.
55 changes: 25 additions & 30 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -612,9 +612,9 @@
fastq "^1.6.0"

"@npmcli/arborist@^2.3.0", "@npmcli/arborist@^2.5.0":
version "2.5.0"
resolved "https://registry.yarnpkg.com/@npmcli/arborist/-/arborist-2.5.0.tgz#3a2bf74e47b4cbe20e0983291f291a2ec2c3d06f"
integrity sha512-YPSkV/8vofpbAJyeu52J12YnC5VTkYIcfcNkRoSW6qjfQG+QybgbJtCbcdx+M0YxfdzDKS6iDTjpNMoETZ8HOA==
version "2.6.1"
resolved "https://registry.yarnpkg.com/@npmcli/arborist/-/arborist-2.6.1.tgz#bdbd1311cc857583ffc85f4d3f24a50683303dda"
integrity "sha1-vb0TEcyFdYP/yF9NPySlBoMwPdo= sha512-OOlntFIOAo7RplEQaYXlA5U5NXE+EwZtnTCsit4Wtme5+llGiea6GBytuV8dOzdPMPlNx3fQQjBUE9E8k76yjQ=="
dependencies:
"@npmcli/installed-package-contents" "^1.0.7"
"@npmcli/map-workspaces" "^1.0.2"
Expand Down Expand Up @@ -1815,9 +1815,9 @@ byte-size@^7.0.1:
integrity sha512-crQdqyCwhokxwV1UyDzLZanhkugAgft7vt0qbbdt60C6Zf3CAiGmtUCylbtYwrU6loOUw3euGrNtW1J651ot1A==

cacache@^15.0.3, cacache@^15.0.5, cacache@^15.0.6:
version "15.0.6"
resolved "https://registry.yarnpkg.com/cacache/-/cacache-15.0.6.tgz#65a8c580fda15b59150fb76bf3f3a8e45d583099"
integrity sha512-g1WYDMct/jzW+JdWEyjaX2zoBkZ6ZT9VpOyp2I/VMtDsNLffNat3kqPFfi1eDRSK9/SuKGyORDHcQMcPF8sQ/w==
version "15.2.0"
resolved "https://registry.yarnpkg.com/cacache/-/cacache-15.2.0.tgz#73af75f77c58e72d8c630a7a2858cb18ef523389"
integrity "sha1-c69193xY5y2MYwp6KFjLGO9SM4k= sha512-uKoJSHmnrqXgthDFx/IU6ED/5xd+NNGe+Bb+kLZy7Ku4P+BaiWEUflAKPZ7eAzsYGcsAGASJZsybXp+quEcHTw=="
dependencies:
"@npmcli/move-file" "^1.0.1"
chownr "^2.0.0"
Expand Down Expand Up @@ -3602,9 +3602,9 @@ fs.realpath@^1.0.0:
integrity sha1-FQStJSMVjKpA20onh8sBQRmU6k8=

fsevents@^2.1.2:
version "2.3.1"
resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.1.tgz#b209ab14c61012636c8863507edf7fb68cc54e9f"
integrity sha512-YR47Eg4hChJGAB1O3yEAOkGO+rlzutoICGqGo9EZ4lKWokzZRSyIW1QmTzqjtw8MJdj9srP869CuWw/hyzSiBw==
version "2.3.2"
resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a"
integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==

fsevents@~2.1.2:
version "2.1.3"
Expand Down Expand Up @@ -4109,9 +4109,9 @@ iconv-lite@0.4.24:
safer-buffer ">= 2.1.2 < 3"

iconv-lite@^0.6.2:
version "0.6.2"
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.2.tgz#ce13d1875b0c3a674bd6a04b7f76b01b1b6ded01"
integrity sha512-2y91h5OpQlolefMPmUlivelittSWy0rP+oYVpn6A7GwVHNE8AWzoYOBNmlwks3LobaJxgHCYZAnyNo2GgpNRNQ==
version "0.6.3"
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.3.tgz#a52f80bf38da1952eb5c681790719871a1a72501"
integrity "sha1-pS+AvzjaGVLrXGgXkHGYcaGnJQE= sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw=="
dependencies:
safer-buffer ">= 2.1.2 < 3.0.0"

Expand Down Expand Up @@ -5360,9 +5360,9 @@ libnpmdiff@^2.0.4:
tar "^6.1.0"

libnpmexec@^1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/libnpmexec/-/libnpmexec-1.1.1.tgz#aa510cc74612cac3945f7e2fd2ac4c650fa7ef66"
integrity sha512-uw6H2dzC6F6fdq7lAxfzXPimHCJ3/g6ycFKcv2Q2QXuNZ94EDmNPpMO6f4mwiC5F6Lyy/WK0IL7AZwRhmSvUdQ==
version "1.2.0"
resolved "https://registry.yarnpkg.com/libnpmexec/-/libnpmexec-1.2.0.tgz#ab0294ab63bb599b3ac6921129b7a706d4d56da2"
integrity "sha1-qwKUq2O7WZs6xpIRKbenBtTVbaI= sha512-LkxnH2wsMUI4thsgUK0r+EFZ5iCjKlp21J68dFY7AzD5uaaIPqO3lqVvYbyl1Umz1R4rY9t3vFa1fF3hzo6Y2Q=="
dependencies:
"@npmcli/arborist" "^2.3.0"
"@npmcli/ci-detect" "^1.3.0"
Expand Down Expand Up @@ -6183,9 +6183,9 @@ normalize-url@^4.1.0:
integrity sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ==

npm-audit-report@^2.1.4:
version "2.1.4"
resolved "https://registry.yarnpkg.com/npm-audit-report/-/npm-audit-report-2.1.4.tgz#b14c4625131fb7bcacc4b1c83842af1f58c92c98"
integrity sha512-Tz7rnfskSdZ0msTzt2mENC/B+H2QI8u0jN0ck7o3zDsQYIQrek/l3MjEc+CARer+64LsVTU6ZIqNuh0X55QPhw==
version "2.1.5"
resolved "https://registry.yarnpkg.com/npm-audit-report/-/npm-audit-report-2.1.5.tgz#a5b8850abe2e8452fce976c8960dd432981737b5"
integrity "sha1-pbiFCr4uhFL86XbIlg3UMpgXN7U= sha512-YB8qOoEmBhUH1UJgh1xFAv7Jg1d+xoNhsDYiFQlEFThEBui0W1vIz2ZK6FVg4WZjwEdl7uBQlm1jy3MUfyHeEw=="
dependencies:
chalk "^4.0.0"

Expand Down Expand Up @@ -6252,9 +6252,9 @@ npm-profile@^5.0.3:
npm-registry-fetch "^10.0.0"

npm-registry-fetch@^10.0.0, npm-registry-fetch@^10.1.1:
version "10.1.1"
resolved "https://registry.yarnpkg.com/npm-registry-fetch/-/npm-registry-fetch-10.1.1.tgz#97bc7a0fca5e8f76cc5162185b8de8caa8bea639"
integrity sha512-F6a3l+ffCQ7hvvN16YG5bpm1rPZntCg66PLHDQ1apWJPOCUVHoKnL2w5fqEaTVhp42dmossTyXeR7hTGirfXrg==
version "10.1.2"
resolved "https://registry.yarnpkg.com/npm-registry-fetch/-/npm-registry-fetch-10.1.2.tgz#11ffe03d813c653e768bdf762cfc5f1afe91b8bd"
integrity "sha1-Ef/gPYE8ZT52i992LPxfGv6RuL0= sha512-KsM/TdPmntqgBFlfsbkOLkkE9ovZo7VpVcd+/eTdYszCrgy5zFl5JzWm+OxavFaEWlbkirpkou+ZYI00RmOBFA=="
dependencies:
lru-cache "^6.0.0"
make-fetch-happen "^8.0.9"
Expand Down Expand Up @@ -8962,15 +8962,10 @@ write@1.0.3:
dependencies:
mkdirp "^0.5.1"

"ws@>= 2.2.3":
version "7.4.4"
resolved "https://registry.yarnpkg.com/ws/-/ws-7.4.4.tgz#383bc9742cb202292c9077ceab6f6047b17f2d59"
integrity sha512-Qm8k8ojNQIMx7S+Zp8u/uHOx7Qazv3Yv4q68MiWWWOJhiwG5W3x7iqmRtJo8xxrciZUY4vRxUTJCKuRnF28ZZw==

ws@^7.2.3:
version "7.4.2"
resolved "https://registry.yarnpkg.com/ws/-/ws-7.4.2.tgz#782100048e54eb36fe9843363ab1c68672b261dd"
integrity sha512-T4tewALS3+qsrpGI/8dqNMLIVdq/g/85U98HPMa6F0m6xTbvhXU6RCQLqPH3+SlomNV/LdY6RXEbBpMH6EOJnA==
"ws@>= 2.2.3", ws@^7.2.3:
version "7.4.6"
resolved "https://registry.yarnpkg.com/ws/-/ws-7.4.6.tgz#5654ca8ecdeee47c33a9a4bf6d28e2be2980377c"
integrity sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A==

xdg-basedir@^4.0.0:
version "4.0.0"
Expand Down

0 comments on commit 6effe12

Please sign in to comment.