Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added timestamp support #1884

Open
wants to merge 31 commits into
base: SNOW-873466-arrow-batches
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 26 commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
4d64dab
Initial version
sfc-gh-astachowski Aug 23, 2024
986967d
Initial tests
sfc-gh-astachowski Aug 26, 2024
c9f6287
Formatting
sfc-gh-astachowski Aug 26, 2024
0fc7b5a
Import formatting
sfc-gh-astachowski Aug 26, 2024
b409510
Added missing interface definitions
sfc-gh-astachowski Aug 26, 2024
1de2c39
Implemented review feedback
sfc-gh-astachowski Aug 27, 2024
969c59c
Further review feedback
sfc-gh-astachowski Aug 28, 2024
f12fdeb
Merge branch 'SNOW-873466-arrow-batches' into arrow-batches-all-simpl…
sfc-gh-astachowski Aug 30, 2024
a1cfe08
Added handling of remaining types
sfc-gh-astachowski Sep 3, 2024
53638df
Merge branch 'SNOW-873466-arrow-batches' into arrow-batches-all-simpl…
sfc-gh-astachowski Sep 3, 2024
0fd7d0f
Add null time zone check
sfc-gh-astachowski Sep 3, 2024
73d6b4d
Removed timestamp support
sfc-gh-astachowski Sep 3, 2024
7424acf
Added timestamp support
sfc-gh-astachowski Sep 3, 2024
3dd20a9
Formatting
sfc-gh-astachowski Sep 3, 2024
28fb57a
Removed old comments
sfc-gh-astachowski Sep 3, 2024
afa1142
Fixed memory leak and added assertion of no leaks in tests.
sfc-gh-astachowski Sep 3, 2024
7c9ab7a
Fixed memory leaks and added assertions of no memory leaks.
sfc-gh-astachowski Sep 3, 2024
31bb11b
Merge branch 'SNOW-873466-arrow-batches' into arrow-batches-all-simpl…
sfc-gh-astachowski Sep 6, 2024
ec255ee
Merge branch 'arrow-batches-all-simple-types' into arrow-batches-time…
sfc-gh-astachowski Sep 6, 2024
69cfe59
Merge fixes
sfc-gh-astachowski Sep 6, 2024
e26e56c
Merge branch 'SNOW-873466-arrow-batches' into arrow-batches-all-simpl…
sfc-gh-astachowski Sep 6, 2024
1c34bb9
Added null check
sfc-gh-astachowski Sep 6, 2024
de52645
Merge branch 'arrow-batches-all-simple-types' into arrow-batches-time…
sfc-gh-astachowski Sep 6, 2024
ef5238c
Formatting
sfc-gh-astachowski Sep 6, 2024
f4ace2d
Merge branch 'SNOW-873466-arrow-batches' into arrow-batches-all-simpl…
sfc-gh-astachowski Sep 9, 2024
660c0de
Merge branch 'arrow-batches-all-simple-types' into arrow-batches-time…
sfc-gh-astachowski Sep 9, 2024
45de13f
Merge branch 'SNOW-873466-arrow-batches' into arrow-batches-all-simpl…
sfc-gh-astachowski Sep 20, 2024
44e608d
Merge fixes
sfc-gh-astachowski Sep 20, 2024
400fd13
Added try-with-resources statements
sfc-gh-astachowski Sep 20, 2024
56eb83e
Merge branch 'arrow-batches-all-simple-types' into arrow-batches-time…
sfc-gh-astachowski Sep 20, 2024
318bc4f
Merge fixes and introduced constants
sfc-gh-astachowski Sep 24, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -814,10 +814,13 @@ public boolean hasNext() {
public ArrowBatch next() throws SQLException {
if (!firstFetched) {
firstFetched = true;
return currentChunkIterator.getChunk().getArrowBatch(SFArrowResultSet.this);
return currentChunkIterator
.getChunk()
.getArrowBatch(SFArrowResultSet.this, useSessionTimezone ? sessionTimeZone : null);
} else {
nextChunkIndex++;
return fetchNextChunk().getArrowBatch(SFArrowResultSet.this);
return fetchNextChunk()
.getArrowBatch(SFArrowResultSet.this, useSessionTimezone ? sessionTimeZone : null);
}
}
}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package net.snowflake.client.core.arrow.fullvectorconverters;

import java.util.Map;
import java.util.TimeZone;
import net.snowflake.client.core.DataConversionContext;
import net.snowflake.client.core.SFBaseSession;
import net.snowflake.client.core.SFException;
Expand Down Expand Up @@ -34,55 +35,27 @@ static Types.MinorType deduceType(ValueVector vector) {
break;
}
case TIME:
return Types.MinorType.TIMEMILLI;
case TIMESTAMP_LTZ:
{
String scaleStr = vector.getField().getMetadata().get("scale");
int sfScale = Integer.parseInt(scaleStr);
switch (sfScale) {
case 0:
return Types.MinorType.TIMESTAMPSECTZ;
case 3:
return Types.MinorType.TIMESTAMPMILLITZ;
case 6:
return Types.MinorType.TIMESTAMPMICROTZ;
case 9:
return Types.MinorType.TIMESTAMPNANOTZ;
if (sfScale == 0) {
return Types.MinorType.TIMESEC;
}
break;
}
case TIMESTAMP_TZ:
{
String scaleStr = vector.getField().getMetadata().get("scale");
int sfScale = Integer.parseInt(scaleStr);
switch (sfScale) {
case 0:
return Types.MinorType.TIMESTAMPSECTZ;
case 3:
return Types.MinorType.TIMESTAMPMILLITZ;
case 6:
return Types.MinorType.TIMESTAMPMICROTZ;
case 9:
return Types.MinorType.TIMESTAMPNANOTZ;
if (sfScale <= 3) {
return Types.MinorType.TIMEMILLI;
}
break;
}
case TIMESTAMP_NTZ:
{
String scaleStr = vector.getField().getMetadata().get("scale");
int sfScale = Integer.parseInt(scaleStr);
switch (sfScale) {
case 0:
return Types.MinorType.TIMESTAMPSEC;
case 3:
return Types.MinorType.TIMESTAMPMILLI;
case 6:
return Types.MinorType.TIMESTAMPMICRO;
case 9:
return Types.MinorType.TIMESTAMPNANO;
if (sfScale <= 6) {
return Types.MinorType.TIMEMICRO;
}
if (sfScale <= 9) {
return Types.MinorType.TIMENANO;
}
break;
}
case TIMESTAMP_NTZ:
return Types.MinorType.TIMESTAMPNANO;
case TIMESTAMP_LTZ:
case TIMESTAMP_TZ:
return Types.MinorType.TIMESTAMPNANOTZ;
}
}
return type;
Expand All @@ -93,6 +66,7 @@ static FieldVector convert(
ValueVector vector,
DataConversionContext context,
SFBaseSession session,
TimeZone timeZoneToUse,
int idx,
Object targetType)
throws SnowflakeSQLException {
Expand All @@ -112,6 +86,29 @@ static FieldVector convert(
return new BigIntVectorConverter(allocator, vector, context, session, idx).convert();
case DECIMAL:
return new DecimalVectorConverter(allocator, vector, context, session, idx).convert();
case FLOAT8:
return new FloatVectorConverter(allocator, vector, context, session, idx).convert();
case BIT:
return new BitVectorConverter(allocator, vector, context, session, idx).convert();
case VARBINARY:
return new BinaryVectorConverter(allocator, vector, context, session, idx).convert();
case DATEDAY:
return new DateVectorConverter(allocator, vector, context, session, idx, timeZoneToUse)
.convert();
case TIMESEC:
return new TimeSecVectorConverter(allocator, vector).convert();
case TIMEMILLI:
return new TimeMilliVectorConverter(allocator, vector).convert();
case TIMEMICRO:
return new TimeMicroVectorConverter(allocator, vector).convert();
case TIMENANO:
return new TimeNanoVectorConverter(allocator, vector).convert();
case TIMESTAMPNANOTZ:
return new TimestampVectorConverter(allocator, vector, context, timeZoneToUse, false)
.convert();
case TIMESTAMPNANO:
return new TimestampVectorConverter(allocator, vector, context, timeZoneToUse, true)
.convert();
}
}
} catch (SFException ex) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
package net.snowflake.client.core.arrow.fullvectorconverters;

import net.snowflake.client.core.DataConversionContext;
import net.snowflake.client.core.SFBaseSession;
import net.snowflake.client.core.SFException;
import net.snowflake.client.core.SnowflakeJdbcInternalApi;
import net.snowflake.client.core.arrow.ArrowVectorConverter;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.ValueVector;
import org.apache.arrow.vector.VarBinaryVector;

@SnowflakeJdbcInternalApi
public class BinaryVectorConverter extends SimpleArrowFullVectorConverter<VarBinaryVector> {
public BinaryVectorConverter(
RootAllocator allocator,
ValueVector vector,
DataConversionContext context,
SFBaseSession session,
int idx) {
super(allocator, vector, context, session, idx);
}

@Override
protected boolean matchingType() {
return vector instanceof VarBinaryVector;
}

@Override
protected VarBinaryVector initVector() {
VarBinaryVector resultVector = new VarBinaryVector(vector.getName(), allocator);
resultVector.allocateNew(vector.getValueCount());
return resultVector;
}

@Override
protected void convertValue(ArrowVectorConverter from, VarBinaryVector to, int idx)
throws SFException {
to.set(idx, from.toBytes(idx));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
package net.snowflake.client.core.arrow.fullvectorconverters;

import net.snowflake.client.core.DataConversionContext;
import net.snowflake.client.core.SFBaseSession;
import net.snowflake.client.core.SFException;
import net.snowflake.client.core.SnowflakeJdbcInternalApi;
import net.snowflake.client.core.arrow.ArrowVectorConverter;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.BitVector;
import org.apache.arrow.vector.ValueVector;

@SnowflakeJdbcInternalApi
public class BitVectorConverter extends SimpleArrowFullVectorConverter<BitVector> {

public BitVectorConverter(
RootAllocator allocator,
ValueVector vector,
DataConversionContext context,
SFBaseSession session,
int idx) {
super(allocator, vector, context, session, idx);
}

@Override
protected boolean matchingType() {
return vector instanceof BitVector;
}

@Override
protected BitVector initVector() {
BitVector resultVector = new BitVector(vector.getName(), allocator);
resultVector.allocateNew(vector.getValueCount());
return resultVector;
}

@Override
protected void convertValue(ArrowVectorConverter from, BitVector to, int idx) throws SFException {
to.set(idx, from.toBoolean(idx) ? 1 : 0);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
package net.snowflake.client.core.arrow.fullvectorconverters;

import java.util.TimeZone;
import net.snowflake.client.core.DataConversionContext;
import net.snowflake.client.core.SFBaseSession;
import net.snowflake.client.core.SFException;
import net.snowflake.client.core.SnowflakeJdbcInternalApi;
import net.snowflake.client.core.arrow.ArrowVectorConverter;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.DateDayVector;
import org.apache.arrow.vector.ValueVector;

@SnowflakeJdbcInternalApi
public class DateVectorConverter extends SimpleArrowFullVectorConverter<DateDayVector> {
private TimeZone timeZone;

public DateVectorConverter(
RootAllocator allocator,
ValueVector vector,
DataConversionContext context,
SFBaseSession session,
int idx,
TimeZone timeZone) {
super(allocator, vector, context, session, idx);
this.timeZone = timeZone;
}

@Override
protected boolean matchingType() {
return vector instanceof DateDayVector;
}

@Override
protected DateDayVector initVector() {
DateDayVector resultVector = new DateDayVector(vector.getName(), allocator);
resultVector.allocateNew(vector.getValueCount());
return resultVector;
}

@Override
protected void additionalConverterInit(ArrowVectorConverter converter) {
if (timeZone != null) {
converter.setSessionTimeZone(timeZone);
converter.setUseSessionTimezone(true);
}
}

@Override
protected void convertValue(ArrowVectorConverter from, DateDayVector to, int idx)
throws SFException {
to.set(idx, (int) (from.toDate(idx, null, false).getTime() / (1000 * 3600 * 24)));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package net.snowflake.client.core.arrow.fullvectorconverters;

import net.snowflake.client.core.DataConversionContext;
import net.snowflake.client.core.SFBaseSession;
import net.snowflake.client.core.SFException;
import net.snowflake.client.core.SnowflakeJdbcInternalApi;
import net.snowflake.client.core.arrow.ArrowVectorConverter;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.Float8Vector;
import org.apache.arrow.vector.ValueVector;

@SnowflakeJdbcInternalApi
public class FloatVectorConverter extends SimpleArrowFullVectorConverter<Float8Vector> {

public FloatVectorConverter(
RootAllocator allocator,
ValueVector vector,
DataConversionContext context,
SFBaseSession session,
int idx) {
super(allocator, vector, context, session, idx);
}

@Override
protected boolean matchingType() {
return vector instanceof Float8Vector;
}

@Override
protected Float8Vector initVector() {
Float8Vector resultVector = new Float8Vector(vector.getName(), allocator);
resultVector.allocateNew(vector.getValueCount());
return resultVector;
}

@Override
protected void convertValue(ArrowVectorConverter from, Float8Vector to, int idx)
throws SFException {
to.set(idx, from.toDouble(idx));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ public SimpleArrowFullVectorConverter(

protected abstract void convertValue(ArrowVectorConverter from, T to, int idx) throws SFException;

protected void additionalConverterInit(ArrowVectorConverter converter) {}

public FieldVector convert() throws SFException, SnowflakeSQLException {
if (matchingType()) {
return (FieldVector) vector;
Expand All @@ -48,10 +50,14 @@ public FieldVector convert() throws SFException, SnowflakeSQLException {
T converted = initVector();
ArrowVectorConverter converter =
ArrowVectorConverterUtil.initConverter(vector, context, session, idx);
additionalConverterInit(converter);
for (int i = 0; i < size; i++) {
convertValue(converter, converted, i);
if (!vector.isNull(i)) {
convertValue(converter, converted, i);
}
}
converted.setValueCount(size);
vector.close();
return converted;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
package net.snowflake.client.core.arrow.fullvectorconverters;

import net.snowflake.client.core.SnowflakeJdbcInternalApi;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.TimeMicroVector;
import org.apache.arrow.vector.ValueVector;

@SnowflakeJdbcInternalApi
public class TimeMicroVectorConverter extends TimeVectorConverter<TimeMicroVector> {

public TimeMicroVectorConverter(RootAllocator allocator, ValueVector vector) {
super(allocator, vector);
}

@Override
protected TimeMicroVector initVector() {
return new TimeMicroVector(vector.getName(), allocator);
}

@Override
protected void convertValue(TimeMicroVector dstVector, int idx, long value) {
dstVector.set(idx, value);
}

@Override
protected int targetScale() {
return 6;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
package net.snowflake.client.core.arrow.fullvectorconverters;

import net.snowflake.client.core.SnowflakeJdbcInternalApi;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.TimeMilliVector;
import org.apache.arrow.vector.ValueVector;

@SnowflakeJdbcInternalApi
public class TimeMilliVectorConverter extends TimeVectorConverter<TimeMilliVector> {
public TimeMilliVectorConverter(RootAllocator allocator, ValueVector vector) {
super(allocator, vector);
}

@Override
protected TimeMilliVector initVector() {
return new TimeMilliVector(vector.getName(), allocator);
}

@Override
protected void convertValue(TimeMilliVector dstVector, int idx, long value) {
dstVector.set(idx, (int) value);
}

@Override
protected int targetScale() {
return 3;
}
}
Loading