From 7284ac54164cb357a3b62827d59c01ba9b6fd45c Mon Sep 17 00:00:00 2001 From: Cindy Peng <148148319+cindy-peng@users.noreply.github.com> Date: Mon, 21 Oct 2024 19:11:43 -0700 Subject: [PATCH 01/33] Add vector value type --- .../com/google/cloud/datastore/ValueType.java | 7 +- .../google/cloud/datastore/VectorValue.java | 156 ++++++++++++++++++ .../com/google/cloud/datastore/ValueTest.java | 3 + 3 files changed, 164 insertions(+), 2 deletions(-) create mode 100644 google-cloud-datastore/src/main/java/com/google/cloud/datastore/VectorValue.java diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/ValueType.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/ValueType.java index 13e3c7af6..df2fb4099 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/ValueType.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/ValueType.java @@ -19,7 +19,7 @@ import com.google.common.collect.ImmutableMap; /** - * The type of a Datastore property. + * The type of Datastore property. * * @see Google @@ -61,7 +61,10 @@ public enum ValueType { RAW_VALUE(RawValue.MARSHALLER), /** Represents a {@link LatLng} value. */ - LAT_LNG(LatLngValue.MARSHALLER); + LAT_LNG(LatLngValue.MARSHALLER), + + /** Represents a {@link Vector} value. */ + VECTOR(VectorValue.MARSHALLER); private static final ImmutableMap DESCRIPTOR_TO_TYPE_MAP; diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/VectorValue.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/VectorValue.java new file mode 100644 index 000000000..42c9a469b --- /dev/null +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/VectorValue.java @@ -0,0 +1,156 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.datastore; + +import static com.google.datastore.v1.Value.ARRAY_VALUE_FIELD_NUMBER; + +import com.google.cloud.Timestamp; +import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; +import java.util.ArrayList; +import java.util.List; + +/** A Google Cloud Datastore Vector value. A list value is a list of {@link Value} objects. */ +//public final class VectorValue extends Value>> { { +public final class VectorValue extends Value>> { + + private static final long serialVersionUID = -5121887228607148859L; + + static final BaseMarshaller>, VectorValue, Builder> MARSHALLER = + new BaseMarshaller>, VectorValue, Builder>() { + private static final long serialVersionUID = 7720473855548179943L; + + @Override + public int getProtoFieldId() { + return ARRAY_VALUE_FIELD_NUMBER; + } + + @Override + public Builder newBuilder(List> values) { + return VectorValue.newBuilder().set(values); + } + + @Override + protected List> getValue(com.google.datastore.v1.Value from) { + List> properties = new ArrayList<>(from.getArrayValue().getValuesCount()); + for (com.google.datastore.v1.Value valuePb : from.getArrayValue().getValuesList()) { + properties.add((Value) Value.fromPb(valuePb)); + } + return properties; + } + + @Override + protected void setValue(VectorValue from, com.google.datastore.v1.Value.Builder to) { + List propertiesPb = new ArrayList<>(); + for (Value property : from.get()) { + propertiesPb.add(property.toPb()); + } + to.setArrayValue( + com.google.datastore.v1.ArrayValue.newBuilder().addAllValues(propertiesPb)); + } + }; + + public static final class Builder + extends Value.BaseBuilder>, VectorValue, Builder> { + private ImmutableList.Builder> vectorBuilder = ImmutableList.builder(); + + private Builder() { + super(ValueType.VECTOR); + } + + + /** Adds the provided double values to the {@code VectorValue} builder. */ + public VectorValue.Builder addValue(Value first, Value... other) { + for (Value value : other) { + vectorBuilder.add(value); + } + return this; + } + + public VectorValue.Builder addValue(double first, double... other) { + vectorBuilder.add(DoubleValue.of(first)); + for (double value : other) { + vectorBuilder.add(DoubleValue.of(value)); + } + return this; + } + + /** + * Sets the list of values of this {@code ListValue} builder to {@code values}. The provided + * list is copied. + * + * @see com.google.cloud.datastore.Value.BaseBuilder#set(java.lang.Object) + */ + @Override + public Builder set(List> values) { + vectorBuilder = ImmutableList.builder(); + for (Value value : values) { + addValue(value); + } + return this; + } + + @Override + public List> get() { + return vectorBuilder.build(); + } + + /** + * Creates a {@code ListValue} object. + */ + @Override + public VectorValue build() { + return new VectorValue(this); + } + } + + + public VectorValue(List> values) { + this(newBuilder().set(values)); + } + + private VectorValue(Builder builder) { + super(builder); + } + + /** + * Returns a builder for the list value object. + */ + @Override + public Builder toBuilder() { + return new Builder().mergeFrom(this); + } + + /** + * Creates a {@code ListValue} object given a number of double values. + */ + public static VectorValue of(double first, double... other) { + return newBuilder().addValue(first, other).build(); + } + + /** + * Returns a builder for {@code ListValue} objects. + */ + public static Builder newBuilder() { + return new Builder(); + } + + public static VectorValue.Builder newBuilder(double first, double... other) { + return new VectorValue.Builder().addValue(first, other); + } + } + diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/ValueTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/ValueTest.java index 8d53dc736..1b171c7f9 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/ValueTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/ValueTest.java @@ -58,6 +58,9 @@ public class ValueTest { .put(ValueType.RAW_VALUE, new Object[] {RawValue.class, RAW_VALUE.get()}) .put(ValueType.LAT_LNG, new Object[] {LatLngValue.class, LAT_LNG_VALUE.get()}) .put(ValueType.STRING, new Object[] {StringValue.class, STRING_VALUE.get()}) + .put( + ValueType.VECTOR, + new Object[] {ListValue.class, ImmutableList.of(1.25D, 1.30D, 1.35D)}) .buildOrThrow(); private ImmutableMap> typeToValue; From fe5e966f2ef3edd00703d6865ff544563456d1e3 Mon Sep 17 00:00:00 2001 From: Cindy Peng <148148319+cindy-peng@users.noreply.github.com> Date: Tue, 22 Oct 2024 16:16:03 -0700 Subject: [PATCH 02/33] Add vectorValue type --- .../google/cloud/datastore/BaseEntity.java | 11 ++++++++ .../com/google/cloud/datastore/Value.java | 19 +++++++++++--- .../google/cloud/datastore/VectorValue.java | 26 +++++++++++-------- .../cloud/datastore/BaseEntityTest.java | 15 ++++++++++- .../com/google/cloud/datastore/ValueTest.java | 6 ++--- 5 files changed, 59 insertions(+), 18 deletions(-) diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/BaseEntity.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/BaseEntity.java index 608dc7187..d50770215 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/BaseEntity.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/BaseEntity.java @@ -615,6 +615,17 @@ public > List getList(String name) { return (List) getValue(name).get(); } + /** + * Returns the property value as a vector. + * + * @throws DatastoreException if no such property + * @throws ClassCastException if value is not a vector + */ + @SuppressWarnings("unchecked") + public List getVector(String name) { + return (List) getValue(name).get(); + } + /** * Returns the property value as a blob. * diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/Value.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/Value.java index 4bd0a5133..1315b8d47 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/Value.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/Value.java @@ -16,6 +16,7 @@ package com.google.cloud.datastore; +import static com.google.cloud.datastore.VectorValue.VECTOR_MEANING; import static com.google.common.base.Preconditions.checkNotNull; import com.google.cloud.GcpLaunchStage; @@ -214,8 +215,20 @@ com.google.datastore.v1.Value toPb() { public static Value fromPb(com.google.datastore.v1.Value proto) { ValueTypeCase descriptorId = proto.getValueTypeCase(); ValueType valueType = ValueType.getByDescriptorId(descriptorId.getNumber()); - return valueType == null - ? RawValue.MARSHALLER.fromProto(proto).build() - : valueType.getMarshaller().fromProto(proto).build(); + if (valueType == null) + return RawValue.MARSHALLER.fromProto(proto).build(); + + Value returnValue = valueType.getMarshaller().fromProto(proto).build(); + if (valueType == ValueType.LIST && proto.getMeaning() == VECTOR_MEANING) + { + for (com.google.datastore.v1.Value item : proto.getArrayValue().getValuesList()) { + if (item.getValueTypeCase() != ValueTypeCase.DOUBLE_VALUE) { + return returnValue; + } + } + returnValue = VectorValue.MARSHALLER.fromProto(proto).build(); + } + + return returnValue; } } diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/VectorValue.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/VectorValue.java index 42c9a469b..a3485f1d7 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/VectorValue.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/VectorValue.java @@ -16,27 +16,24 @@ package com.google.cloud.datastore; -import static com.google.datastore.v1.Value.ARRAY_VALUE_FIELD_NUMBER; - -import com.google.cloud.Timestamp; -import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import java.util.ArrayList; import java.util.List; /** A Google Cloud Datastore Vector value. A list value is a list of {@link Value} objects. */ -//public final class VectorValue extends Value>> { { public final class VectorValue extends Value>> { private static final long serialVersionUID = -5121887228607148859L; + public static final int VECTOR_MEANING = 31; + static final BaseMarshaller>, VectorValue, Builder> MARSHALLER = new BaseMarshaller>, VectorValue, Builder>() { private static final long serialVersionUID = 7720473855548179943L; @Override public int getProtoFieldId() { - return ARRAY_VALUE_FIELD_NUMBER; + return -1; } @Override @@ -72,9 +69,9 @@ private Builder() { super(ValueType.VECTOR); } - /** Adds the provided double values to the {@code VectorValue} builder. */ public VectorValue.Builder addValue(Value first, Value... other) { + vectorBuilder.add(first); for (Value value : other) { vectorBuilder.add(value); } @@ -118,7 +115,6 @@ public VectorValue build() { } } - public VectorValue(List> values) { this(newBuilder().set(values)); } @@ -136,12 +132,17 @@ public Builder toBuilder() { } /** - * Creates a {@code ListValue} object given a number of double values. + * Creates a {@code VectorValue} object given a number of double values. */ public static VectorValue of(double first, double... other) { return newBuilder().addValue(first, other).build(); } + /** Creates a {@code VectorValue} object given a list of {@code Value} objects. */ + public static VectorValue of(List> values) { + return new VectorValue(values); + } + /** * Returns a builder for {@code ListValue} objects. */ @@ -149,8 +150,11 @@ public static Builder newBuilder() { return new Builder(); } - public static VectorValue.Builder newBuilder(double first, double... other) { - return new VectorValue.Builder().addValue(first, other); + public static Builder newBuilder(double first, double... other) { + VectorValue.Builder builder = new VectorValue.Builder(); + builder.setExcludeFromIndexes(true); + builder.setMeaning(VECTOR_MEANING); + return builder.addValue(first, other); } } diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/BaseEntityTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/BaseEntityTest.java index 1b5380ab9..83fb83a2d 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/BaseEntityTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/BaseEntityTest.java @@ -16,6 +16,7 @@ package com.google.cloud.datastore; +import static com.google.cloud.datastore.VectorValue.VECTOR_MEANING; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -35,6 +36,7 @@ public class BaseEntityTest { private static final Blob BLOB = Blob.copyFrom(new byte[] {1, 2}); private static final Timestamp TIMESTAMP = Timestamp.now(); private static final LatLng LAT_LNG = new LatLng(37.422035, -122.084124); + private static final VectorValue VECTOR = VectorValue.newBuilder(1.78, 2.56, 3.88).setMeaning(VECTOR_MEANING).setExcludeFromIndexes(true).build(); private static final Key KEY = Key.newBuilder("ds1", "k1", "n1").build(); private static final Entity ENTITY = Entity.newBuilder(KEY).set("name", "foo").build(); private static final IncompleteKey INCOMPLETE_KEY = IncompleteKey.newBuilder("ds1", "k1").build(); @@ -76,6 +78,7 @@ public void setUp() { builder.set("stringList", "s1", "s2", "s3"); builder.set("longList", 1, 23, 456); builder.set("latLngList", LAT_LNG, LAT_LNG); + builder.set("vector", VECTOR); } @Test @@ -182,6 +185,16 @@ public void testGetEntity() { assertEquals(PARTIAL_ENTITY, entity.getEntity("entity")); } + @Test + public void testGetVector() { + BaseEntity entity = builder.build(); + List vectorList = entity.getVector("vector"); + assertEquals(3, vectorList.size()); + assertEquals(Double.valueOf(1.78), vectorList.get(0).get()); + assertEquals(Double.valueOf(2.56), vectorList.get(1).get()); + assertEquals(Double.valueOf(3.88), vectorList.get(2).get()); + } + @Test public void testGetList() { BaseEntity entity = builder.build(); @@ -229,7 +242,7 @@ public void testNames() { .add("entity", "partialEntity", "null", "timestamp", "blob", "key", "blobList") .add( "booleanList", "timestampList", "doubleList", "keyList", "entityList", "stringList") - .add("longList", "latLng", "latLngList") + .add("longList", "latLng", "latLngList", "vector") .build(); BaseEntity entity = builder.build(); assertEquals(names, entity.getNames()); diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/ValueTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/ValueTest.java index 1b171c7f9..9ab0e7692 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/ValueTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/ValueTest.java @@ -16,6 +16,7 @@ package com.google.cloud.datastore; +import static com.google.cloud.datastore.VectorValue.VECTOR_MEANING; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -42,6 +43,7 @@ public class ValueTest { private static final RawValue RAW_VALUE = RawValue.of(STRING_VALUE.toPb()); private static final LatLngValue LAT_LNG_VALUE = LatLngValue.of(new LatLng(37.422035, -122.084124)); + private static final VectorValue VECTOR_VALUE = VectorValue.newBuilder(1.78, 2.56, 3.88).build(); private static final ImmutableMap TYPES = ImmutableMap.builder() .put(ValueType.NULL, new Object[] {NullValue.class, NULL_VALUE.get()}) @@ -57,10 +59,8 @@ public class ValueTest { .put(ValueType.LONG, new Object[] {LongValue.class, 123L}) .put(ValueType.RAW_VALUE, new Object[] {RawValue.class, RAW_VALUE.get()}) .put(ValueType.LAT_LNG, new Object[] {LatLngValue.class, LAT_LNG_VALUE.get()}) + .put(ValueType.VECTOR, new Object[] {VectorValue.class, VECTOR_VALUE.get()}) .put(ValueType.STRING, new Object[] {StringValue.class, STRING_VALUE.get()}) - .put( - ValueType.VECTOR, - new Object[] {ListValue.class, ImmutableList.of(1.25D, 1.30D, 1.35D)}) .buildOrThrow(); private ImmutableMap> typeToValue; From 9c19afaaf2ff06efb990688c8709459093a431b6 Mon Sep 17 00:00:00 2001 From: Cindy Peng <148148319+cindy-peng@users.noreply.github.com> Date: Tue, 22 Oct 2024 16:56:16 -0700 Subject: [PATCH 03/33] Add vector value test --- .../google/cloud/datastore/VectorValue.java | 5 +- .../cloud/datastore/VectorValueTest.java | 61 +++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 google-cloud-datastore/src/test/java/com/google/cloud/datastore/VectorValueTest.java diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/VectorValue.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/VectorValue.java index a3485f1d7..a700f93d9 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/VectorValue.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/VectorValue.java @@ -147,7 +147,10 @@ public static VectorValue of(List> values) { * Returns a builder for {@code ListValue} objects. */ public static Builder newBuilder() { - return new Builder(); + Builder builder = new VectorValue.Builder(); + builder.setExcludeFromIndexes(true); + builder.setMeaning(VECTOR_MEANING); + return builder; } public static Builder newBuilder(double first, double... other) { diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/VectorValueTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/VectorValueTest.java new file mode 100644 index 000000000..07bacd04d --- /dev/null +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/VectorValueTest.java @@ -0,0 +1,61 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.datastore; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import com.google.common.collect.ImmutableList; +import org.junit.Test; + +import java.util.List; + +public class VectorValueTest { + private static final List> vectorList = + ImmutableList.of(DoubleValue.of(1.2), DoubleValue.of(3.6)); + @Test + public void testToBuilder() { + // StringValue value = StringValue.of(CONTENT); + VectorValue value = VectorValue.of(0.3, 4.2, 3.7); + assertEquals(value, value.toBuilder().build()); + } + + @Test + public void testOf() { + VectorValue value = VectorValue.of(0.3, 4.2, 3.7); + assertEquals(ImmutableList.of(DoubleValue.of(0.3), DoubleValue.of(4.2), DoubleValue.of(3.7)), + value.get()); + assertTrue(value.excludeFromIndexes()); + assertEquals(31, value.getMeaning()); + VectorValue value1 = VectorValue.of(vectorList); + assertEquals(vectorList, + value1.get()); + assertTrue(value1.excludeFromIndexes()); + assertEquals(31, value1.getMeaning()); + } + + @SuppressWarnings("deprecation") + @Test + public void testBuilder() { + VectorValue.Builder builder = VectorValue.newBuilder(0.3, 4.2, 3.7); + VectorValue value = builder.setExcludeFromIndexes(true).build(); + assertEquals(ImmutableList.of(DoubleValue.of(0.3), DoubleValue.of(4.2), DoubleValue.of(3.7)), + value.get()); + assertEquals(31, value.getMeaning()); + assertTrue(value.excludeFromIndexes()); + } +} From 8163c957a978d0dbed0e6a830315b30ac3bac5b1 Mon Sep 17 00:00:00 2001 From: Cindy Peng <148148319+cindy-peng@users.noreply.github.com> Date: Sat, 26 Oct 2024 12:39:46 -0700 Subject: [PATCH 04/33] Add unit tests and system tests --- .../google/cloud/datastore/FindNearest.java | 170 ++++++++++++++++++ .../cloud/datastore/StructuredQuery.java | 29 ++- .../StructuredQueryProtoPreparer.java | 3 + .../google/cloud/datastore/ProtoTestData.java | 28 +++ .../StructuredQueryProtoPreparerTest.java | 12 +- .../cloud/datastore/StructuredQueryTest.java | 10 ++ .../com/google/cloud/datastore/ValueTest.java | 9 +- .../datastore/it/ITDatastoreConceptsTest.java | 58 +++--- 8 files changed, 294 insertions(+), 25 deletions(-) create mode 100644 google-cloud-datastore/src/main/java/com/google/cloud/datastore/FindNearest.java diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/FindNearest.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/FindNearest.java new file mode 100644 index 000000000..d0b6c0a20 --- /dev/null +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/FindNearest.java @@ -0,0 +1,170 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.cloud.datastore; + +import com.google.common.base.MoreObjects; +import com.google.common.base.MoreObjects.ToStringHelper; +import com.google.protobuf.ByteString; +import com.google.protobuf.DoubleValue; +import com.google.protobuf.Int32Value; + +import java.io.Serializable; +import java.util.Objects; + +import javax.annotation.Nullable; + + +/** + * A query that finds the entities whose vector fields are closest to a certain query vector. + * Create an instance of `FindNearest` with {@link Query#findNearest}. + */ +public final class FindNearest implements Serializable { + + private final String vectorProperty; + private final VectorValue queryVector; + private final DistanceMeasure measure; + private final int limit; + /* + Optional. Optional name of the field to output the result of the vector + * distance calculation. + */ + private final @Nullable String distanceResultField; + private final @Nullable Double distanceThreshold; + + private static final long serialVersionUID = 4688656124180403551L; + + /** Creates a VectorQuery */ + public FindNearest(String vectorProperty, VectorValue queryVector, DistanceMeasure measure, int limit, @Nullable String distanceResultField,@Nullable Double distanceThreshold) { + this.vectorProperty = vectorProperty; + this.queryVector = queryVector; + this.measure = measure; + this.limit = limit; + this.distanceResultField = distanceResultField; + this.distanceThreshold = distanceThreshold; + } + + public FindNearest(String vectorProperty, VectorValue queryVector, DistanceMeasure measure, int limit) { + this(vectorProperty, queryVector, measure, limit, null, null); + } + + public FindNearest(String vectorProperty, VectorValue queryVector, DistanceMeasure measure, int limit, @Nullable String distanceResultField) { + this(vectorProperty, queryVector, measure, limit, distanceResultField, null); + } + + public FindNearest(String vectorProperty, VectorValue queryVector, DistanceMeasure measure, int limit, @Nullable Double distanceThreshold) { + this(vectorProperty, queryVector, measure, limit, null, distanceThreshold); + } + + @Override + public int hashCode() { + return Objects.hash(vectorProperty, queryVector, measure, limit, distanceResultField, distanceThreshold); + } + + /** + * Returns true if this VectorQuery is equal to the provided object. + * + * @param obj The object to compare against. + * @return Whether this VectorQuery is equal to the provided object. + */ + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || !(obj instanceof FindNearest)) { + return false; + } + FindNearest otherQuery = (FindNearest) obj; + return Objects.equals(vectorProperty, otherQuery.vectorProperty) + && Objects.equals(queryVector, otherQuery.queryVector) + && Objects.equals(distanceResultField, otherQuery.distanceResultField) + && Objects.equals(distanceThreshold, otherQuery.distanceThreshold) + && limit == otherQuery.limit + && measure == otherQuery.measure; + } + + @Override + public String toString() { + ToStringHelper toStringHelper = MoreObjects.toStringHelper(this); + toStringHelper.add("vectorProperty", vectorProperty); + toStringHelper.add("queryVector", queryVector); + toStringHelper.add("measure", measure); + toStringHelper.add("limit", limit); + toStringHelper.add("distanceResultField", distanceResultField); + toStringHelper.add("distanceThreshold", distanceThreshold); + return toStringHelper.toString(); + } + + static FindNearest fromPb(com.google.datastore.v1.FindNearest findNearestPb) { + String vectorProperty = findNearestPb.getVectorProperty().getName(); + VectorValue queryVector = VectorValue.MARSHALLER.fromProto(findNearestPb.getQueryVector()).build(); + DistanceMeasure distanceMeasure = DistanceMeasure.valueOf(findNearestPb.getDistanceMeasure().toString()); + int limit = findNearestPb.getLimit().getValue(); + String distanceResultField = findNearestPb.getDistanceResultProperty() == null || findNearestPb.getDistanceResultProperty().isEmpty() ? null:findNearestPb.getDistanceResultProperty(); + Double distanceThreshold = findNearestPb.getDistanceThreshold() == null || findNearestPb.getDistanceThreshold() == DoubleValue.getDefaultInstance() ? null : findNearestPb.getDistanceThreshold().getValue(); + return new FindNearest(vectorProperty,queryVector, distanceMeasure, limit, distanceResultField, distanceThreshold); + } + + com.google.datastore.v1.FindNearest toPb() { + com.google.datastore.v1.FindNearest.Builder findNearestPb = com.google.datastore.v1.FindNearest.newBuilder(); + findNearestPb.getVectorPropertyBuilder().setName(vectorProperty); + findNearestPb.setQueryVector(queryVector.toPb()); + findNearestPb.setDistanceMeasure(toProto(measure)); + findNearestPb.setLimit(Int32Value.of(limit)); + if (distanceResultField != null) + { + findNearestPb.setDistanceResultProperty(distanceResultField); + } + if (distanceThreshold != null) { + findNearestPb.setDistanceThreshold(DoubleValue.of(distanceThreshold)); + } + return findNearestPb.build(); + } + + protected static com.google.datastore.v1.FindNearest.DistanceMeasure toProto( + DistanceMeasure distanceMeasure) { + switch (distanceMeasure) { + case COSINE: + return com.google.datastore.v1.FindNearest.DistanceMeasure.COSINE; + case EUCLIDEAN: + return com.google.datastore.v1.FindNearest.DistanceMeasure.EUCLIDEAN; + case DOT_PRODUCT: + return com.google.datastore.v1.FindNearest.DistanceMeasure.DOT_PRODUCT; + default: + return com.google.datastore.v1.FindNearest.DistanceMeasure.UNRECOGNIZED; + } + } + + /** + * The distance measure to use when comparing vectors in a {@link FindNearest query}. + * + * @see com.google.cloud.firestore.Query#findNearest + */ + public enum DistanceMeasure { + /** + * COSINE distance compares vectors based on the angle between them, which allows you to measure + * similarity that isn't based on the vectors' magnitude. We recommend using DOT_PRODUCT with + * unit normalized vectors instead of COSINE distance, which is mathematically equivalent with + * better performance. + */ + COSINE, + /** Measures the EUCLIDEAN distance between the vectors. */ + EUCLIDEAN, + /** Similar to cosine but is affected by the magnitude of the vectors. */ + DOT_PRODUCT + } +} diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/StructuredQuery.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/StructuredQuery.java index 30cd05759..7541dbd75 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/StructuredQuery.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/StructuredQuery.java @@ -101,6 +101,7 @@ public abstract class StructuredQuery extends Query implements RecordQuery private final Cursor endCursor; private final int offset; private final Integer limit; + private final FindNearest findNearest; private final ResultType resultType; @@ -731,6 +732,9 @@ public interface Builder { /** Adds settings to the existing order by clause. */ Builder addOrderBy(OrderBy orderBy, OrderBy... others); + /** Sets the find_nearest for the query. */ + Builder setFindNearest(FindNearest findNearest); + StructuredQuery build(); } @@ -753,6 +757,7 @@ abstract static class BuilderImpl> implements Bui private Cursor endCursor; private int offset; private Integer limit; + private FindNearest findNearest; BuilderImpl(ResultType resultType) { this.resultType = resultType; @@ -770,6 +775,7 @@ abstract static class BuilderImpl> implements Bui endCursor = query.endCursor; offset = query.offset; limit = query.limit; + findNearest = query.findNearest; } @SuppressWarnings("unchecked") @@ -841,6 +847,13 @@ public B addOrderBy(OrderBy orderBy, OrderBy... others) { return self(); } + @Override + public B setFindNearest(FindNearest findNearest) { + Preconditions.checkArgument(findNearest != null, "vector query must not be null"); + this.findNearest = findNearest; + return self(); + } + B clearProjection() { projection.clear(); return self(); @@ -904,6 +917,9 @@ B mergeFrom(com.google.datastore.v1.Query queryPb) { for (com.google.datastore.v1.PropertyReference distinctOnPb : queryPb.getDistinctOnList()) { addDistinctOn(distinctOnPb.getName()); } + if (queryPb.getFindNearest() != null) { + setFindNearest(FindNearest.fromPb(queryPb.getFindNearest())); + } return self(); } } @@ -920,6 +936,7 @@ B mergeFrom(com.google.datastore.v1.Query queryPb) { endCursor = builder.endCursor; offset = builder.offset; limit = builder.limit; + findNearest = builder.findNearest; } @Override @@ -935,6 +952,7 @@ public String toString() { .add("orderBy", orderBy) .add("projection", projection) .add("distinctOn", distinctOn) + .add("findNearest", findNearest) .toString(); } @@ -950,7 +968,8 @@ public int hashCode() { filter, orderBy, projection, - distinctOn); + distinctOn, + findNearest); } @Override @@ -971,7 +990,8 @@ public boolean equals(Object obj) { && Objects.equals(filter, other.filter) && Objects.equals(orderBy, other.orderBy) && Objects.equals(projection, other.projection) - && Objects.equals(distinctOn, other.distinctOn); + && Objects.equals(distinctOn, other.distinctOn) + && Objects.equals(findNearest, other.findNearest); } /** Returns the kind for this query. */ @@ -1023,6 +1043,11 @@ public Integer getLimit() { return limit; } + /** Returns the vector query for this query. */ + public FindNearest getFindNearest() { + return findNearest; + } + public abstract Builder toBuilder(); @InternalApi diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/StructuredQueryProtoPreparer.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/StructuredQueryProtoPreparer.java index fda6f8f4a..c7e39f3d4 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/StructuredQueryProtoPreparer.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/StructuredQueryProtoPreparer.java @@ -60,6 +60,9 @@ public Query prepare(StructuredQuery query) { .build(); queryPb.addProjection(expressionPb); } + if (query.getFindNearest() != null) { + queryPb.setFindNearest(query.getFindNearest().toPb()); + } return queryPb.build(); } diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/ProtoTestData.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/ProtoTestData.java index 8e2ba890a..a5763080f 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/ProtoTestData.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/ProtoTestData.java @@ -27,6 +27,11 @@ import com.google.datastore.v1.PropertyOrder; import com.google.datastore.v1.PropertyReference; import com.google.datastore.v1.Value; +import com.google.protobuf.DoubleValue; +import com.google.protobuf.Int32Value; +import com.google.cloud.datastore.FindNearest.DistanceMeasure; + +import javax.annotation.Nullable; public class ProtoTestData { @@ -83,4 +88,27 @@ public static PropertyOrder propertyOrder(String value) { public static Projection projection(String value) { return Projection.newBuilder().setProperty(propertyReference(value)).build(); } + + public static com.google.datastore.v1.FindNearest FindNearest(String vectorProperty, VectorValue queryVector, DistanceMeasure measure, int limit) { + return FindNearest(vectorProperty, queryVector, measure, limit, null, null); + } + + public static com.google.datastore.v1.FindNearest FindNearest(String vectorProperty, VectorValue queryVector, DistanceMeasure measure, int limit, @Nullable String distanceResultField, @Nullable Double distanceThreshold){ + com.google.datastore.v1.FindNearest.Builder builder = com.google.datastore.v1.FindNearest.newBuilder() + .setVectorProperty(propertyReference(vectorProperty)) + .setQueryVector(queryVector.toPb()) + .setDistanceMeasure(FindNearest.toProto(measure)) + .setLimit(Int32Value.of(limit)); + + if (distanceResultField != null) + { + builder.setDistanceResultProperty(distanceResultField); + } + if (distanceThreshold != null) + { + builder.setDistanceThreshold(DoubleValue.of(distanceThreshold)); + } + + return builder.build(); + } } diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/StructuredQueryProtoPreparerTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/StructuredQueryProtoPreparerTest.java index 60937fc28..a9720d813 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/StructuredQueryProtoPreparerTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/StructuredQueryProtoPreparerTest.java @@ -1,5 +1,5 @@ /* - * Copyright 2022 Google LLC + * Copyright 2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ import static com.google.cloud.datastore.ProtoTestData.propertyFilter; import static com.google.cloud.datastore.ProtoTestData.propertyOrder; import static com.google.cloud.datastore.ProtoTestData.propertyReference; +import static com.google.cloud.datastore.ProtoTestData.FindNearest; import static com.google.cloud.datastore.Query.newEntityQueryBuilder; import static com.google.common.truth.Truth.assertThat; import static com.google.datastore.v1.PropertyFilter.Operator.EQUAL; @@ -86,6 +87,15 @@ public void testFilter() { assertThat(queryProto.getFilter()).isEqualTo(propertyFilter("done", EQUAL, booleanValue(true))); } + @Test + public void testFindNearest() { + VectorValue VECTOR_VALUE = VectorValue.newBuilder(1.78, 2.56, 3.88).build(); + FindNearest FIND_NEAREST = new FindNearest("vector_property", VECTOR_VALUE, FindNearest.DistanceMeasure.COSINE, 1); + Query queryProto = protoPreparer.prepare(newEntityQueryBuilder().setFindNearest(FIND_NEAREST).build()); + assertThat(queryProto.getFindNearest()) + .isEqualTo(FindNearest("vector_property", VECTOR_VALUE, FindNearest.DistanceMeasure.COSINE, 1)); + } + @Test public void testOrderBy() { Query queryProto = diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/StructuredQueryTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/StructuredQueryTest.java index c59337586..c596c9682 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/StructuredQueryTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/StructuredQueryTest.java @@ -50,6 +50,8 @@ public class StructuredQueryTest { private static final String DISTINCT_ON1 = "p6"; private static final String DISTINCT_ON2 = "p7"; private static final List DISTINCT_ON = ImmutableList.of(DISTINCT_ON1, DISTINCT_ON2); + private static final VectorValue VECTOR_VALUE = VectorValue.newBuilder(1.78, 2.56, 3.88).build(); + private static final FindNearest FIND_NEAREST = new FindNearest("vector_property", VECTOR_VALUE, FindNearest.DistanceMeasure.COSINE, 1); private static final EntityQuery ENTITY_QUERY = Query.newEntityQueryBuilder() .setNamespace(NAMESPACE) @@ -60,6 +62,7 @@ public class StructuredQueryTest { .setLimit(LIMIT) .setFilter(AND_FILTER) .setOrderBy(ORDER_BY_1, ORDER_BY_2) + .setFindNearest(FIND_NEAREST) .build(); private static final KeyQuery KEY_QUERY = Query.newKeyQueryBuilder() @@ -71,6 +74,7 @@ public class StructuredQueryTest { .setLimit(LIMIT) .setFilter(OR_FILTER) .setOrderBy(ORDER_BY_1, ORDER_BY_2) + .setFindNearest(FIND_NEAREST) .build(); private static final ProjectionEntityQuery PROJECTION_QUERY = Query.newProjectionEntityQueryBuilder() @@ -82,6 +86,7 @@ public class StructuredQueryTest { .setLimit(LIMIT) .setFilter(AND_FILTER) .setOrderBy(ORDER_BY_1, ORDER_BY_2) + .setFindNearest(FIND_NEAREST) .setProjection(PROJECTION1, PROJECTION2) .setDistinctOn(DISTINCT_ON1, DISTINCT_ON2) .build(); @@ -105,6 +110,7 @@ public void testKeyQueryBuilder() { assertEquals(ORDER_BY, KEY_QUERY.getOrderBy()); assertEquals(ImmutableList.of(StructuredQuery.KEY_PROPERTY_NAME), KEY_QUERY.getProjection()); assertTrue(KEY_QUERY.getDistinctOn().isEmpty()); + assertEquals(LIMIT, KEY_QUERY.getLimit()); } @Test @@ -123,6 +129,7 @@ private void compareBaseBuilderFields(StructuredQuery query) { assertEquals(LIMIT, query.getLimit()); assertEquals(AND_FILTER, query.getFilter()); assertEquals(ORDER_BY, query.getOrderBy()); + assertEquals(FIND_NEAREST, query.getFindNearest()); } @Test @@ -149,6 +156,9 @@ private void compareMergedQuery(StructuredQuery expected, StructuredQuery @Test public void testToAndFromPb() { + EntityQuery a = ENTITY_QUERY; + StructuredQuery pb = StructuredQuery.fromPb( + ResultType.ENTITY, ENTITY_QUERY.getNamespace(), ENTITY_QUERY.toPb()); assertEquals( ENTITY_QUERY, StructuredQuery.fromPb( diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/ValueTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/ValueTest.java index 9ab0e7692..2b4070115 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/ValueTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/ValueTest.java @@ -16,7 +16,6 @@ package com.google.cloud.datastore; -import static com.google.cloud.datastore.VectorValue.VECTOR_MEANING; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; @@ -126,7 +125,13 @@ public void testType() { @Test public void testExcludeFromIndexes() { for (Map.Entry> entry : typeToValue.entrySet()) { - assertFalse(entry.getValue().excludeFromIndexes()); + if (entry.getKey() == ValueType.VECTOR) + { + assertTrue(entry.getValue().excludeFromIndexes()); + } + else { + assertFalse(entry.getValue().excludeFromIndexes()); + } } TestBuilder builder = new TestBuilder(); assertFalse(builder.build().excludeFromIndexes()); diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreConceptsTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreConceptsTest.java index 770065778..0723cb8bc 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreConceptsTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreConceptsTest.java @@ -23,29 +23,10 @@ import static org.junit.Assert.fail; import com.google.cloud.Timestamp; -import com.google.cloud.datastore.Cursor; -import com.google.cloud.datastore.Datastore; -import com.google.cloud.datastore.DatastoreException; -import com.google.cloud.datastore.DatastoreOptions; -import com.google.cloud.datastore.Entity; -import com.google.cloud.datastore.EntityQuery; -import com.google.cloud.datastore.FullEntity; -import com.google.cloud.datastore.IncompleteKey; -import com.google.cloud.datastore.Key; -import com.google.cloud.datastore.KeyFactory; -import com.google.cloud.datastore.KeyQuery; -import com.google.cloud.datastore.ListValue; -import com.google.cloud.datastore.PathElement; -import com.google.cloud.datastore.ProjectionEntity; -import com.google.cloud.datastore.Query; -import com.google.cloud.datastore.QueryResults; -import com.google.cloud.datastore.ReadOption; -import com.google.cloud.datastore.StringValue; -import com.google.cloud.datastore.StructuredQuery; +import com.google.cloud.datastore.*; import com.google.cloud.datastore.StructuredQuery.CompositeFilter; import com.google.cloud.datastore.StructuredQuery.OrderBy; import com.google.cloud.datastore.StructuredQuery.PropertyFilter; -import com.google.cloud.datastore.Transaction; import com.google.cloud.datastore.testing.RemoteDatastoreHelper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -175,6 +156,9 @@ private void setUpQueryTests() { "description", StringValue.newBuilder("Learn Cloud Datastore").setExcludeFromIndexes(true).build()) .set("tag", "fun", "l", "programming", "learn") + .set( + "vector_property", + VectorValue.newBuilder(3.0, 1.0, 2.0).setExcludeFromIndexes(true).build()) .build()); } @@ -591,6 +575,40 @@ public void testEqualAndInequalityRange() { assertValidQuery(query); } + @Test + public void testVectorSearch() { + VectorValue vectorValue = VectorValue.newBuilder(1.78, 2.56, 3.88).build(); + FindNearest vectorQuery = new FindNearest("vector_property", vectorValue, FindNearest.DistanceMeasure.COSINE, 1, "distance"); + + Query query = + Query.newEntityQueryBuilder() + .setFindNearest(vectorQuery) + .build(); + assertValidQuery(query); + } + + @Test + public void testVectorSearchWithEmptyVector() { + VectorValue emptyVector = VectorValue.newBuilder().build(); + FindNearest vectorQuery = new FindNearest("vector_property", emptyVector, FindNearest.DistanceMeasure.EUCLIDEAN, 1); + Query query = + Query.newEntityQueryBuilder() + .setFindNearest(vectorQuery) + .build(); + assertInvalidQuery(query); + } + + @Test + public void testVectorSearchWithUnmatchedVectorSize() { + VectorValue vectorValue = VectorValue.newBuilder(1.78, 2.56, 3.88, 4.33).build(); + FindNearest vectorQuery = new FindNearest("vector_property", vectorValue, FindNearest.DistanceMeasure.DOT_PRODUCT, 1); + Query query = + Query.newEntityQueryBuilder() + .setFindNearest(vectorQuery) + .build(); + assertInvalidQuery(query); + } + @Test public void testInequalitySort() { Query query = From 6df2d8a47451370fd6fa4c191792b18d8576adc4 Mon Sep 17 00:00:00 2001 From: Cindy Peng <148148319+cindy-peng@users.noreply.github.com> Date: Sat, 26 Oct 2024 12:53:20 -0700 Subject: [PATCH 05/33] Fix formatting --- .../google/cloud/datastore/FindNearest.java | 82 +++++-- .../com/google/cloud/datastore/Value.java | 6 +- .../google/cloud/datastore/VectorValue.java | 231 +++++++++--------- .../cloud/datastore/BaseEntityTest.java | 6 +- .../google/cloud/datastore/ProtoTestData.java | 23 +- .../StructuredQueryProtoPreparerTest.java | 11 +- .../cloud/datastore/StructuredQueryTest.java | 7 +- .../com/google/cloud/datastore/ValueTest.java | 6 +- .../cloud/datastore/VectorValueTest.java | 21 +- .../datastore/it/ITDatastoreConceptsTest.java | 25 +- 10 files changed, 222 insertions(+), 196 deletions(-) diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/FindNearest.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/FindNearest.java index d0b6c0a20..855c62d07 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/FindNearest.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/FindNearest.java @@ -18,19 +18,15 @@ import com.google.common.base.MoreObjects; import com.google.common.base.MoreObjects.ToStringHelper; -import com.google.protobuf.ByteString; import com.google.protobuf.DoubleValue; import com.google.protobuf.Int32Value; - import java.io.Serializable; import java.util.Objects; - import javax.annotation.Nullable; - /** - * A query that finds the entities whose vector fields are closest to a certain query vector. - * Create an instance of `FindNearest` with {@link Query#findNearest}. + * A query that finds the entities whose vector fields are closest to a certain query vector. Create + * an instance of `FindNearest` with {@link Query#findNearest}. */ public final class FindNearest implements Serializable { @@ -48,7 +44,13 @@ public final class FindNearest implements Serializable { private static final long serialVersionUID = 4688656124180403551L; /** Creates a VectorQuery */ - public FindNearest(String vectorProperty, VectorValue queryVector, DistanceMeasure measure, int limit, @Nullable String distanceResultField,@Nullable Double distanceThreshold) { + public FindNearest( + String vectorProperty, + VectorValue queryVector, + DistanceMeasure measure, + int limit, + @Nullable String distanceResultField, + @Nullable Double distanceThreshold) { this.vectorProperty = vectorProperty; this.queryVector = queryVector; this.measure = measure; @@ -57,21 +59,33 @@ public FindNearest(String vectorProperty, VectorValue queryVector, DistanceMeasu this.distanceThreshold = distanceThreshold; } - public FindNearest(String vectorProperty, VectorValue queryVector, DistanceMeasure measure, int limit) { + public FindNearest( + String vectorProperty, VectorValue queryVector, DistanceMeasure measure, int limit) { this(vectorProperty, queryVector, measure, limit, null, null); } - public FindNearest(String vectorProperty, VectorValue queryVector, DistanceMeasure measure, int limit, @Nullable String distanceResultField) { + public FindNearest( + String vectorProperty, + VectorValue queryVector, + DistanceMeasure measure, + int limit, + @Nullable String distanceResultField) { this(vectorProperty, queryVector, measure, limit, distanceResultField, null); } - public FindNearest(String vectorProperty, VectorValue queryVector, DistanceMeasure measure, int limit, @Nullable Double distanceThreshold) { + public FindNearest( + String vectorProperty, + VectorValue queryVector, + DistanceMeasure measure, + int limit, + @Nullable Double distanceThreshold) { this(vectorProperty, queryVector, measure, limit, null, distanceThreshold); } @Override public int hashCode() { - return Objects.hash(vectorProperty, queryVector, measure, limit, distanceResultField, distanceThreshold); + return Objects.hash( + vectorProperty, queryVector, measure, limit, distanceResultField, distanceThreshold); } /** @@ -90,11 +104,11 @@ public boolean equals(Object obj) { } FindNearest otherQuery = (FindNearest) obj; return Objects.equals(vectorProperty, otherQuery.vectorProperty) - && Objects.equals(queryVector, otherQuery.queryVector) - && Objects.equals(distanceResultField, otherQuery.distanceResultField) - && Objects.equals(distanceThreshold, otherQuery.distanceThreshold) - && limit == otherQuery.limit - && measure == otherQuery.measure; + && Objects.equals(queryVector, otherQuery.queryVector) + && Objects.equals(distanceResultField, otherQuery.distanceResultField) + && Objects.equals(distanceThreshold, otherQuery.distanceThreshold) + && limit == otherQuery.limit + && measure == otherQuery.measure; } @Override @@ -111,22 +125,38 @@ public String toString() { static FindNearest fromPb(com.google.datastore.v1.FindNearest findNearestPb) { String vectorProperty = findNearestPb.getVectorProperty().getName(); - VectorValue queryVector = VectorValue.MARSHALLER.fromProto(findNearestPb.getQueryVector()).build(); - DistanceMeasure distanceMeasure = DistanceMeasure.valueOf(findNearestPb.getDistanceMeasure().toString()); + VectorValue queryVector = + VectorValue.MARSHALLER.fromProto(findNearestPb.getQueryVector()).build(); + DistanceMeasure distanceMeasure = + DistanceMeasure.valueOf(findNearestPb.getDistanceMeasure().toString()); int limit = findNearestPb.getLimit().getValue(); - String distanceResultField = findNearestPb.getDistanceResultProperty() == null || findNearestPb.getDistanceResultProperty().isEmpty() ? null:findNearestPb.getDistanceResultProperty(); - Double distanceThreshold = findNearestPb.getDistanceThreshold() == null || findNearestPb.getDistanceThreshold() == DoubleValue.getDefaultInstance() ? null : findNearestPb.getDistanceThreshold().getValue(); - return new FindNearest(vectorProperty,queryVector, distanceMeasure, limit, distanceResultField, distanceThreshold); + String distanceResultField = + findNearestPb.getDistanceResultProperty() == null + || findNearestPb.getDistanceResultProperty().isEmpty() + ? null + : findNearestPb.getDistanceResultProperty(); + Double distanceThreshold = + findNearestPb.getDistanceThreshold() == null + || findNearestPb.getDistanceThreshold() == DoubleValue.getDefaultInstance() + ? null + : findNearestPb.getDistanceThreshold().getValue(); + return new FindNearest( + vectorProperty, + queryVector, + distanceMeasure, + limit, + distanceResultField, + distanceThreshold); } com.google.datastore.v1.FindNearest toPb() { - com.google.datastore.v1.FindNearest.Builder findNearestPb = com.google.datastore.v1.FindNearest.newBuilder(); + com.google.datastore.v1.FindNearest.Builder findNearestPb = + com.google.datastore.v1.FindNearest.newBuilder(); findNearestPb.getVectorPropertyBuilder().setName(vectorProperty); findNearestPb.setQueryVector(queryVector.toPb()); findNearestPb.setDistanceMeasure(toProto(measure)); findNearestPb.setLimit(Int32Value.of(limit)); - if (distanceResultField != null) - { + if (distanceResultField != null) { findNearestPb.setDistanceResultProperty(distanceResultField); } if (distanceThreshold != null) { @@ -136,7 +166,7 @@ com.google.datastore.v1.FindNearest toPb() { } protected static com.google.datastore.v1.FindNearest.DistanceMeasure toProto( - DistanceMeasure distanceMeasure) { + DistanceMeasure distanceMeasure) { switch (distanceMeasure) { case COSINE: return com.google.datastore.v1.FindNearest.DistanceMeasure.COSINE; @@ -152,7 +182,7 @@ protected static com.google.datastore.v1.FindNearest.DistanceMeasure toProto( /** * The distance measure to use when comparing vectors in a {@link FindNearest query}. * - * @see com.google.cloud.firestore.Query#findNearest + * @see com.google.cloud.datastore.Query#findNearest */ public enum DistanceMeasure { /** diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/Value.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/Value.java index 1315b8d47..93bd4298c 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/Value.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/Value.java @@ -215,12 +215,10 @@ com.google.datastore.v1.Value toPb() { public static Value fromPb(com.google.datastore.v1.Value proto) { ValueTypeCase descriptorId = proto.getValueTypeCase(); ValueType valueType = ValueType.getByDescriptorId(descriptorId.getNumber()); - if (valueType == null) - return RawValue.MARSHALLER.fromProto(proto).build(); + if (valueType == null) return RawValue.MARSHALLER.fromProto(proto).build(); Value returnValue = valueType.getMarshaller().fromProto(proto).build(); - if (valueType == ValueType.LIST && proto.getMeaning() == VECTOR_MEANING) - { + if (valueType == ValueType.LIST && proto.getMeaning() == VECTOR_MEANING) { for (com.google.datastore.v1.Value item : proto.getArrayValue().getValuesList()) { if (item.getValueTypeCase() != ValueTypeCase.DOUBLE_VALUE) { return returnValue; diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/VectorValue.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/VectorValue.java index a700f93d9..38a2903dc 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/VectorValue.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/VectorValue.java @@ -23,141 +23,132 @@ /** A Google Cloud Datastore Vector value. A list value is a list of {@link Value} objects. */ public final class VectorValue extends Value>> { - private static final long serialVersionUID = -5121887228607148859L; - - public static final int VECTOR_MEANING = 31; - - static final BaseMarshaller>, VectorValue, Builder> MARSHALLER = - new BaseMarshaller>, VectorValue, Builder>() { - private static final long serialVersionUID = 7720473855548179943L; - - @Override - public int getProtoFieldId() { - return -1; - } - - @Override - public Builder newBuilder(List> values) { - return VectorValue.newBuilder().set(values); - } - - @Override - protected List> getValue(com.google.datastore.v1.Value from) { - List> properties = new ArrayList<>(from.getArrayValue().getValuesCount()); - for (com.google.datastore.v1.Value valuePb : from.getArrayValue().getValuesList()) { - properties.add((Value) Value.fromPb(valuePb)); - } - return properties; - } - - @Override - protected void setValue(VectorValue from, com.google.datastore.v1.Value.Builder to) { - List propertiesPb = new ArrayList<>(); - for (Value property : from.get()) { - propertiesPb.add(property.toPb()); - } - to.setArrayValue( - com.google.datastore.v1.ArrayValue.newBuilder().addAllValues(propertiesPb)); - } - }; - - public static final class Builder - extends Value.BaseBuilder>, VectorValue, Builder> { - private ImmutableList.Builder> vectorBuilder = ImmutableList.builder(); - - private Builder() { - super(ValueType.VECTOR); - } + private static final long serialVersionUID = -5121887228607148859L; - /** Adds the provided double values to the {@code VectorValue} builder. */ - public VectorValue.Builder addValue(Value first, Value... other) { - vectorBuilder.add(first); - for (Value value : other) { - vectorBuilder.add(value); - } - return this; - } + public static final int VECTOR_MEANING = 31; - public VectorValue.Builder addValue(double first, double... other) { - vectorBuilder.add(DoubleValue.of(first)); - for (double value : other) { - vectorBuilder.add(DoubleValue.of(value)); - } - return this; - } + static final BaseMarshaller>, VectorValue, Builder> MARSHALLER = + new BaseMarshaller>, VectorValue, Builder>() { + private static final long serialVersionUID = 7720473855548179943L; - /** - * Sets the list of values of this {@code ListValue} builder to {@code values}. The provided - * list is copied. - * - * @see com.google.cloud.datastore.Value.BaseBuilder#set(java.lang.Object) - */ - @Override - public Builder set(List> values) { - vectorBuilder = ImmutableList.builder(); - for (Value value : values) { - addValue(value); + @Override + public int getProtoFieldId() { + return -1; } - return this; - } - @Override - public List> get() { - return vectorBuilder.build(); - } + @Override + public Builder newBuilder(List> values) { + return VectorValue.newBuilder().set(values); + } - /** - * Creates a {@code ListValue} object. - */ - @Override - public VectorValue build() { - return new VectorValue(this); - } - } + @Override + protected List> getValue(com.google.datastore.v1.Value from) { + List> properties = new ArrayList<>(from.getArrayValue().getValuesCount()); + for (com.google.datastore.v1.Value valuePb : from.getArrayValue().getValuesList()) { + properties.add((Value) Value.fromPb(valuePb)); + } + return properties; + } - public VectorValue(List> values) { - this(newBuilder().set(values)); - } + @Override + protected void setValue(VectorValue from, com.google.datastore.v1.Value.Builder to) { + List propertiesPb = new ArrayList<>(); + for (Value property : from.get()) { + propertiesPb.add(property.toPb()); + } + to.setArrayValue( + com.google.datastore.v1.ArrayValue.newBuilder().addAllValues(propertiesPb)); + } + }; - private VectorValue(Builder builder) { - super(builder); - } + public static final class Builder + extends Value.BaseBuilder>, VectorValue, Builder> { + private ImmutableList.Builder> vectorBuilder = ImmutableList.builder(); - /** - * Returns a builder for the list value object. - */ - @Override - public Builder toBuilder() { - return new Builder().mergeFrom(this); - } + private Builder() { + super(ValueType.VECTOR); + } - /** - * Creates a {@code VectorValue} object given a number of double values. - */ - public static VectorValue of(double first, double... other) { - return newBuilder().addValue(first, other).build(); + /** Adds the provided double values to the {@code VectorValue} builder. */ + public VectorValue.Builder addValue(Value first, Value... other) { + vectorBuilder.add(first); + for (Value value : other) { + vectorBuilder.add(value); } + return this; + } - /** Creates a {@code VectorValue} object given a list of {@code Value} objects. */ - public static VectorValue of(List> values) { - return new VectorValue(values); + public VectorValue.Builder addValue(double first, double... other) { + vectorBuilder.add(DoubleValue.of(first)); + for (double value : other) { + vectorBuilder.add(DoubleValue.of(value)); } + return this; + } - /** - * Returns a builder for {@code ListValue} objects. - */ - public static Builder newBuilder() { - Builder builder = new VectorValue.Builder(); - builder.setExcludeFromIndexes(true); - builder.setMeaning(VECTOR_MEANING); - return builder; + /** + * Sets the list of values of this {@code ListValue} builder to {@code values}. The provided + * list is copied. + * + * @see com.google.cloud.datastore.Value.BaseBuilder#set(java.lang.Object) + */ + @Override + public Builder set(List> values) { + vectorBuilder = ImmutableList.builder(); + for (Value value : values) { + addValue(value); } + return this; + } - public static Builder newBuilder(double first, double... other) { - VectorValue.Builder builder = new VectorValue.Builder(); - builder.setExcludeFromIndexes(true); - builder.setMeaning(VECTOR_MEANING); - return builder.addValue(first, other); - } + @Override + public List> get() { + return vectorBuilder.build(); } + /** Creates a {@code ListValue} object. */ + @Override + public VectorValue build() { + return new VectorValue(this); + } + } + + public VectorValue(List> values) { + this(newBuilder().set(values)); + } + + private VectorValue(Builder builder) { + super(builder); + } + + /** Returns a builder for the list value object. */ + @Override + public Builder toBuilder() { + return new Builder().mergeFrom(this); + } + + /** Creates a {@code VectorValue} object given a number of double values. */ + public static VectorValue of(double first, double... other) { + return newBuilder().addValue(first, other).build(); + } + + /** Creates a {@code VectorValue} object given a list of {@code Value} objects. */ + public static VectorValue of(List> values) { + return new VectorValue(values); + } + + /** Returns a builder for {@code ListValue} objects. */ + public static Builder newBuilder() { + Builder builder = new VectorValue.Builder(); + builder.setExcludeFromIndexes(true); + builder.setMeaning(VECTOR_MEANING); + return builder; + } + + public static Builder newBuilder(double first, double... other) { + VectorValue.Builder builder = new VectorValue.Builder(); + builder.setExcludeFromIndexes(true); + builder.setMeaning(VECTOR_MEANING); + return builder.addValue(first, other); + } +} diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/BaseEntityTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/BaseEntityTest.java index 83fb83a2d..fc33c64b0 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/BaseEntityTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/BaseEntityTest.java @@ -36,7 +36,11 @@ public class BaseEntityTest { private static final Blob BLOB = Blob.copyFrom(new byte[] {1, 2}); private static final Timestamp TIMESTAMP = Timestamp.now(); private static final LatLng LAT_LNG = new LatLng(37.422035, -122.084124); - private static final VectorValue VECTOR = VectorValue.newBuilder(1.78, 2.56, 3.88).setMeaning(VECTOR_MEANING).setExcludeFromIndexes(true).build(); + private static final VectorValue VECTOR = + VectorValue.newBuilder(1.78, 2.56, 3.88) + .setMeaning(VECTOR_MEANING) + .setExcludeFromIndexes(true) + .build(); private static final Key KEY = Key.newBuilder("ds1", "k1", "n1").build(); private static final Entity ENTITY = Entity.newBuilder(KEY).set("name", "foo").build(); private static final IncompleteKey INCOMPLETE_KEY = IncompleteKey.newBuilder("ds1", "k1").build(); diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/ProtoTestData.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/ProtoTestData.java index a5763080f..57f71039c 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/ProtoTestData.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/ProtoTestData.java @@ -17,6 +17,7 @@ import static com.google.datastore.v1.PropertyOrder.Direction.ASCENDING; +import com.google.cloud.datastore.FindNearest.DistanceMeasure; import com.google.datastore.v1.AggregationQuery.Aggregation; import com.google.datastore.v1.AggregationQuery.Aggregation.Count; import com.google.datastore.v1.Filter; @@ -29,8 +30,6 @@ import com.google.datastore.v1.Value; import com.google.protobuf.DoubleValue; import com.google.protobuf.Int32Value; -import com.google.cloud.datastore.FindNearest.DistanceMeasure; - import javax.annotation.Nullable; public class ProtoTestData { @@ -89,23 +88,29 @@ public static Projection projection(String value) { return Projection.newBuilder().setProperty(propertyReference(value)).build(); } - public static com.google.datastore.v1.FindNearest FindNearest(String vectorProperty, VectorValue queryVector, DistanceMeasure measure, int limit) { + public static com.google.datastore.v1.FindNearest FindNearest( + String vectorProperty, VectorValue queryVector, DistanceMeasure measure, int limit) { return FindNearest(vectorProperty, queryVector, measure, limit, null, null); } - public static com.google.datastore.v1.FindNearest FindNearest(String vectorProperty, VectorValue queryVector, DistanceMeasure measure, int limit, @Nullable String distanceResultField, @Nullable Double distanceThreshold){ - com.google.datastore.v1.FindNearest.Builder builder = com.google.datastore.v1.FindNearest.newBuilder() + public static com.google.datastore.v1.FindNearest FindNearest( + String vectorProperty, + VectorValue queryVector, + DistanceMeasure measure, + int limit, + @Nullable String distanceResultField, + @Nullable Double distanceThreshold) { + com.google.datastore.v1.FindNearest.Builder builder = + com.google.datastore.v1.FindNearest.newBuilder() .setVectorProperty(propertyReference(vectorProperty)) .setQueryVector(queryVector.toPb()) .setDistanceMeasure(FindNearest.toProto(measure)) .setLimit(Int32Value.of(limit)); - if (distanceResultField != null) - { + if (distanceResultField != null) { builder.setDistanceResultProperty(distanceResultField); } - if (distanceThreshold != null) - { + if (distanceThreshold != null) { builder.setDistanceThreshold(DoubleValue.of(distanceThreshold)); } diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/StructuredQueryProtoPreparerTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/StructuredQueryProtoPreparerTest.java index a9720d813..549a8876b 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/StructuredQueryProtoPreparerTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/StructuredQueryProtoPreparerTest.java @@ -15,12 +15,12 @@ */ package com.google.cloud.datastore; +import static com.google.cloud.datastore.ProtoTestData.FindNearest; import static com.google.cloud.datastore.ProtoTestData.booleanValue; import static com.google.cloud.datastore.ProtoTestData.projection; import static com.google.cloud.datastore.ProtoTestData.propertyFilter; import static com.google.cloud.datastore.ProtoTestData.propertyOrder; import static com.google.cloud.datastore.ProtoTestData.propertyReference; -import static com.google.cloud.datastore.ProtoTestData.FindNearest; import static com.google.cloud.datastore.Query.newEntityQueryBuilder; import static com.google.common.truth.Truth.assertThat; import static com.google.datastore.v1.PropertyFilter.Operator.EQUAL; @@ -90,10 +90,13 @@ public void testFilter() { @Test public void testFindNearest() { VectorValue VECTOR_VALUE = VectorValue.newBuilder(1.78, 2.56, 3.88).build(); - FindNearest FIND_NEAREST = new FindNearest("vector_property", VECTOR_VALUE, FindNearest.DistanceMeasure.COSINE, 1); - Query queryProto = protoPreparer.prepare(newEntityQueryBuilder().setFindNearest(FIND_NEAREST).build()); + FindNearest FIND_NEAREST = + new FindNearest("vector_property", VECTOR_VALUE, FindNearest.DistanceMeasure.COSINE, 1); + Query queryProto = + protoPreparer.prepare(newEntityQueryBuilder().setFindNearest(FIND_NEAREST).build()); assertThat(queryProto.getFindNearest()) - .isEqualTo(FindNearest("vector_property", VECTOR_VALUE, FindNearest.DistanceMeasure.COSINE, 1)); + .isEqualTo( + FindNearest("vector_property", VECTOR_VALUE, FindNearest.DistanceMeasure.COSINE, 1)); } @Test diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/StructuredQueryTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/StructuredQueryTest.java index c596c9682..c23bd3a9c 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/StructuredQueryTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/StructuredQueryTest.java @@ -51,7 +51,8 @@ public class StructuredQueryTest { private static final String DISTINCT_ON2 = "p7"; private static final List DISTINCT_ON = ImmutableList.of(DISTINCT_ON1, DISTINCT_ON2); private static final VectorValue VECTOR_VALUE = VectorValue.newBuilder(1.78, 2.56, 3.88).build(); - private static final FindNearest FIND_NEAREST = new FindNearest("vector_property", VECTOR_VALUE, FindNearest.DistanceMeasure.COSINE, 1); + private static final FindNearest FIND_NEAREST = + new FindNearest("vector_property", VECTOR_VALUE, FindNearest.DistanceMeasure.COSINE, 1); private static final EntityQuery ENTITY_QUERY = Query.newEntityQueryBuilder() .setNamespace(NAMESPACE) @@ -157,8 +158,8 @@ private void compareMergedQuery(StructuredQuery expected, StructuredQuery @Test public void testToAndFromPb() { EntityQuery a = ENTITY_QUERY; - StructuredQuery pb = StructuredQuery.fromPb( - ResultType.ENTITY, ENTITY_QUERY.getNamespace(), ENTITY_QUERY.toPb()); + StructuredQuery pb = + StructuredQuery.fromPb(ResultType.ENTITY, ENTITY_QUERY.getNamespace(), ENTITY_QUERY.toPb()); assertEquals( ENTITY_QUERY, StructuredQuery.fromPb( diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/ValueTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/ValueTest.java index 2b4070115..dd36919a1 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/ValueTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/ValueTest.java @@ -125,11 +125,9 @@ public void testType() { @Test public void testExcludeFromIndexes() { for (Map.Entry> entry : typeToValue.entrySet()) { - if (entry.getKey() == ValueType.VECTOR) - { + if (entry.getKey() == ValueType.VECTOR) { assertTrue(entry.getValue().excludeFromIndexes()); - } - else { + } else { assertFalse(entry.getValue().excludeFromIndexes()); } } diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/VectorValueTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/VectorValueTest.java index 07bacd04d..d95e740a0 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/VectorValueTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/VectorValueTest.java @@ -20,16 +20,16 @@ import static org.junit.Assert.assertTrue; import com.google.common.collect.ImmutableList; -import org.junit.Test; - import java.util.List; +import org.junit.Test; public class VectorValueTest { private static final List> vectorList = - ImmutableList.of(DoubleValue.of(1.2), DoubleValue.of(3.6)); + ImmutableList.of(DoubleValue.of(1.2), DoubleValue.of(3.6)); + @Test public void testToBuilder() { - // StringValue value = StringValue.of(CONTENT); + // StringValue value = StringValue.of(CONTENT); VectorValue value = VectorValue.of(0.3, 4.2, 3.7); assertEquals(value, value.toBuilder().build()); } @@ -37,13 +37,13 @@ public void testToBuilder() { @Test public void testOf() { VectorValue value = VectorValue.of(0.3, 4.2, 3.7); - assertEquals(ImmutableList.of(DoubleValue.of(0.3), DoubleValue.of(4.2), DoubleValue.of(3.7)), - value.get()); + assertEquals( + ImmutableList.of(DoubleValue.of(0.3), DoubleValue.of(4.2), DoubleValue.of(3.7)), + value.get()); assertTrue(value.excludeFromIndexes()); assertEquals(31, value.getMeaning()); VectorValue value1 = VectorValue.of(vectorList); - assertEquals(vectorList, - value1.get()); + assertEquals(vectorList, value1.get()); assertTrue(value1.excludeFromIndexes()); assertEquals(31, value1.getMeaning()); } @@ -53,8 +53,9 @@ public void testOf() { public void testBuilder() { VectorValue.Builder builder = VectorValue.newBuilder(0.3, 4.2, 3.7); VectorValue value = builder.setExcludeFromIndexes(true).build(); - assertEquals(ImmutableList.of(DoubleValue.of(0.3), DoubleValue.of(4.2), DoubleValue.of(3.7)), - value.get()); + assertEquals( + ImmutableList.of(DoubleValue.of(0.3), DoubleValue.of(4.2), DoubleValue.of(3.7)), + value.get()); assertEquals(31, value.getMeaning()); assertTrue(value.excludeFromIndexes()); } diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreConceptsTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreConceptsTest.java index 0723cb8bc..d96b083ef 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreConceptsTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreConceptsTest.java @@ -578,34 +578,29 @@ public void testEqualAndInequalityRange() { @Test public void testVectorSearch() { VectorValue vectorValue = VectorValue.newBuilder(1.78, 2.56, 3.88).build(); - FindNearest vectorQuery = new FindNearest("vector_property", vectorValue, FindNearest.DistanceMeasure.COSINE, 1, "distance"); + FindNearest vectorQuery = + new FindNearest( + "vector_property", vectorValue, FindNearest.DistanceMeasure.COSINE, 1, "distance"); - Query query = - Query.newEntityQueryBuilder() - .setFindNearest(vectorQuery) - .build(); + Query query = Query.newEntityQueryBuilder().setFindNearest(vectorQuery).build(); assertValidQuery(query); } @Test public void testVectorSearchWithEmptyVector() { VectorValue emptyVector = VectorValue.newBuilder().build(); - FindNearest vectorQuery = new FindNearest("vector_property", emptyVector, FindNearest.DistanceMeasure.EUCLIDEAN, 1); - Query query = - Query.newEntityQueryBuilder() - .setFindNearest(vectorQuery) - .build(); + FindNearest vectorQuery = + new FindNearest("vector_property", emptyVector, FindNearest.DistanceMeasure.EUCLIDEAN, 1); + Query query = Query.newEntityQueryBuilder().setFindNearest(vectorQuery).build(); assertInvalidQuery(query); } @Test public void testVectorSearchWithUnmatchedVectorSize() { VectorValue vectorValue = VectorValue.newBuilder(1.78, 2.56, 3.88, 4.33).build(); - FindNearest vectorQuery = new FindNearest("vector_property", vectorValue, FindNearest.DistanceMeasure.DOT_PRODUCT, 1); - Query query = - Query.newEntityQueryBuilder() - .setFindNearest(vectorQuery) - .build(); + FindNearest vectorQuery = + new FindNearest("vector_property", vectorValue, FindNearest.DistanceMeasure.DOT_PRODUCT, 1); + Query query = Query.newEntityQueryBuilder().setFindNearest(vectorQuery).build(); assertInvalidQuery(query); } From 32fdd4102cfec7bb4da81e4624f328dc37d3df62 Mon Sep 17 00:00:00 2001 From: Cindy Peng <148148319+cindy-peng@users.noreply.github.com> Date: Mon, 28 Oct 2024 10:56:26 -0700 Subject: [PATCH 06/33] Fix empty FindNearest pb instance --- .../src/main/java/com/google/cloud/datastore/FindNearest.java | 1 + .../main/java/com/google/cloud/datastore/StructuredQuery.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/FindNearest.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/FindNearest.java index 855c62d07..ea32db89c 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/FindNearest.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/FindNearest.java @@ -191,6 +191,7 @@ public enum DistanceMeasure { * unit normalized vectors instead of COSINE distance, which is mathematically equivalent with * better performance. */ + DISTANCE_MEASURE_UNSPECIFIED, COSINE, /** Measures the EUCLIDEAN distance between the vectors. */ EUCLIDEAN, diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/StructuredQuery.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/StructuredQuery.java index 7541dbd75..80732b037 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/StructuredQuery.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/StructuredQuery.java @@ -917,7 +917,7 @@ B mergeFrom(com.google.datastore.v1.Query queryPb) { for (com.google.datastore.v1.PropertyReference distinctOnPb : queryPb.getDistinctOnList()) { addDistinctOn(distinctOnPb.getName()); } - if (queryPb.getFindNearest() != null) { + if (queryPb.getFindNearest() != null && queryPb.getFindNearest() != com.google.datastore.v1.FindNearest.getDefaultInstance()) { setFindNearest(FindNearest.fromPb(queryPb.getFindNearest())); } return self(); From 2b113992bce3f7f72609a53e681dd8d63a56083d Mon Sep 17 00:00:00 2001 From: Cindy Peng <148148319+cindy-peng@users.noreply.github.com> Date: Mon, 28 Oct 2024 11:36:09 -0700 Subject: [PATCH 07/33] Fix formatting --- .../main/java/com/google/cloud/datastore/StructuredQuery.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/StructuredQuery.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/StructuredQuery.java index 80732b037..bd6b9f222 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/StructuredQuery.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/StructuredQuery.java @@ -917,7 +917,8 @@ B mergeFrom(com.google.datastore.v1.Query queryPb) { for (com.google.datastore.v1.PropertyReference distinctOnPb : queryPb.getDistinctOnList()) { addDistinctOn(distinctOnPb.getName()); } - if (queryPb.getFindNearest() != null && queryPb.getFindNearest() != com.google.datastore.v1.FindNearest.getDefaultInstance()) { + if (queryPb.getFindNearest() != null + && queryPb.getFindNearest() != com.google.datastore.v1.FindNearest.getDefaultInstance()) { setFindNearest(FindNearest.fromPb(queryPb.getFindNearest())); } return self(); From 6165685fda0cfbb6b10a2ec727cf1e984da2d49b Mon Sep 17 00:00:00 2001 From: Cindy Peng <148148319+cindy-peng@users.noreply.github.com> Date: Mon, 28 Oct 2024 12:48:48 -0700 Subject: [PATCH 08/33] Fix javadoc --- .../datastore/snippets/ConceptsTest.java | 56 ++++++++++++------- .../google/cloud/datastore/FindNearest.java | 32 ++++++----- .../com/google/cloud/datastore/ValueType.java | 2 +- 3 files changed, 56 insertions(+), 34 deletions(-) rename {samples/snippets/src/test/java/com => com}/google/datastore/snippets/ConceptsTest.java (96%) diff --git a/samples/snippets/src/test/java/com/google/datastore/snippets/ConceptsTest.java b/com/google/datastore/snippets/ConceptsTest.java similarity index 96% rename from samples/snippets/src/test/java/com/google/datastore/snippets/ConceptsTest.java rename to com/google/datastore/snippets/ConceptsTest.java index 1397728ba..30c646f4d 100644 --- a/samples/snippets/src/test/java/com/google/datastore/snippets/ConceptsTest.java +++ b/com/google/datastore/snippets/ConceptsTest.java @@ -24,29 +24,10 @@ import static org.junit.Assert.assertNull; import com.google.cloud.Timestamp; -import com.google.cloud.datastore.Cursor; -import com.google.cloud.datastore.Datastore; -import com.google.cloud.datastore.DatastoreException; -import com.google.cloud.datastore.DatastoreOptions; -import com.google.cloud.datastore.Entity; -import com.google.cloud.datastore.EntityQuery; -import com.google.cloud.datastore.FullEntity; -import com.google.cloud.datastore.IncompleteKey; -import com.google.cloud.datastore.Key; -import com.google.cloud.datastore.KeyFactory; -import com.google.cloud.datastore.KeyQuery; -import com.google.cloud.datastore.ListValue; -import com.google.cloud.datastore.PathElement; -import com.google.cloud.datastore.ProjectionEntity; -import com.google.cloud.datastore.Query; -import com.google.cloud.datastore.QueryResults; -import com.google.cloud.datastore.ReadOption; -import com.google.cloud.datastore.StringValue; -import com.google.cloud.datastore.StructuredQuery; +import com.google.cloud.datastore.*; import com.google.cloud.datastore.StructuredQuery.CompositeFilter; import com.google.cloud.datastore.StructuredQuery.OrderBy; import com.google.cloud.datastore.StructuredQuery.PropertyFilter; -import com.google.cloud.datastore.Transaction; import com.google.cloud.datastore.testing.LocalDatastoreHelper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -399,6 +380,9 @@ private void setUpQueryTests() { "description", StringValue.newBuilder("Learn Cloud Datastore").setExcludeFromIndexes(true).build()) .set("tag", "fun", "l", "programming", "learn") + .set( + "vector_property", + VectorValue.newBuilder(3.0, 1.0, 2.0).setExcludeFromIndexes(true).build()) .build()); } @@ -717,6 +701,38 @@ public void testInequalitySortInvalidNotFirst() { assertInvalidQuery(query); } + @Test + public void testVectorSearch() { + setUpQueryTests(); + VectorValue vectorValue = VectorValue.newBuilder(1.78, 2.56, 3.88).build(); + FindNearest vectorQuery = + new FindNearest( + "vector_property", vectorValue, FindNearest.DistanceMeasure.COSINE, 1, "distance"); + + Query query = Query.newEntityQueryBuilder().setFindNearest(vectorQuery).build(); + assertValidQuery(query); + } + + @Test + public void testVectorSearchWithEmptyVector() { + setUpQueryTests(); + VectorValue emptyVector = VectorValue.newBuilder().build(); + FindNearest vectorQuery = + new FindNearest("vector_property", emptyVector, FindNearest.DistanceMeasure.EUCLIDEAN, 1); + Query query = Query.newEntityQueryBuilder().setFindNearest(vectorQuery).build(); + assertInvalidQuery(query); + } + + @Test + public void testVectorSearchWithUnmatchedVectorSize() { + setUpQueryTests(); + VectorValue vectorValue = VectorValue.newBuilder(1.78, 2.56, 3.88, 4.33).build(); + FindNearest vectorQuery = + new FindNearest("vector_property", vectorValue, FindNearest.DistanceMeasure.DOT_PRODUCT, 1); + Query query = Query.newEntityQueryBuilder().setFindNearest(vectorQuery).build(); + assertInvalidQuery(query); + } + @Test public void testLimit() { setUpQueryTests(); diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/FindNearest.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/FindNearest.java index ea32db89c..d8c2176f0 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/FindNearest.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/FindNearest.java @@ -26,24 +26,34 @@ /** * A query that finds the entities whose vector fields are closest to a certain query vector. Create - * an instance of `FindNearest` with {@link Query#findNearest}. + * an instance of `FindNearest` with {@link Query}. */ public final class FindNearest implements Serializable { + /** An indexed vector property to search upon. */ private final String vectorProperty; + /** The query vector that we are searching on. */ private final VectorValue queryVector; + /** The Distance Measure to use, required. */ private final DistanceMeasure measure; + /** The number of nearest neighbors to return. Must be a positive integer of no more than 100. */ private final int limit; - /* - Optional. Optional name of the field to output the result of the vector - * distance calculation. + + /** + * Optional. Optional name of the field to output the result of the vector distance calculation. */ private final @Nullable String distanceResultField; + + /** + * Optional. Option to specify a threshold for which no less similar documents will be returned. + * The behavior of the specified `distance_measure` will affect the meaning of the distance + * threshold. + */ private final @Nullable Double distanceThreshold; private static final long serialVersionUID = 4688656124180403551L; - /** Creates a VectorQuery */ + /** Creates a FindNearest query. */ public FindNearest( String vectorProperty, VectorValue queryVector, @@ -89,10 +99,10 @@ public int hashCode() { } /** - * Returns true if this VectorQuery is equal to the provided object. + * Returns true if this FindNearest query is equal to the provided object. * * @param obj The object to compare against. - * @return Whether this VectorQuery is equal to the provided object. + * @return Whether this FindNearest query is equal to the provided object. */ @Override public boolean equals(Object obj) { @@ -179,19 +189,15 @@ protected static com.google.datastore.v1.FindNearest.DistanceMeasure toProto( } } - /** - * The distance measure to use when comparing vectors in a {@link FindNearest query}. - * - * @see com.google.cloud.datastore.Query#findNearest - */ + /** The distance measure to use when comparing vectors in a {@link FindNearest query}. */ public enum DistanceMeasure { + DISTANCE_MEASURE_UNSPECIFIED, /** * COSINE distance compares vectors based on the angle between them, which allows you to measure * similarity that isn't based on the vectors' magnitude. We recommend using DOT_PRODUCT with * unit normalized vectors instead of COSINE distance, which is mathematically equivalent with * better performance. */ - DISTANCE_MEASURE_UNSPECIFIED, COSINE, /** Measures the EUCLIDEAN distance between the vectors. */ EUCLIDEAN, diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/ValueType.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/ValueType.java index df2fb4099..d52b43236 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/ValueType.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/ValueType.java @@ -63,7 +63,7 @@ public enum ValueType { /** Represents a {@link LatLng} value. */ LAT_LNG(LatLngValue.MARSHALLER), - /** Represents a {@link Vector} value. */ + /** Represents a {@link VectorValue} value. */ VECTOR(VectorValue.MARSHALLER); private static final ImmutableMap DESCRIPTOR_TO_TYPE_MAP; From 17b411f01a59779b492d77807a7aa9dcc809133a Mon Sep 17 00:00:00 2001 From: Cindy Peng <148148319+cindy-peng@users.noreply.github.com> Date: Thu, 24 Oct 2024 13:59:15 -0700 Subject: [PATCH 09/33] fix(sample): change update entity sample to use transaction (#1633) --- com/google/datastore/snippets/ConceptsTest.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/com/google/datastore/snippets/ConceptsTest.java b/com/google/datastore/snippets/ConceptsTest.java index 30c646f4d..73753f047 100644 --- a/com/google/datastore/snippets/ConceptsTest.java +++ b/com/google/datastore/snippets/ConceptsTest.java @@ -278,8 +278,17 @@ public void testLookup() { public void testUpdate() { datastore.put(testEntity); // [START datastore_update] - Entity task = Entity.newBuilder(datastore.get(taskKey)).set("priority", 5).build(); - datastore.update(task); + Entity task; + Transaction txn = datastore.newTransaction(); + try { + task = Entity.newBuilder(txn.get(taskKey)).set("priority", 5).build(); + txn.put(task); + txn.commit(); + } finally { + if (txn.isActive()) { + txn.rollback(); + } + } // [END datastore_update] assertEquals(task, datastore.get(taskKey)); } From 2aafa17a86f27c4d9b883dfcacbaf2c568d62820 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Fri, 25 Oct 2024 00:59:23 +0200 Subject: [PATCH 10/33] deps: update dependency com.google.cloud:sdk-platform-java-config to v3.38.0 (#1632) --- .github/workflows/unmanaged_dependency_check.yaml | 2 +- .kokoro/presubmit/graalvm-native-17.cfg | 2 +- .kokoro/presubmit/graalvm-native.cfg | 2 +- google-cloud-datastore-bom/pom.xml | 2 +- pom.xml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/unmanaged_dependency_check.yaml b/.github/workflows/unmanaged_dependency_check.yaml index 8e4ccadbd..e53ea6bf9 100644 --- a/.github/workflows/unmanaged_dependency_check.yaml +++ b/.github/workflows/unmanaged_dependency_check.yaml @@ -14,6 +14,6 @@ jobs: shell: bash run: .kokoro/build.sh - name: Unmanaged dependency check - uses: googleapis/sdk-platform-java/java-shared-dependencies/unmanaged-dependency-check@google-cloud-shared-dependencies/v3.37.0 + uses: googleapis/sdk-platform-java/java-shared-dependencies/unmanaged-dependency-check@google-cloud-shared-dependencies/v3.38.0 with: bom-path: google-cloud-datastore-bom/pom.xml diff --git a/.kokoro/presubmit/graalvm-native-17.cfg b/.kokoro/presubmit/graalvm-native-17.cfg index 5cb66b4fe..49b4cde5f 100644 --- a/.kokoro/presubmit/graalvm-native-17.cfg +++ b/.kokoro/presubmit/graalvm-native-17.cfg @@ -3,7 +3,7 @@ # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:3.37.0" + value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:3.38.0" } env_vars: { diff --git a/.kokoro/presubmit/graalvm-native.cfg b/.kokoro/presubmit/graalvm-native.cfg index 11c4b309b..d486f7e50 100644 --- a/.kokoro/presubmit/graalvm-native.cfg +++ b/.kokoro/presubmit/graalvm-native.cfg @@ -3,7 +3,7 @@ # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:3.37.0" + value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:3.38.0" } env_vars: { diff --git a/google-cloud-datastore-bom/pom.xml b/google-cloud-datastore-bom/pom.xml index 73f8b5cbe..b5d0bd919 100644 --- a/google-cloud-datastore-bom/pom.xml +++ b/google-cloud-datastore-bom/pom.xml @@ -8,7 +8,7 @@ com.google.cloud sdk-platform-java-config - 3.37.0 + 3.38.0 Google Cloud datastore BOM diff --git a/pom.xml b/pom.xml index 108aca675..2ea152f6c 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,7 @@ com.google.cloud sdk-platform-java-config - 3.37.0 + 3.38.0 From 949a0aecfe0d988aeb0e1271f6bafa63e0526ae7 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Fri, 25 Oct 2024 09:22:22 -0700 Subject: [PATCH 11/33] chore(main): release 2.24.0 (#1631) * chore(main): release 2.24.0 * chore: generate libraries at Thu Oct 24 23:00:05 UTC 2024 --------- Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> Co-authored-by: cloud-java-bot --- CHANGELOG.md | 18 ++++++++++++++++++ README.md | 6 +++--- datastore-v1-proto-client/pom.xml | 4 ++-- google-cloud-datastore-bom/pom.xml | 10 +++++----- google-cloud-datastore/pom.xml | 4 ++-- grpc-google-cloud-datastore-admin-v1/pom.xml | 4 ++-- pom.xml | 12 ++++++------ proto-google-cloud-datastore-admin-v1/pom.xml | 4 ++-- proto-google-cloud-datastore-v1/pom.xml | 4 ++-- samples/snapshot/pom.xml | 2 +- versions.txt | 12 ++++++------ 11 files changed, 49 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 34aeb6370..cd4b9d6f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## [2.24.0](https://github.com/googleapis/java-datastore/compare/v2.23.0...v2.24.0) (2024-10-24) + + +### Features + +* Add FindNearest API to the stable branch ([3512ba2](https://github.com/googleapis/java-datastore/commit/3512ba2f1bcd358e3c39c36944e05873b3f25f51)) + + +### Bug Fixes + +* **sample:** Change update entity sample to use transaction ([#1633](https://github.com/googleapis/java-datastore/issues/1633)) ([c44f17a](https://github.com/googleapis/java-datastore/commit/c44f17a7bb93d688367611ee2533c59c940ae61f)) + + +### Dependencies + +* Update dependency com.google.cloud:sdk-platform-java-config to v3.38.0 ([#1632](https://github.com/googleapis/java-datastore/issues/1632)) ([6453f1e](https://github.com/googleapis/java-datastore/commit/6453f1e44f370a13434ef68295ae5638612032c8)) +* Update googleapis/sdk-platform-java action to v2.48.0 ([#1628](https://github.com/googleapis/java-datastore/issues/1628)) ([d3bce79](https://github.com/googleapis/java-datastore/commit/d3bce79467254b3128a8f16d5754e91d29ece525)) + ## [2.23.0](https://github.com/googleapis/java-datastore/compare/v2.22.0...v2.23.0) (2024-10-14) diff --git a/README.md b/README.md index a6a76d6c5..bd795fe50 100644 --- a/README.md +++ b/README.md @@ -56,13 +56,13 @@ implementation 'com.google.cloud:google-cloud-datastore' If you are using Gradle without BOM, add this to your dependencies: ```Groovy -implementation 'com.google.cloud:google-cloud-datastore:2.23.0' +implementation 'com.google.cloud:google-cloud-datastore:2.24.0' ``` If you are using SBT, add this to your dependencies: ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-datastore" % "2.23.0" +libraryDependencies += "com.google.cloud" % "google-cloud-datastore" % "2.24.0" ``` ## Authentication @@ -479,7 +479,7 @@ Java is a registered trademark of Oracle and/or its affiliates. [kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-datastore/java11.html [stability-image]: https://img.shields.io/badge/stability-stable-green [maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-datastore.svg -[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-datastore/2.23.0 +[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-datastore/2.24.0 [authentication]: https://github.com/googleapis/google-cloud-java#authentication [auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes [predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles diff --git a/datastore-v1-proto-client/pom.xml b/datastore-v1-proto-client/pom.xml index 6e3f7cfc9..59dfb4979 100644 --- a/datastore-v1-proto-client/pom.xml +++ b/datastore-v1-proto-client/pom.xml @@ -19,12 +19,12 @@ 4.0.0 com.google.cloud.datastore datastore-v1-proto-client - 2.23.1-SNAPSHOT + 2.24.0 com.google.cloud google-cloud-datastore-parent - 2.23.1-SNAPSHOT + 2.24.0 jar diff --git a/google-cloud-datastore-bom/pom.xml b/google-cloud-datastore-bom/pom.xml index b5d0bd919..8f22264e0 100644 --- a/google-cloud-datastore-bom/pom.xml +++ b/google-cloud-datastore-bom/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-datastore-bom - 2.23.1-SNAPSHOT + 2.24.0 pom com.google.cloud @@ -52,22 +52,22 @@ com.google.cloud google-cloud-datastore - 2.23.1-SNAPSHOT + 2.24.0 com.google.api.grpc grpc-google-cloud-datastore-admin-v1 - 2.23.1-SNAPSHOT + 2.24.0 com.google.api.grpc proto-google-cloud-datastore-v1 - 0.114.1-SNAPSHOT + 0.115.0 com.google.api.grpc proto-google-cloud-datastore-admin-v1 - 2.23.1-SNAPSHOT + 2.24.0 diff --git a/google-cloud-datastore/pom.xml b/google-cloud-datastore/pom.xml index 11e6620ea..b6a7829a5 100644 --- a/google-cloud-datastore/pom.xml +++ b/google-cloud-datastore/pom.xml @@ -2,7 +2,7 @@ 4.0.0 google-cloud-datastore - 2.23.1-SNAPSHOT + 2.24.0 jar Google Cloud Datastore https://github.com/googleapis/java-datastore @@ -12,7 +12,7 @@ com.google.cloud google-cloud-datastore-parent - 2.23.1-SNAPSHOT + 2.24.0 google-cloud-datastore diff --git a/grpc-google-cloud-datastore-admin-v1/pom.xml b/grpc-google-cloud-datastore-admin-v1/pom.xml index 36820f768..d0e64f478 100644 --- a/grpc-google-cloud-datastore-admin-v1/pom.xml +++ b/grpc-google-cloud-datastore-admin-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-datastore-admin-v1 - 2.23.1-SNAPSHOT + 2.24.0 grpc-google-cloud-datastore-admin-v1 GRPC library for google-cloud-datastore com.google.cloud google-cloud-datastore-parent - 2.23.1-SNAPSHOT + 2.24.0 diff --git a/pom.xml b/pom.xml index 2ea152f6c..c0e18bc93 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-datastore-parent pom - 2.23.1-SNAPSHOT + 2.24.0 Google Cloud Datastore Parent https://github.com/googleapis/java-datastore @@ -159,27 +159,27 @@ com.google.api.grpc proto-google-cloud-datastore-admin-v1 - 2.23.1-SNAPSHOT + 2.24.0 com.google.api.grpc grpc-google-cloud-datastore-admin-v1 - 2.23.1-SNAPSHOT + 2.24.0 com.google.cloud google-cloud-datastore - 2.23.1-SNAPSHOT + 2.24.0 com.google.api.grpc proto-google-cloud-datastore-v1 - 0.114.1-SNAPSHOT + 0.115.0 com.google.cloud.datastore datastore-v1-proto-client - 2.23.1-SNAPSHOT + 2.24.0 com.google.api.grpc diff --git a/proto-google-cloud-datastore-admin-v1/pom.xml b/proto-google-cloud-datastore-admin-v1/pom.xml index 621327b74..fe5016f50 100644 --- a/proto-google-cloud-datastore-admin-v1/pom.xml +++ b/proto-google-cloud-datastore-admin-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-datastore-admin-v1 - 2.23.1-SNAPSHOT + 2.24.0 proto-google-cloud-datastore-admin-v1 Proto library for google-cloud-datastore com.google.cloud google-cloud-datastore-parent - 2.23.1-SNAPSHOT + 2.24.0 diff --git a/proto-google-cloud-datastore-v1/pom.xml b/proto-google-cloud-datastore-v1/pom.xml index c07b7321e..fb92c3d74 100644 --- a/proto-google-cloud-datastore-v1/pom.xml +++ b/proto-google-cloud-datastore-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-datastore-v1 - 0.114.1-SNAPSHOT + 0.115.0 proto-google-cloud-datastore-v1 PROTO library for proto-google-cloud-datastore-v1 com.google.cloud google-cloud-datastore-parent - 2.23.1-SNAPSHOT + 2.24.0 diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 04401d182..3c12a1725 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -28,7 +28,7 @@ com.google.cloud google-cloud-datastore - 2.23.1-SNAPSHOT + 2.24.0 diff --git a/versions.txt b/versions.txt index 1d3d32591..cbf5e4466 100644 --- a/versions.txt +++ b/versions.txt @@ -1,9 +1,9 @@ # Format: # module:released-version:current-version -google-cloud-datastore:2.23.0:2.23.1-SNAPSHOT -google-cloud-datastore-bom:2.23.0:2.23.1-SNAPSHOT -proto-google-cloud-datastore-v1:0.114.0:0.114.1-SNAPSHOT -datastore-v1-proto-client:2.23.0:2.23.1-SNAPSHOT -proto-google-cloud-datastore-admin-v1:2.23.0:2.23.1-SNAPSHOT -grpc-google-cloud-datastore-admin-v1:2.23.0:2.23.1-SNAPSHOT +google-cloud-datastore:2.24.0:2.24.0 +google-cloud-datastore-bom:2.24.0:2.24.0 +proto-google-cloud-datastore-v1:0.115.0:0.115.0 +datastore-v1-proto-client:2.24.0:2.24.0 +proto-google-cloud-datastore-admin-v1:2.24.0:2.24.0 +grpc-google-cloud-datastore-admin-v1:2.24.0:2.24.0 From 11b322792832eadde21828253e3289b370be7ab3 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Mon, 28 Oct 2024 17:34:40 +0100 Subject: [PATCH 12/33] deps: update googleapis/sdk-platform-java action to v2.49.0 (#1638) --- .github/workflows/hermetic_library_generation.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/hermetic_library_generation.yaml b/.github/workflows/hermetic_library_generation.yaml index 4943e3e93..46b80edc1 100644 --- a/.github/workflows/hermetic_library_generation.yaml +++ b/.github/workflows/hermetic_library_generation.yaml @@ -37,7 +37,7 @@ jobs: with: fetch-depth: 0 token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} - - uses: googleapis/sdk-platform-java/.github/scripts@v2.48.0 + - uses: googleapis/sdk-platform-java/.github/scripts@v2.49.0 if: env.SHOULD_RUN == 'true' with: base_ref: ${{ github.base_ref }} From 742c7b94178370f36c497dde61b662bd62451772 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 28 Oct 2024 09:34:58 -0700 Subject: [PATCH 13/33] chore(main): release 2.24.1-SNAPSHOT (#1635) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- datastore-v1-proto-client/pom.xml | 4 ++-- google-cloud-datastore-bom/pom.xml | 10 +++++----- google-cloud-datastore/pom.xml | 4 ++-- grpc-google-cloud-datastore-admin-v1/pom.xml | 4 ++-- pom.xml | 12 ++++++------ proto-google-cloud-datastore-admin-v1/pom.xml | 4 ++-- proto-google-cloud-datastore-v1/pom.xml | 4 ++-- samples/snapshot/pom.xml | 2 +- versions.txt | 12 ++++++------ 9 files changed, 28 insertions(+), 28 deletions(-) diff --git a/datastore-v1-proto-client/pom.xml b/datastore-v1-proto-client/pom.xml index 59dfb4979..7393d37da 100644 --- a/datastore-v1-proto-client/pom.xml +++ b/datastore-v1-proto-client/pom.xml @@ -19,12 +19,12 @@ 4.0.0 com.google.cloud.datastore datastore-v1-proto-client - 2.24.0 + 2.24.1-SNAPSHOT com.google.cloud google-cloud-datastore-parent - 2.24.0 + 2.24.1-SNAPSHOT jar diff --git a/google-cloud-datastore-bom/pom.xml b/google-cloud-datastore-bom/pom.xml index 8f22264e0..ea57703ec 100644 --- a/google-cloud-datastore-bom/pom.xml +++ b/google-cloud-datastore-bom/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-datastore-bom - 2.24.0 + 2.24.1-SNAPSHOT pom com.google.cloud @@ -52,22 +52,22 @@ com.google.cloud google-cloud-datastore - 2.24.0 + 2.24.1-SNAPSHOT com.google.api.grpc grpc-google-cloud-datastore-admin-v1 - 2.24.0 + 2.24.1-SNAPSHOT com.google.api.grpc proto-google-cloud-datastore-v1 - 0.115.0 + 0.115.1-SNAPSHOT com.google.api.grpc proto-google-cloud-datastore-admin-v1 - 2.24.0 + 2.24.1-SNAPSHOT diff --git a/google-cloud-datastore/pom.xml b/google-cloud-datastore/pom.xml index b6a7829a5..115c1ec4a 100644 --- a/google-cloud-datastore/pom.xml +++ b/google-cloud-datastore/pom.xml @@ -2,7 +2,7 @@ 4.0.0 google-cloud-datastore - 2.24.0 + 2.24.1-SNAPSHOT jar Google Cloud Datastore https://github.com/googleapis/java-datastore @@ -12,7 +12,7 @@ com.google.cloud google-cloud-datastore-parent - 2.24.0 + 2.24.1-SNAPSHOT google-cloud-datastore diff --git a/grpc-google-cloud-datastore-admin-v1/pom.xml b/grpc-google-cloud-datastore-admin-v1/pom.xml index d0e64f478..879820cf2 100644 --- a/grpc-google-cloud-datastore-admin-v1/pom.xml +++ b/grpc-google-cloud-datastore-admin-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-datastore-admin-v1 - 2.24.0 + 2.24.1-SNAPSHOT grpc-google-cloud-datastore-admin-v1 GRPC library for google-cloud-datastore com.google.cloud google-cloud-datastore-parent - 2.24.0 + 2.24.1-SNAPSHOT diff --git a/pom.xml b/pom.xml index c0e18bc93..a14bafbe8 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-datastore-parent pom - 2.24.0 + 2.24.1-SNAPSHOT Google Cloud Datastore Parent https://github.com/googleapis/java-datastore @@ -159,27 +159,27 @@ com.google.api.grpc proto-google-cloud-datastore-admin-v1 - 2.24.0 + 2.24.1-SNAPSHOT com.google.api.grpc grpc-google-cloud-datastore-admin-v1 - 2.24.0 + 2.24.1-SNAPSHOT com.google.cloud google-cloud-datastore - 2.24.0 + 2.24.1-SNAPSHOT com.google.api.grpc proto-google-cloud-datastore-v1 - 0.115.0 + 0.115.1-SNAPSHOT com.google.cloud.datastore datastore-v1-proto-client - 2.24.0 + 2.24.1-SNAPSHOT com.google.api.grpc diff --git a/proto-google-cloud-datastore-admin-v1/pom.xml b/proto-google-cloud-datastore-admin-v1/pom.xml index fe5016f50..ca0d206f6 100644 --- a/proto-google-cloud-datastore-admin-v1/pom.xml +++ b/proto-google-cloud-datastore-admin-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-datastore-admin-v1 - 2.24.0 + 2.24.1-SNAPSHOT proto-google-cloud-datastore-admin-v1 Proto library for google-cloud-datastore com.google.cloud google-cloud-datastore-parent - 2.24.0 + 2.24.1-SNAPSHOT diff --git a/proto-google-cloud-datastore-v1/pom.xml b/proto-google-cloud-datastore-v1/pom.xml index fb92c3d74..ffec1dc62 100644 --- a/proto-google-cloud-datastore-v1/pom.xml +++ b/proto-google-cloud-datastore-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-datastore-v1 - 0.115.0 + 0.115.1-SNAPSHOT proto-google-cloud-datastore-v1 PROTO library for proto-google-cloud-datastore-v1 com.google.cloud google-cloud-datastore-parent - 2.24.0 + 2.24.1-SNAPSHOT diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index 3c12a1725..cfb3a5b62 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -28,7 +28,7 @@ com.google.cloud google-cloud-datastore - 2.24.0 + 2.24.1-SNAPSHOT diff --git a/versions.txt b/versions.txt index cbf5e4466..71a58858f 100644 --- a/versions.txt +++ b/versions.txt @@ -1,9 +1,9 @@ # Format: # module:released-version:current-version -google-cloud-datastore:2.24.0:2.24.0 -google-cloud-datastore-bom:2.24.0:2.24.0 -proto-google-cloud-datastore-v1:0.115.0:0.115.0 -datastore-v1-proto-client:2.24.0:2.24.0 -proto-google-cloud-datastore-admin-v1:2.24.0:2.24.0 -grpc-google-cloud-datastore-admin-v1:2.24.0:2.24.0 +google-cloud-datastore:2.24.0:2.24.1-SNAPSHOT +google-cloud-datastore-bom:2.24.0:2.24.1-SNAPSHOT +proto-google-cloud-datastore-v1:0.115.0:0.115.1-SNAPSHOT +datastore-v1-proto-client:2.24.0:2.24.1-SNAPSHOT +proto-google-cloud-datastore-admin-v1:2.24.0:2.24.1-SNAPSHOT +grpc-google-cloud-datastore-admin-v1:2.24.0:2.24.1-SNAPSHOT From 089b68e9ad959b73b2ffdc7c8f30324f6cf3cb75 Mon Sep 17 00:00:00 2001 From: cloud-java-bot <122572305+cloud-java-bot@users.noreply.github.com> Date: Mon, 28 Oct 2024 12:58:20 -0400 Subject: [PATCH 14/33] chore: Update generation configuration at Sun Oct 27 02:26:19 UTC 2024 (#1634) * chore: Update generation configuration at Fri Oct 25 02:24:17 UTC 2024 * chore: Update generation configuration at Sat Oct 26 02:21:39 UTC 2024 * chore: Update generation configuration at Sun Oct 27 02:26:19 UTC 2024 --- generation_config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/generation_config.yaml b/generation_config.yaml index 5efd31c25..f2751f7dc 100644 --- a/generation_config.yaml +++ b/generation_config.yaml @@ -1,5 +1,5 @@ -gapic_generator_version: 2.48.0 -googleapis_commitish: 7d0624db6f5b5b7d8f5cf36571b33ded2067c5e1 +gapic_generator_version: 2.49.0 +googleapis_commitish: 48d30c4966ef9ea31b897e13f75d8f94070cc8e9 libraries_bom_version: 26.49.0 libraries: - api_shortname: datastore From 81980d218e1928f42e65f935431a8abef789d918 Mon Sep 17 00:00:00 2001 From: Mend Renovate Date: Mon, 28 Oct 2024 18:16:30 +0100 Subject: [PATCH 15/33] deps: update dependency com.google.cloud:sdk-platform-java-config to v3.39.0 (#1640) --- .github/workflows/unmanaged_dependency_check.yaml | 2 +- .kokoro/presubmit/graalvm-native-17.cfg | 2 +- .kokoro/presubmit/graalvm-native.cfg | 2 +- google-cloud-datastore-bom/pom.xml | 2 +- pom.xml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/unmanaged_dependency_check.yaml b/.github/workflows/unmanaged_dependency_check.yaml index e53ea6bf9..f1796d286 100644 --- a/.github/workflows/unmanaged_dependency_check.yaml +++ b/.github/workflows/unmanaged_dependency_check.yaml @@ -14,6 +14,6 @@ jobs: shell: bash run: .kokoro/build.sh - name: Unmanaged dependency check - uses: googleapis/sdk-platform-java/java-shared-dependencies/unmanaged-dependency-check@google-cloud-shared-dependencies/v3.38.0 + uses: googleapis/sdk-platform-java/java-shared-dependencies/unmanaged-dependency-check@google-cloud-shared-dependencies/v3.39.0 with: bom-path: google-cloud-datastore-bom/pom.xml diff --git a/.kokoro/presubmit/graalvm-native-17.cfg b/.kokoro/presubmit/graalvm-native-17.cfg index 49b4cde5f..e8c154f48 100644 --- a/.kokoro/presubmit/graalvm-native-17.cfg +++ b/.kokoro/presubmit/graalvm-native-17.cfg @@ -3,7 +3,7 @@ # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:3.38.0" + value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:3.39.0" } env_vars: { diff --git a/.kokoro/presubmit/graalvm-native.cfg b/.kokoro/presubmit/graalvm-native.cfg index d486f7e50..046dc8df4 100644 --- a/.kokoro/presubmit/graalvm-native.cfg +++ b/.kokoro/presubmit/graalvm-native.cfg @@ -3,7 +3,7 @@ # Configure the docker image for kokoro-trampoline. env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:3.38.0" + value: "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:3.39.0" } env_vars: { diff --git a/google-cloud-datastore-bom/pom.xml b/google-cloud-datastore-bom/pom.xml index ea57703ec..15b54408c 100644 --- a/google-cloud-datastore-bom/pom.xml +++ b/google-cloud-datastore-bom/pom.xml @@ -8,7 +8,7 @@ com.google.cloud sdk-platform-java-config - 3.38.0 + 3.39.0 Google Cloud datastore BOM diff --git a/pom.xml b/pom.xml index a14bafbe8..415bbbf63 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,7 @@ com.google.cloud sdk-platform-java-config - 3.38.0 + 3.39.0 From 546cf8147ee3eb6932ecdf929aac3ce19ba7a54f Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 28 Oct 2024 11:48:12 -0700 Subject: [PATCH 16/33] chore(main): release 2.24.1 (#1641) * chore(main): release 2.24.1 * chore: generate libraries at Mon Oct 28 17:17:32 UTC 2024 --------- Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> Co-authored-by: cloud-java-bot --- CHANGELOG.md | 8 ++++++++ README.md | 6 +++--- datastore-v1-proto-client/pom.xml | 4 ++-- google-cloud-datastore-bom/pom.xml | 10 +++++----- google-cloud-datastore/pom.xml | 4 ++-- grpc-google-cloud-datastore-admin-v1/pom.xml | 4 ++-- pom.xml | 12 ++++++------ proto-google-cloud-datastore-admin-v1/pom.xml | 4 ++-- proto-google-cloud-datastore-v1/pom.xml | 4 ++-- samples/snapshot/pom.xml | 2 +- versions.txt | 12 ++++++------ 11 files changed, 39 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cd4b9d6f9..b76fbc206 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [2.24.1](https://github.com/googleapis/java-datastore/compare/v2.24.0...v2.24.1) (2024-10-28) + + +### Dependencies + +* Update dependency com.google.cloud:sdk-platform-java-config to v3.39.0 ([#1640](https://github.com/googleapis/java-datastore/issues/1640)) ([fe61f66](https://github.com/googleapis/java-datastore/commit/fe61f6691a5e3c8fbfc974b6fe613a69652241ca)) +* Update googleapis/sdk-platform-java action to v2.49.0 ([#1638](https://github.com/googleapis/java-datastore/issues/1638)) ([57598d7](https://github.com/googleapis/java-datastore/commit/57598d7d59cd6917f23a653403613e4edc160c64)) + ## [2.24.0](https://github.com/googleapis/java-datastore/compare/v2.23.0...v2.24.0) (2024-10-24) diff --git a/README.md b/README.md index bd795fe50..ef03d1261 100644 --- a/README.md +++ b/README.md @@ -56,13 +56,13 @@ implementation 'com.google.cloud:google-cloud-datastore' If you are using Gradle without BOM, add this to your dependencies: ```Groovy -implementation 'com.google.cloud:google-cloud-datastore:2.24.0' +implementation 'com.google.cloud:google-cloud-datastore:2.24.1' ``` If you are using SBT, add this to your dependencies: ```Scala -libraryDependencies += "com.google.cloud" % "google-cloud-datastore" % "2.24.0" +libraryDependencies += "com.google.cloud" % "google-cloud-datastore" % "2.24.1" ``` ## Authentication @@ -479,7 +479,7 @@ Java is a registered trademark of Oracle and/or its affiliates. [kokoro-badge-link-5]: http://storage.googleapis.com/cloud-devrel-public/java/badges/java-datastore/java11.html [stability-image]: https://img.shields.io/badge/stability-stable-green [maven-version-image]: https://img.shields.io/maven-central/v/com.google.cloud/google-cloud-datastore.svg -[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-datastore/2.24.0 +[maven-version-link]: https://central.sonatype.com/artifact/com.google.cloud/google-cloud-datastore/2.24.1 [authentication]: https://github.com/googleapis/google-cloud-java#authentication [auth-scopes]: https://developers.google.com/identity/protocols/oauth2/scopes [predefined-iam-roles]: https://cloud.google.com/iam/docs/understanding-roles#predefined_roles diff --git a/datastore-v1-proto-client/pom.xml b/datastore-v1-proto-client/pom.xml index 7393d37da..235be990e 100644 --- a/datastore-v1-proto-client/pom.xml +++ b/datastore-v1-proto-client/pom.xml @@ -19,12 +19,12 @@ 4.0.0 com.google.cloud.datastore datastore-v1-proto-client - 2.24.1-SNAPSHOT + 2.24.1 com.google.cloud google-cloud-datastore-parent - 2.24.1-SNAPSHOT + 2.24.1 jar diff --git a/google-cloud-datastore-bom/pom.xml b/google-cloud-datastore-bom/pom.xml index 15b54408c..627c32f58 100644 --- a/google-cloud-datastore-bom/pom.xml +++ b/google-cloud-datastore-bom/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-datastore-bom - 2.24.1-SNAPSHOT + 2.24.1 pom com.google.cloud @@ -52,22 +52,22 @@ com.google.cloud google-cloud-datastore - 2.24.1-SNAPSHOT + 2.24.1 com.google.api.grpc grpc-google-cloud-datastore-admin-v1 - 2.24.1-SNAPSHOT + 2.24.1 com.google.api.grpc proto-google-cloud-datastore-v1 - 0.115.1-SNAPSHOT + 0.115.1 com.google.api.grpc proto-google-cloud-datastore-admin-v1 - 2.24.1-SNAPSHOT + 2.24.1 diff --git a/google-cloud-datastore/pom.xml b/google-cloud-datastore/pom.xml index 115c1ec4a..6f39d3922 100644 --- a/google-cloud-datastore/pom.xml +++ b/google-cloud-datastore/pom.xml @@ -2,7 +2,7 @@ 4.0.0 google-cloud-datastore - 2.24.1-SNAPSHOT + 2.24.1 jar Google Cloud Datastore https://github.com/googleapis/java-datastore @@ -12,7 +12,7 @@ com.google.cloud google-cloud-datastore-parent - 2.24.1-SNAPSHOT + 2.24.1 google-cloud-datastore diff --git a/grpc-google-cloud-datastore-admin-v1/pom.xml b/grpc-google-cloud-datastore-admin-v1/pom.xml index 879820cf2..ebe1b66cd 100644 --- a/grpc-google-cloud-datastore-admin-v1/pom.xml +++ b/grpc-google-cloud-datastore-admin-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-cloud-datastore-admin-v1 - 2.24.1-SNAPSHOT + 2.24.1 grpc-google-cloud-datastore-admin-v1 GRPC library for google-cloud-datastore com.google.cloud google-cloud-datastore-parent - 2.24.1-SNAPSHOT + 2.24.1 diff --git a/pom.xml b/pom.xml index 415bbbf63..b4211aa32 100644 --- a/pom.xml +++ b/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-datastore-parent pom - 2.24.1-SNAPSHOT + 2.24.1 Google Cloud Datastore Parent https://github.com/googleapis/java-datastore @@ -159,27 +159,27 @@ com.google.api.grpc proto-google-cloud-datastore-admin-v1 - 2.24.1-SNAPSHOT + 2.24.1 com.google.api.grpc grpc-google-cloud-datastore-admin-v1 - 2.24.1-SNAPSHOT + 2.24.1 com.google.cloud google-cloud-datastore - 2.24.1-SNAPSHOT + 2.24.1 com.google.api.grpc proto-google-cloud-datastore-v1 - 0.115.1-SNAPSHOT + 0.115.1 com.google.cloud.datastore datastore-v1-proto-client - 2.24.1-SNAPSHOT + 2.24.1 com.google.api.grpc diff --git a/proto-google-cloud-datastore-admin-v1/pom.xml b/proto-google-cloud-datastore-admin-v1/pom.xml index ca0d206f6..1c19e4ab3 100644 --- a/proto-google-cloud-datastore-admin-v1/pom.xml +++ b/proto-google-cloud-datastore-admin-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-datastore-admin-v1 - 2.24.1-SNAPSHOT + 2.24.1 proto-google-cloud-datastore-admin-v1 Proto library for google-cloud-datastore com.google.cloud google-cloud-datastore-parent - 2.24.1-SNAPSHOT + 2.24.1 diff --git a/proto-google-cloud-datastore-v1/pom.xml b/proto-google-cloud-datastore-v1/pom.xml index ffec1dc62..f8d1b8c04 100644 --- a/proto-google-cloud-datastore-v1/pom.xml +++ b/proto-google-cloud-datastore-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-cloud-datastore-v1 - 0.115.1-SNAPSHOT + 0.115.1 proto-google-cloud-datastore-v1 PROTO library for proto-google-cloud-datastore-v1 com.google.cloud google-cloud-datastore-parent - 2.24.1-SNAPSHOT + 2.24.1 diff --git a/samples/snapshot/pom.xml b/samples/snapshot/pom.xml index cfb3a5b62..46881f45b 100644 --- a/samples/snapshot/pom.xml +++ b/samples/snapshot/pom.xml @@ -28,7 +28,7 @@ com.google.cloud google-cloud-datastore - 2.24.1-SNAPSHOT + 2.24.1 diff --git a/versions.txt b/versions.txt index 71a58858f..6939d61e9 100644 --- a/versions.txt +++ b/versions.txt @@ -1,9 +1,9 @@ # Format: # module:released-version:current-version -google-cloud-datastore:2.24.0:2.24.1-SNAPSHOT -google-cloud-datastore-bom:2.24.0:2.24.1-SNAPSHOT -proto-google-cloud-datastore-v1:0.115.0:0.115.1-SNAPSHOT -datastore-v1-proto-client:2.24.0:2.24.1-SNAPSHOT -proto-google-cloud-datastore-admin-v1:2.24.0:2.24.1-SNAPSHOT -grpc-google-cloud-datastore-admin-v1:2.24.0:2.24.1-SNAPSHOT +google-cloud-datastore:2.24.1:2.24.1 +google-cloud-datastore-bom:2.24.1:2.24.1 +proto-google-cloud-datastore-v1:0.115.1:0.115.1 +datastore-v1-proto-client:2.24.1:2.24.1 +proto-google-cloud-datastore-admin-v1:2.24.1:2.24.1 +grpc-google-cloud-datastore-admin-v1:2.24.1:2.24.1 From c495bb6b5f06e8a9fe5592fa96c99f73c3ebed06 Mon Sep 17 00:00:00 2001 From: cloud-java-bot Date: Mon, 28 Oct 2024 20:28:18 +0000 Subject: [PATCH 17/33] chore: generate libraries at Mon Oct 28 20:25:23 UTC 2024 --- com/google/datastore/snippets/ConceptsTest.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/com/google/datastore/snippets/ConceptsTest.java b/com/google/datastore/snippets/ConceptsTest.java index 73753f047..90f6fc26f 100644 --- a/com/google/datastore/snippets/ConceptsTest.java +++ b/com/google/datastore/snippets/ConceptsTest.java @@ -390,8 +390,8 @@ private void setUpQueryTests() { StringValue.newBuilder("Learn Cloud Datastore").setExcludeFromIndexes(true).build()) .set("tag", "fun", "l", "programming", "learn") .set( - "vector_property", - VectorValue.newBuilder(3.0, 1.0, 2.0).setExcludeFromIndexes(true).build()) + "vector_property", + VectorValue.newBuilder(3.0, 1.0, 2.0).setExcludeFromIndexes(true).build()) .build()); } @@ -715,8 +715,8 @@ public void testVectorSearch() { setUpQueryTests(); VectorValue vectorValue = VectorValue.newBuilder(1.78, 2.56, 3.88).build(); FindNearest vectorQuery = - new FindNearest( - "vector_property", vectorValue, FindNearest.DistanceMeasure.COSINE, 1, "distance"); + new FindNearest( + "vector_property", vectorValue, FindNearest.DistanceMeasure.COSINE, 1, "distance"); Query query = Query.newEntityQueryBuilder().setFindNearest(vectorQuery).build(); assertValidQuery(query); @@ -727,7 +727,7 @@ public void testVectorSearchWithEmptyVector() { setUpQueryTests(); VectorValue emptyVector = VectorValue.newBuilder().build(); FindNearest vectorQuery = - new FindNearest("vector_property", emptyVector, FindNearest.DistanceMeasure.EUCLIDEAN, 1); + new FindNearest("vector_property", emptyVector, FindNearest.DistanceMeasure.EUCLIDEAN, 1); Query query = Query.newEntityQueryBuilder().setFindNearest(vectorQuery).build(); assertInvalidQuery(query); } @@ -737,7 +737,7 @@ public void testVectorSearchWithUnmatchedVectorSize() { setUpQueryTests(); VectorValue vectorValue = VectorValue.newBuilder(1.78, 2.56, 3.88, 4.33).build(); FindNearest vectorQuery = - new FindNearest("vector_property", vectorValue, FindNearest.DistanceMeasure.DOT_PRODUCT, 1); + new FindNearest("vector_property", vectorValue, FindNearest.DistanceMeasure.DOT_PRODUCT, 1); Query query = Query.newEntityQueryBuilder().setFindNearest(vectorQuery).build(); assertInvalidQuery(query); } From e93ce5ca9c6aca416f0d089ff45898e6ceefbe3f Mon Sep 17 00:00:00 2001 From: Cindy Peng <148148319+cindy-peng@users.noreply.github.com> Date: Mon, 28 Oct 2024 13:29:15 -0700 Subject: [PATCH 18/33] Fix import --- .../datastore/snippets/ConceptsTest.java | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/com/google/datastore/snippets/ConceptsTest.java b/com/google/datastore/snippets/ConceptsTest.java index 90f6fc26f..7b2233d2a 100644 --- a/com/google/datastore/snippets/ConceptsTest.java +++ b/com/google/datastore/snippets/ConceptsTest.java @@ -24,10 +24,31 @@ import static org.junit.Assert.assertNull; import com.google.cloud.Timestamp; -import com.google.cloud.datastore.*; +import com.google.cloud.datastore.Cursor; +import com.google.cloud.datastore.Datastore; +import com.google.cloud.datastore.DatastoreException; +import com.google.cloud.datastore.DatastoreOptions; +import com.google.cloud.datastore.Entity; +import com.google.cloud.datastore.EntityQuery; +import com.google.cloud.datastore.FullEntity; +import com.google.cloud.datastore.IncompleteKey; +import com.google.cloud.datastore.Key; +import com.google.cloud.datastore.KeyFactory; +import com.google.cloud.datastore.KeyQuery; +import com.google.cloud.datastore.ListValue; +import com.google.cloud.datastore.VectorValue; +import com.google.cloud.datastore.FindNearest; +import com.google.cloud.datastore.PathElement; +import com.google.cloud.datastore.ProjectionEntity; +import com.google.cloud.datastore.Query; +import com.google.cloud.datastore.QueryResults; +import com.google.cloud.datastore.ReadOption; +import com.google.cloud.datastore.StringValue; +import com.google.cloud.datastore.StructuredQuery; import com.google.cloud.datastore.StructuredQuery.CompositeFilter; import com.google.cloud.datastore.StructuredQuery.OrderBy; import com.google.cloud.datastore.StructuredQuery.PropertyFilter; +import com.google.cloud.datastore.Transaction; import com.google.cloud.datastore.testing.LocalDatastoreHelper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; From 0072de4425683ed9742f02da4b95e26cde52557d Mon Sep 17 00:00:00 2001 From: cloud-java-bot Date: Mon, 28 Oct 2024 20:33:07 +0000 Subject: [PATCH 19/33] chore: generate libraries at Mon Oct 28 20:30:34 UTC 2024 --- com/google/datastore/snippets/ConceptsTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/com/google/datastore/snippets/ConceptsTest.java b/com/google/datastore/snippets/ConceptsTest.java index 7b2233d2a..6f8301400 100644 --- a/com/google/datastore/snippets/ConceptsTest.java +++ b/com/google/datastore/snippets/ConceptsTest.java @@ -30,14 +30,13 @@ import com.google.cloud.datastore.DatastoreOptions; import com.google.cloud.datastore.Entity; import com.google.cloud.datastore.EntityQuery; +import com.google.cloud.datastore.FindNearest; import com.google.cloud.datastore.FullEntity; import com.google.cloud.datastore.IncompleteKey; import com.google.cloud.datastore.Key; import com.google.cloud.datastore.KeyFactory; import com.google.cloud.datastore.KeyQuery; import com.google.cloud.datastore.ListValue; -import com.google.cloud.datastore.VectorValue; -import com.google.cloud.datastore.FindNearest; import com.google.cloud.datastore.PathElement; import com.google.cloud.datastore.ProjectionEntity; import com.google.cloud.datastore.Query; @@ -49,6 +48,7 @@ import com.google.cloud.datastore.StructuredQuery.OrderBy; import com.google.cloud.datastore.StructuredQuery.PropertyFilter; import com.google.cloud.datastore.Transaction; +import com.google.cloud.datastore.VectorValue; import com.google.cloud.datastore.testing.LocalDatastoreHelper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; From a37146752a14dd7f0647d034ca6a5354ea8ceaed Mon Sep 17 00:00:00 2001 From: Cindy Peng <148148319+cindy-peng@users.noreply.github.com> Date: Wed, 30 Oct 2024 11:49:37 -0700 Subject: [PATCH 20/33] Add Integration test --- .../datastore/snippets/ConceptsTest.java | 22 +------- .../cloud/datastore/it/ITDatastoreTest.java | 54 +++++++++++++++++++ 2 files changed, 56 insertions(+), 20 deletions(-) diff --git a/com/google/datastore/snippets/ConceptsTest.java b/com/google/datastore/snippets/ConceptsTest.java index 6f8301400..8ffaf0f63 100644 --- a/com/google/datastore/snippets/ConceptsTest.java +++ b/com/google/datastore/snippets/ConceptsTest.java @@ -734,35 +734,17 @@ public void testInequalitySortInvalidNotFirst() { @Test public void testVectorSearch() { setUpQueryTests(); + // [START datastore_vector_search] VectorValue vectorValue = VectorValue.newBuilder(1.78, 2.56, 3.88).build(); FindNearest vectorQuery = new FindNearest( "vector_property", vectorValue, FindNearest.DistanceMeasure.COSINE, 1, "distance"); Query query = Query.newEntityQueryBuilder().setFindNearest(vectorQuery).build(); + // [END datastore_vector_search] assertValidQuery(query); } - @Test - public void testVectorSearchWithEmptyVector() { - setUpQueryTests(); - VectorValue emptyVector = VectorValue.newBuilder().build(); - FindNearest vectorQuery = - new FindNearest("vector_property", emptyVector, FindNearest.DistanceMeasure.EUCLIDEAN, 1); - Query query = Query.newEntityQueryBuilder().setFindNearest(vectorQuery).build(); - assertInvalidQuery(query); - } - - @Test - public void testVectorSearchWithUnmatchedVectorSize() { - setUpQueryTests(); - VectorValue vectorValue = VectorValue.newBuilder(1.78, 2.56, 3.88, 4.33).build(); - FindNearest vectorQuery = - new FindNearest("vector_property", vectorValue, FindNearest.DistanceMeasure.DOT_PRODUCT, 1); - Query query = Query.newEntityQueryBuilder().setFindNearest(vectorQuery).build(); - assertInvalidQuery(query); - } - @Test public void testLimit() { setUpQueryTests(); diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreTest.java index c012d28c9..598f83327 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreTest.java @@ -56,6 +56,8 @@ import com.google.cloud.datastore.LatLngValue; import com.google.cloud.datastore.ListValue; import com.google.cloud.datastore.NullValue; +import com.google.cloud.datastore.VectorValue; +import com.google.cloud.datastore.FindNearest; import com.google.cloud.datastore.PathElement; import com.google.cloud.datastore.ProjectionEntity; import com.google.cloud.datastore.Query; @@ -2115,6 +2117,58 @@ public void testQueryWithStartCursor() { datastore.delete(entity1.getKey(), entity2.getKey(), entity3.getKey()); } + @Test + public void testQueryWithVectorSearch() { + Entity entity1 = + Entity.newBuilder( + Key.newBuilder(PROJECT_ID, KIND1, "name-01", options.getDatabaseId()).build()) + .set( + "vector_property", + VectorValue.newBuilder(3.0, 1.0, 2.0).setExcludeFromIndexes(true).build()) + .build(); + datastore.put(entity1); + + VectorValue vectorValue = VectorValue.newBuilder(1.78, 2.56, 3.88).build(); + + // Query to find the nearest 1 neighbor + FindNearest vectorQuery = + new FindNearest( + "vector_property", vectorValue, FindNearest.DistanceMeasure.COSINE, 1, "distance"); + Query queryWithVectorSearch = Query.newEntityQueryBuilder().setKind(KIND1).setFindNearest(vectorQuery).build(); + QueryResults vectorSearchResult = datastore.run(queryWithVectorSearch); + assertTrue(vectorSearchResult.hasNext()); + assertEquals(entity1, vectorSearchResult.next()); + assertFalse(vectorSearchResult.hasNext()); + + Entity entity2 = + Entity.newBuilder( + Key.newBuilder(PROJECT_ID, KIND1, "name-02", options.getDatabaseId()).build()) + .set( + "vector_property", + VectorValue.newBuilder(5.0, 0.7, 2.0).setExcludeFromIndexes(true).build()) + .build(); + Entity entity3 = + Entity.newBuilder( + Key.newBuilder(PROJECT_ID, KIND1, "name-03", options.getDatabaseId()).build()) + .set( + "vector_property", + VectorValue.newBuilder(2.0, 1.7, 1.0).setExcludeFromIndexes(true).build()) + .build(); + datastore.put(entity2, entity3); + + // Query to find the nearest 2 neighbors + FindNearest vectorQueryWithLimit = + new FindNearest( + "vector_property", VectorValue.newBuilder(2.8, 2.56, 3.88).build(), FindNearest.DistanceMeasure.EUCLIDEAN, 2, "distance"); + + Query queryWithVectorSearchLimit = Query.newEntityQueryBuilder().setKind(KIND1).setFindNearest(vectorQueryWithLimit).build(); + QueryResults resultsWithVectorLimit = datastore.run(queryWithVectorSearchLimit); + List resultsCopy = makeResultsCopy(resultsWithVectorLimit); + assertEquals(2, resultsCopy.size()); + + datastore.delete(entity1.getKey(), entity2.getKey(), entity3.getKey()); + } + @Test public void testQueryWithReadTime() throws InterruptedException { Entity entity1 = From c8340bffb8342c0aced302d5bea20f790fd71df0 Mon Sep 17 00:00:00 2001 From: Cindy Peng <148148319+cindy-peng@users.noreply.github.com> Date: Wed, 30 Oct 2024 11:58:03 -0700 Subject: [PATCH 21/33] Add comment and fix formatting --- .../com/google/cloud/datastore/Value.java | 1 + .../cloud/datastore/it/ITDatastoreTest.java | 60 ++++++++++--------- 2 files changed, 34 insertions(+), 27 deletions(-) diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/Value.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/Value.java index 93bd4298c..40b4e59a4 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/Value.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/Value.java @@ -218,6 +218,7 @@ public static Value fromPb(com.google.datastore.v1.Value proto) { if (valueType == null) return RawValue.MARSHALLER.fromProto(proto).build(); Value returnValue = valueType.getMarshaller().fromProto(proto).build(); + // If the proto is a list of doubles with a meaning of 31, use the VectorValue marshaller. if (valueType == ValueType.LIST && proto.getMeaning() == VECTOR_MEANING) { for (com.google.datastore.v1.Value item : proto.getArrayValue().getValuesList()) { if (item.getValueTypeCase() != ValueTypeCase.DOUBLE_VALUE) { diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreTest.java index 598f83327..7abc77db6 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreTest.java @@ -46,6 +46,7 @@ import com.google.cloud.datastore.Entity; import com.google.cloud.datastore.EntityQuery; import com.google.cloud.datastore.EntityValue; +import com.google.cloud.datastore.FindNearest; import com.google.cloud.datastore.FullEntity; import com.google.cloud.datastore.GqlQuery; import com.google.cloud.datastore.IncompleteKey; @@ -56,8 +57,6 @@ import com.google.cloud.datastore.LatLngValue; import com.google.cloud.datastore.ListValue; import com.google.cloud.datastore.NullValue; -import com.google.cloud.datastore.VectorValue; -import com.google.cloud.datastore.FindNearest; import com.google.cloud.datastore.PathElement; import com.google.cloud.datastore.ProjectionEntity; import com.google.cloud.datastore.Query; @@ -72,6 +71,7 @@ import com.google.cloud.datastore.TimestampValue; import com.google.cloud.datastore.Transaction; import com.google.cloud.datastore.ValueType; +import com.google.cloud.datastore.VectorValue; import com.google.cloud.datastore.models.ExecutionStats; import com.google.cloud.datastore.models.ExplainMetrics; import com.google.cloud.datastore.models.ExplainOptions; @@ -2120,48 +2120,54 @@ public void testQueryWithStartCursor() { @Test public void testQueryWithVectorSearch() { Entity entity1 = - Entity.newBuilder( - Key.newBuilder(PROJECT_ID, KIND1, "name-01", options.getDatabaseId()).build()) - .set( - "vector_property", - VectorValue.newBuilder(3.0, 1.0, 2.0).setExcludeFromIndexes(true).build()) - .build(); + Entity.newBuilder( + Key.newBuilder(PROJECT_ID, KIND1, "name-01", options.getDatabaseId()).build()) + .set( + "vector_property", + VectorValue.newBuilder(3.0, 1.0, 2.0).setExcludeFromIndexes(true).build()) + .build(); datastore.put(entity1); VectorValue vectorValue = VectorValue.newBuilder(1.78, 2.56, 3.88).build(); // Query to find the nearest 1 neighbor FindNearest vectorQuery = - new FindNearest( - "vector_property", vectorValue, FindNearest.DistanceMeasure.COSINE, 1, "distance"); - Query queryWithVectorSearch = Query.newEntityQueryBuilder().setKind(KIND1).setFindNearest(vectorQuery).build(); + new FindNearest( + "vector_property", vectorValue, FindNearest.DistanceMeasure.COSINE, 1, "distance"); + Query queryWithVectorSearch = + Query.newEntityQueryBuilder().setKind(KIND1).setFindNearest(vectorQuery).build(); QueryResults vectorSearchResult = datastore.run(queryWithVectorSearch); assertTrue(vectorSearchResult.hasNext()); assertEquals(entity1, vectorSearchResult.next()); assertFalse(vectorSearchResult.hasNext()); Entity entity2 = - Entity.newBuilder( - Key.newBuilder(PROJECT_ID, KIND1, "name-02", options.getDatabaseId()).build()) - .set( - "vector_property", - VectorValue.newBuilder(5.0, 0.7, 2.0).setExcludeFromIndexes(true).build()) - .build(); + Entity.newBuilder( + Key.newBuilder(PROJECT_ID, KIND1, "name-02", options.getDatabaseId()).build()) + .set( + "vector_property", + VectorValue.newBuilder(5.0, 0.7, 2.0).setExcludeFromIndexes(true).build()) + .build(); Entity entity3 = - Entity.newBuilder( - Key.newBuilder(PROJECT_ID, KIND1, "name-03", options.getDatabaseId()).build()) - .set( - "vector_property", - VectorValue.newBuilder(2.0, 1.7, 1.0).setExcludeFromIndexes(true).build()) - .build(); + Entity.newBuilder( + Key.newBuilder(PROJECT_ID, KIND1, "name-03", options.getDatabaseId()).build()) + .set( + "vector_property", + VectorValue.newBuilder(2.0, 1.7, 1.0).setExcludeFromIndexes(true).build()) + .build(); datastore.put(entity2, entity3); // Query to find the nearest 2 neighbors FindNearest vectorQueryWithLimit = - new FindNearest( - "vector_property", VectorValue.newBuilder(2.8, 2.56, 3.88).build(), FindNearest.DistanceMeasure.EUCLIDEAN, 2, "distance"); - - Query queryWithVectorSearchLimit = Query.newEntityQueryBuilder().setKind(KIND1).setFindNearest(vectorQueryWithLimit).build(); + new FindNearest( + "vector_property", + VectorValue.newBuilder(2.8, 2.56, 3.88).build(), + FindNearest.DistanceMeasure.EUCLIDEAN, + 2, + "distance"); + + Query queryWithVectorSearchLimit = + Query.newEntityQueryBuilder().setKind(KIND1).setFindNearest(vectorQueryWithLimit).build(); QueryResults resultsWithVectorLimit = datastore.run(queryWithVectorSearchLimit); List resultsCopy = makeResultsCopy(resultsWithVectorLimit); assertEquals(2, resultsCopy.size()); From 62da35d8437b71409c4fd2c806348d84dfe16394 Mon Sep 17 00:00:00 2001 From: Cindy Peng <148148319+cindy-peng@users.noreply.github.com> Date: Wed, 30 Oct 2024 13:05:11 -0700 Subject: [PATCH 22/33] Modify comment and fix formatting --- .../google/cloud/datastore/VectorValue.java | 13 +++-- .../cloud/datastore/StructuredQueryTest.java | 4 -- .../cloud/datastore/VectorValueTest.java | 1 - .../cloud/datastore/it/ITDatastoreTest.java | 56 +++++++++---------- 4 files changed, 35 insertions(+), 39 deletions(-) diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/VectorValue.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/VectorValue.java index 38a2903dc..944e38405 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/VectorValue.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/VectorValue.java @@ -20,7 +20,9 @@ import java.util.ArrayList; import java.util.List; -/** A Google Cloud Datastore Vector value. A list value is a list of {@link Value} objects. */ +/** + * A Google Cloud Datastore Vector value. A Vector value is a list of Double {@link Value} objects. + */ public final class VectorValue extends Value>> { private static final long serialVersionUID = -5121887228607148859L; @@ -87,7 +89,7 @@ public VectorValue.Builder addValue(double first, double... other) { } /** - * Sets the list of values of this {@code ListValue} builder to {@code values}. The provided + * Sets the list of values of this {@code VectorValue} builder to {@code values}. The provided * list is copied. * * @see com.google.cloud.datastore.Value.BaseBuilder#set(java.lang.Object) @@ -106,7 +108,7 @@ public List> get() { return vectorBuilder.build(); } - /** Creates a {@code ListValue} object. */ + /** Creates a {@code VectorValue} object. */ @Override public VectorValue build() { return new VectorValue(this); @@ -121,7 +123,7 @@ private VectorValue(Builder builder) { super(builder); } - /** Returns a builder for the list value object. */ + /** Returns a builder for the vector value object. */ @Override public Builder toBuilder() { return new Builder().mergeFrom(this); @@ -137,10 +139,9 @@ public static VectorValue of(List> values) { return new VectorValue(values); } - /** Returns a builder for {@code ListValue} objects. */ + /** Returns a builder for {@code VectorValue} objects. */ public static Builder newBuilder() { Builder builder = new VectorValue.Builder(); - builder.setExcludeFromIndexes(true); builder.setMeaning(VECTOR_MEANING); return builder; } diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/StructuredQueryTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/StructuredQueryTest.java index c23bd3a9c..41b346766 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/StructuredQueryTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/StructuredQueryTest.java @@ -111,7 +111,6 @@ public void testKeyQueryBuilder() { assertEquals(ORDER_BY, KEY_QUERY.getOrderBy()); assertEquals(ImmutableList.of(StructuredQuery.KEY_PROPERTY_NAME), KEY_QUERY.getProjection()); assertTrue(KEY_QUERY.getDistinctOn().isEmpty()); - assertEquals(LIMIT, KEY_QUERY.getLimit()); } @Test @@ -157,9 +156,6 @@ private void compareMergedQuery(StructuredQuery expected, StructuredQuery @Test public void testToAndFromPb() { - EntityQuery a = ENTITY_QUERY; - StructuredQuery pb = - StructuredQuery.fromPb(ResultType.ENTITY, ENTITY_QUERY.getNamespace(), ENTITY_QUERY.toPb()); assertEquals( ENTITY_QUERY, StructuredQuery.fromPb( diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/VectorValueTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/VectorValueTest.java index d95e740a0..50f26d0a5 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/VectorValueTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/VectorValueTest.java @@ -29,7 +29,6 @@ public class VectorValueTest { @Test public void testToBuilder() { - // StringValue value = StringValue.of(CONTENT); VectorValue value = VectorValue.of(0.3, 4.2, 3.7); assertEquals(value, value.toBuilder().build()); } diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreTest.java index 7abc77db6..717426adc 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreTest.java @@ -2124,54 +2124,54 @@ public void testQueryWithVectorSearch() { Key.newBuilder(PROJECT_ID, KIND1, "name-01", options.getDatabaseId()).build()) .set( "vector_property", - VectorValue.newBuilder(3.0, 1.0, 2.0).setExcludeFromIndexes(true).build()) + VectorValue.newBuilder(3.0, 9.0, 11.1).setExcludeFromIndexes(true).build()) .build(); - datastore.put(entity1); - - VectorValue vectorValue = VectorValue.newBuilder(1.78, 2.56, 3.88).build(); - - // Query to find the nearest 1 neighbor - FindNearest vectorQuery = - new FindNearest( - "vector_property", vectorValue, FindNearest.DistanceMeasure.COSINE, 1, "distance"); - Query queryWithVectorSearch = - Query.newEntityQueryBuilder().setKind(KIND1).setFindNearest(vectorQuery).build(); - QueryResults vectorSearchResult = datastore.run(queryWithVectorSearch); - assertTrue(vectorSearchResult.hasNext()); - assertEquals(entity1, vectorSearchResult.next()); - assertFalse(vectorSearchResult.hasNext()); - Entity entity2 = Entity.newBuilder( Key.newBuilder(PROJECT_ID, KIND1, "name-02", options.getDatabaseId()).build()) .set( "vector_property", - VectorValue.newBuilder(5.0, 0.7, 2.0).setExcludeFromIndexes(true).build()) + VectorValue.newBuilder(2.8, 2.56, 3.8).setExcludeFromIndexes(true).build()) .build(); Entity entity3 = Entity.newBuilder( Key.newBuilder(PROJECT_ID, KIND1, "name-03", options.getDatabaseId()).build()) .set( "vector_property", - VectorValue.newBuilder(2.0, 1.7, 1.0).setExcludeFromIndexes(true).build()) + VectorValue.newBuilder(2.8, 2.56, 3.88).setExcludeFromIndexes(true).build()) .build(); - datastore.put(entity2, entity3); + datastore.put(entity1, entity2, entity3); - // Query to find the nearest 2 neighbors - FindNearest vectorQueryWithLimit = + // Query to find the nearest 2 neighbors with COSINE distance + FindNearest findNearestQuery = new FindNearest( "vector_property", - VectorValue.newBuilder(2.8, 2.56, 3.88).build(), - FindNearest.DistanceMeasure.EUCLIDEAN, + VectorValue.newBuilder(1.78, 2.56, 3.88).build(), + FindNearest.DistanceMeasure.COSINE, 2, "distance"); - - Query queryWithVectorSearchLimit = - Query.newEntityQueryBuilder().setKind(KIND1).setFindNearest(vectorQueryWithLimit).build(); - QueryResults resultsWithVectorLimit = datastore.run(queryWithVectorSearchLimit); - List resultsCopy = makeResultsCopy(resultsWithVectorLimit); + Query queryWithVectorSearch = + Query.newEntityQueryBuilder().setKind(KIND1).setFindNearest(findNearestQuery).build(); + QueryResults vectorSearchResult = datastore.run(queryWithVectorSearch); + List resultsCopy = makeResultsCopy(vectorSearchResult); + // Should return nearest 2 neighbors assertEquals(2, resultsCopy.size()); + // Query to find the nearest neighbor with EUCLIDEAN distance + FindNearest findNearestWithLimit1 = + new FindNearest( + "vector_property", + VectorValue.newBuilder(2.8, 2.56, 3.88).build(), + FindNearest.DistanceMeasure.EUCLIDEAN, + 1, + "distance"); + Query vectorQueryWithLimit1 = + Query.newEntityQueryBuilder().setKind(KIND1).setFindNearest(findNearestWithLimit1).build(); + QueryResults resultsWithVectorLimit1 = datastore.run(vectorQueryWithLimit1); + assertTrue(resultsWithVectorLimit1.hasNext()); + // entity3 should be the nearest neighbor + assertEquals(entity3, resultsWithVectorLimit1.next()); + assertFalse(resultsWithVectorLimit1.hasNext()); datastore.delete(entity1.getKey(), entity2.getKey(), entity3.getKey()); } From 9694abbfa64a0aeefd5ead1dec41e4456de22b7c Mon Sep 17 00:00:00 2001 From: Cindy Peng <148148319+cindy-peng@users.noreply.github.com> Date: Wed, 30 Oct 2024 13:16:51 -0700 Subject: [PATCH 23/33] Add setExcludeFromIndexes back to vectorvalue builder --- .../src/main/java/com/google/cloud/datastore/VectorValue.java | 1 + 1 file changed, 1 insertion(+) diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/VectorValue.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/VectorValue.java index 944e38405..9d02255fd 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/VectorValue.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/VectorValue.java @@ -142,6 +142,7 @@ public static VectorValue of(List> values) { /** Returns a builder for {@code VectorValue} objects. */ public static Builder newBuilder() { Builder builder = new VectorValue.Builder(); + builder.setExcludeFromIndexes(true); builder.setMeaning(VECTOR_MEANING); return builder; } From f267339f818f42ff8ecdd94fa238afac5f709250 Mon Sep 17 00:00:00 2001 From: Cindy Peng <148148319+cindy-peng@users.noreply.github.com> Date: Thu, 31 Oct 2024 09:41:54 -0700 Subject: [PATCH 24/33] Adjust testVectorSearch sample code --- .../datastore/snippets/ConceptsTest.java | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/com/google/datastore/snippets/ConceptsTest.java b/com/google/datastore/snippets/ConceptsTest.java index 8ffaf0f63..b31cb801e 100644 --- a/com/google/datastore/snippets/ConceptsTest.java +++ b/com/google/datastore/snippets/ConceptsTest.java @@ -731,20 +731,6 @@ public void testInequalitySortInvalidNotFirst() { assertInvalidQuery(query); } - @Test - public void testVectorSearch() { - setUpQueryTests(); - // [START datastore_vector_search] - VectorValue vectorValue = VectorValue.newBuilder(1.78, 2.56, 3.88).build(); - FindNearest vectorQuery = - new FindNearest( - "vector_property", vectorValue, FindNearest.DistanceMeasure.COSINE, 1, "distance"); - - Query query = Query.newEntityQueryBuilder().setFindNearest(vectorQuery).build(); - // [END datastore_vector_search] - assertValidQuery(query); - } - @Test public void testLimit() { setUpQueryTests(); @@ -1212,4 +1198,18 @@ public void testStaleReads() throws InterruptedException { // [END datastore_stale_read] assertValidQueryRealBackend(query); } + + @Test + public void testVectorSearch() { + setUpQueryTests(); + // [START datastore_vector_search] + VectorValue vectorValue = VectorValue.newBuilder(1.78, 2.56, 3.88).build(); + FindNearest vectorQuery = + new FindNearest( + "vector_property", vectorValue, FindNearest.DistanceMeasure.COSINE, 1, "distance"); + + Query query = Query.newEntityQueryBuilder().setFindNearest(vectorQuery).build(); + // [END datastore_vector_search] + assertValidQuery(query); + } } From e2f8e8746598826e3877a7405bebe8c0710ceb63 Mon Sep 17 00:00:00 2001 From: Cindy Peng <148148319+cindy-peng@users.noreply.github.com> Date: Mon, 2 Dec 2024 12:04:13 -0800 Subject: [PATCH 25/33] Add system tests check details --- .../datastore/snippets/ConceptsTest.java | 2 +- .../google/cloud/datastore/VectorValue.java | 2 - .../cloud/datastore/BaseEntityTest.java | 1 - .../com/google/cloud/datastore/ValueTest.java | 6 +- .../cloud/datastore/VectorValueTest.java | 8 +- .../datastore/it/ITDatastoreConceptsTest.java | 17 +- .../cloud/datastore/it/ITDatastoreTest.java | 198 ++++++++++-------- .../src/test/resources/index.yaml | 16 +- .../snippets/src/test/resources/index.yaml | 9 +- 9 files changed, 143 insertions(+), 116 deletions(-) diff --git a/com/google/datastore/snippets/ConceptsTest.java b/com/google/datastore/snippets/ConceptsTest.java index b31cb801e..aba364f42 100644 --- a/com/google/datastore/snippets/ConceptsTest.java +++ b/com/google/datastore/snippets/ConceptsTest.java @@ -412,7 +412,7 @@ private void setUpQueryTests() { .set("tag", "fun", "l", "programming", "learn") .set( "vector_property", - VectorValue.newBuilder(3.0, 1.0, 2.0).setExcludeFromIndexes(true).build()) + VectorValue.newBuilder(3.0, 1.0, 2.0).build()) .build()); } diff --git a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/VectorValue.java b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/VectorValue.java index 9d02255fd..ce27018c4 100644 --- a/google-cloud-datastore/src/main/java/com/google/cloud/datastore/VectorValue.java +++ b/google-cloud-datastore/src/main/java/com/google/cloud/datastore/VectorValue.java @@ -142,14 +142,12 @@ public static VectorValue of(List> values) { /** Returns a builder for {@code VectorValue} objects. */ public static Builder newBuilder() { Builder builder = new VectorValue.Builder(); - builder.setExcludeFromIndexes(true); builder.setMeaning(VECTOR_MEANING); return builder; } public static Builder newBuilder(double first, double... other) { VectorValue.Builder builder = new VectorValue.Builder(); - builder.setExcludeFromIndexes(true); builder.setMeaning(VECTOR_MEANING); return builder.addValue(first, other); } diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/BaseEntityTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/BaseEntityTest.java index fc33c64b0..4187e7b23 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/BaseEntityTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/BaseEntityTest.java @@ -39,7 +39,6 @@ public class BaseEntityTest { private static final VectorValue VECTOR = VectorValue.newBuilder(1.78, 2.56, 3.88) .setMeaning(VECTOR_MEANING) - .setExcludeFromIndexes(true) .build(); private static final Key KEY = Key.newBuilder("ds1", "k1", "n1").build(); private static final Entity ENTITY = Entity.newBuilder(KEY).set("name", "foo").build(); diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/ValueTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/ValueTest.java index dd36919a1..773746281 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/ValueTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/ValueTest.java @@ -125,11 +125,7 @@ public void testType() { @Test public void testExcludeFromIndexes() { for (Map.Entry> entry : typeToValue.entrySet()) { - if (entry.getKey() == ValueType.VECTOR) { - assertTrue(entry.getValue().excludeFromIndexes()); - } else { - assertFalse(entry.getValue().excludeFromIndexes()); - } + assertFalse(entry.getValue().excludeFromIndexes()); } TestBuilder builder = new TestBuilder(); assertFalse(builder.build().excludeFromIndexes()); diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/VectorValueTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/VectorValueTest.java index 50f26d0a5..69330195d 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/VectorValueTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/VectorValueTest.java @@ -39,12 +39,10 @@ public void testOf() { assertEquals( ImmutableList.of(DoubleValue.of(0.3), DoubleValue.of(4.2), DoubleValue.of(3.7)), value.get()); - assertTrue(value.excludeFromIndexes()); assertEquals(31, value.getMeaning()); - VectorValue value1 = VectorValue.of(vectorList); - assertEquals(vectorList, value1.get()); - assertTrue(value1.excludeFromIndexes()); - assertEquals(31, value1.getMeaning()); + VectorValue vectorListValue = VectorValue.of(vectorList); + assertEquals(vectorList, vectorListValue.get()); + assertEquals(31, vectorListValue.getMeaning()); } @SuppressWarnings("deprecation") diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreConceptsTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreConceptsTest.java index d96b083ef..07e59adf4 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreConceptsTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreConceptsTest.java @@ -157,8 +157,8 @@ private void setUpQueryTests() { StringValue.newBuilder("Learn Cloud Datastore").setExcludeFromIndexes(true).build()) .set("tag", "fun", "l", "programming", "learn") .set( - "vector_property", - VectorValue.newBuilder(3.0, 1.0, 2.0).setExcludeFromIndexes(true).build()) + "embedding_field", + VectorValue.newBuilder(3.0, 1.0, 2.0).build()) .build()); } @@ -580,9 +580,10 @@ public void testVectorSearch() { VectorValue vectorValue = VectorValue.newBuilder(1.78, 2.56, 3.88).build(); FindNearest vectorQuery = new FindNearest( - "vector_property", vectorValue, FindNearest.DistanceMeasure.COSINE, 1, "distance"); + "embedding_field", vectorValue, FindNearest.DistanceMeasure.COSINE, 1, "distance"); - Query query = Query.newEntityQueryBuilder().setFindNearest(vectorQuery).build(); + Query query = Query.newEntityQueryBuilder() + .setKind(TASK_CONCEPTS).setFindNearest(vectorQuery).build(); assertValidQuery(query); } @@ -590,8 +591,8 @@ public void testVectorSearch() { public void testVectorSearchWithEmptyVector() { VectorValue emptyVector = VectorValue.newBuilder().build(); FindNearest vectorQuery = - new FindNearest("vector_property", emptyVector, FindNearest.DistanceMeasure.EUCLIDEAN, 1); - Query query = Query.newEntityQueryBuilder().setFindNearest(vectorQuery).build(); + new FindNearest("embedding_field", emptyVector, FindNearest.DistanceMeasure.EUCLIDEAN, 1); + Query query = Query.newEntityQueryBuilder().setKind(TASK_CONCEPTS).setFindNearest(vectorQuery).build(); assertInvalidQuery(query); } @@ -599,8 +600,8 @@ public void testVectorSearchWithEmptyVector() { public void testVectorSearchWithUnmatchedVectorSize() { VectorValue vectorValue = VectorValue.newBuilder(1.78, 2.56, 3.88, 4.33).build(); FindNearest vectorQuery = - new FindNearest("vector_property", vectorValue, FindNearest.DistanceMeasure.DOT_PRODUCT, 1); - Query query = Query.newEntityQueryBuilder().setFindNearest(vectorQuery).build(); + new FindNearest("embedding_field", vectorValue, FindNearest.DistanceMeasure.DOT_PRODUCT, 1); + Query query = Query.newEntityQueryBuilder().setKind(TASK_CONCEPTS).setFindNearest(vectorQuery).build(); assertInvalidQuery(query); } diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreTest.java index 717426adc..0de647f4c 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreTest.java @@ -32,46 +32,12 @@ import com.google.cloud.Timestamp; import com.google.cloud.Tuple; -import com.google.cloud.datastore.AggregationQuery; -import com.google.cloud.datastore.AggregationResult; -import com.google.cloud.datastore.AggregationResults; -import com.google.cloud.datastore.Batch; -import com.google.cloud.datastore.BooleanValue; -import com.google.cloud.datastore.Cursor; -import com.google.cloud.datastore.Datastore; +import com.google.cloud.datastore.*; import com.google.cloud.datastore.Datastore.TransactionCallable; -import com.google.cloud.datastore.DatastoreException; -import com.google.cloud.datastore.DatastoreOptions; -import com.google.cloud.datastore.DatastoreReaderWriter; -import com.google.cloud.datastore.Entity; -import com.google.cloud.datastore.EntityQuery; -import com.google.cloud.datastore.EntityValue; -import com.google.cloud.datastore.FindNearest; -import com.google.cloud.datastore.FullEntity; -import com.google.cloud.datastore.GqlQuery; -import com.google.cloud.datastore.IncompleteKey; -import com.google.cloud.datastore.Key; -import com.google.cloud.datastore.KeyFactory; -import com.google.cloud.datastore.KeyValue; -import com.google.cloud.datastore.LatLng; -import com.google.cloud.datastore.LatLngValue; -import com.google.cloud.datastore.ListValue; -import com.google.cloud.datastore.NullValue; -import com.google.cloud.datastore.PathElement; -import com.google.cloud.datastore.ProjectionEntity; -import com.google.cloud.datastore.Query; import com.google.cloud.datastore.Query.ResultType; -import com.google.cloud.datastore.QueryResults; -import com.google.cloud.datastore.ReadOption; -import com.google.cloud.datastore.StringValue; -import com.google.cloud.datastore.StructuredQuery; import com.google.cloud.datastore.StructuredQuery.CompositeFilter; import com.google.cloud.datastore.StructuredQuery.OrderBy; import com.google.cloud.datastore.StructuredQuery.PropertyFilter; -import com.google.cloud.datastore.TimestampValue; -import com.google.cloud.datastore.Transaction; -import com.google.cloud.datastore.ValueType; -import com.google.cloud.datastore.VectorValue; import com.google.cloud.datastore.models.ExecutionStats; import com.google.cloud.datastore.models.ExplainMetrics; import com.google.cloud.datastore.models.ExplainOptions; @@ -127,6 +93,7 @@ public class ITDatastoreTest { private static final String KIND1 = "kind1"; private static final String KIND2 = "kind2"; private static final String KIND3 = "kind3"; + private static final String VECTOR_KIND = "CoffeeBean"; private static final NullValue NULL_VALUE = NullValue.of(); private static final StringValue STR_VALUE = StringValue.of("str"); private static final BooleanValue BOOL_VALUE = @@ -147,6 +114,7 @@ public class ITDatastoreTest { private static Key KEY4; private static Key KEY5; private static Key KEY6; + private static Key VECTORKEY; private static final String MARKS_KIND = "Marks"; private static FullEntity PARTIAL_ENTITY1; private static FullEntity PARTIAL_ENTITY2; @@ -158,6 +126,9 @@ public class ITDatastoreTest { private static Entity AGGREGATION_ENTITY_1; private static Entity AGGREGATION_ENTITY_2; private static Entity AGGREGATION_ENTITY_3; + private static Entity VECTOR_ENTITY_1; + private static Entity VECTOR_ENTITY_2; + private static Entity VECTOR_ENTITY_3; @Rule public Timeout globalTimeout = Timeout.seconds(100); @@ -178,6 +149,7 @@ public ITDatastoreTest( PROJECT_ID = this.options.getProjectId(); NAMESPACE = this.options.getNamespace(); + System.out.println("Project: " + PROJECT_ID + ", Namespace: " + NAMESPACE + ", db: " + options.getDatabaseId()); ROOT_KEY = Key.newBuilder(PROJECT_ID, "rootkey", "default", options.getDatabaseId()) @@ -200,6 +172,9 @@ public ITDatastoreTest( Key.newBuilder(options.getProjectId(), KIND2, 100, options.getDatabaseId()) .setNamespace(NAMESPACE) .build(); + VECTORKEY = Key.newBuilder(PROJECT_ID, VECTOR_KIND, "bean1", options.getDatabaseId()) + .setNamespace(NAMESPACE) + .build(); LIST_VALUE2 = ListValue.of(Collections.singletonList(KeyValue.of(KEY1))); @@ -249,6 +224,29 @@ public ITDatastoreTest( .set("partial1", PARTIAL_ENTITY2) .set("partial2", ENTITY2) .build(); + VECTOR_ENTITY_1 = + Entity.newBuilder(VECTORKEY) + .set("name", "Arabica") + .set( + "embedding_field", + VectorValue.newBuilder(1.0, 7.0, 11.1).build()) + .build(); + VECTOR_ENTITY_2 = + Entity.newBuilder( + Key.newBuilder(VECTORKEY).setName("bean2").build()) + .set("name", "Robusta") + .set( + "embedding_field", + VectorValue.newBuilder(1.0, 9.0, 11.1).build()).set("vector_distance", 0) + .build(); + VECTOR_ENTITY_3 = + Entity.newBuilder( + Key.newBuilder(VECTORKEY).setName("bean3").build()) + .set("name", "Excelsa") + .set( + "embedding_field", + VectorValue.newBuilder(4.0, 9.0, 11.1).build()) + .build(); Key aggregationKey1 = datastore.newKeyFactory().setKind(MARKS_KIND).newKey(1); Key aggregationKey2 = datastore.newKeyFactory().setKind(MARKS_KIND).newKey(2); @@ -291,7 +289,7 @@ public void tearDown() { @Parameterized.Parameters(name = "database: {2}") public static Iterable data() { return Arrays.asList( - new Object[][] {{OPTIONS_1, DATASTORE_1, "default"}, {OPTIONS_2, DATASTORE_2, "test-db"}}); + new Object[][] {{OPTIONS_1, DATASTORE_1, "default"}, {OPTIONS_2, DATASTORE_2, "test-db"}}); } private Iterator getStronglyConsistentResults(Query scQuery, Query query) @@ -2061,7 +2059,7 @@ public Integer run(DatastoreReaderWriter transaction) { public void testSkippedResults() { Query query = Query.newKeyQueryBuilder().setOffset(Integer.MAX_VALUE).build(); int numberOfEntities = datastore.run(query).getSkippedResults(); - assertEquals(2, numberOfEntities); + assertEquals(5, numberOfEntities); } @Test @@ -2118,61 +2116,77 @@ public void testQueryWithStartCursor() { } @Test - public void testQueryWithVectorSearch() { - Entity entity1 = - Entity.newBuilder( - Key.newBuilder(PROJECT_ID, KIND1, "name-01", options.getDatabaseId()).build()) - .set( - "vector_property", - VectorValue.newBuilder(3.0, 9.0, 11.1).setExcludeFromIndexes(true).build()) - .build(); - Entity entity2 = - Entity.newBuilder( - Key.newBuilder(PROJECT_ID, KIND1, "name-02", options.getDatabaseId()).build()) - .set( - "vector_property", - VectorValue.newBuilder(2.8, 2.56, 3.8).setExcludeFromIndexes(true).build()) - .build(); - Entity entity3 = - Entity.newBuilder( - Key.newBuilder(PROJECT_ID, KIND1, "name-03", options.getDatabaseId()).build()) - .set( - "vector_property", - VectorValue.newBuilder(2.8, 2.56, 3.88).setExcludeFromIndexes(true).build()) - .build(); - datastore.put(entity1, entity2, entity3); + public void testVectorSearchQueryWithLimit() { + datastore.put(VECTOR_ENTITY_1, VECTOR_ENTITY_2, VECTOR_ENTITY_3); + // Test FindNearest query with limit + FindNearest findNearestQueryWithLimit = + new FindNearest( + "embedding_field", + VectorValue.newBuilder(1, 9, 11.1).build(), + FindNearest.DistanceMeasure.DOT_PRODUCT, + 3); + Query queryWithLimit = + Query.newEntityQueryBuilder().setKind(VECTOR_KIND).setFindNearest(findNearestQueryWithLimit).build(); + + QueryResults resultWithLimit = datastore.run(queryWithLimit); + + List resultsCopyWithLimit = makeResultsCopy(resultWithLimit); + + // Verify limit was applied + assertEquals(3, resultsCopyWithLimit.size()); + } - // Query to find the nearest 2 neighbors with COSINE distance - FindNearest findNearestQuery = - new FindNearest( - "vector_property", - VectorValue.newBuilder(1.78, 2.56, 3.88).build(), - FindNearest.DistanceMeasure.COSINE, - 2, - "distance"); - Query queryWithVectorSearch = - Query.newEntityQueryBuilder().setKind(KIND1).setFindNearest(findNearestQuery).build(); - QueryResults vectorSearchResult = datastore.run(queryWithVectorSearch); - List resultsCopy = makeResultsCopy(vectorSearchResult); - // Should return nearest 2 neighbors - assertEquals(2, resultsCopy.size()); - - // Query to find the nearest neighbor with EUCLIDEAN distance - FindNearest findNearestWithLimit1 = - new FindNearest( - "vector_property", - VectorValue.newBuilder(2.8, 2.56, 3.88).build(), - FindNearest.DistanceMeasure.EUCLIDEAN, - 1, - "distance"); - Query vectorQueryWithLimit1 = - Query.newEntityQueryBuilder().setKind(KIND1).setFindNearest(findNearestWithLimit1).build(); - QueryResults resultsWithVectorLimit1 = datastore.run(vectorQueryWithLimit1); - assertTrue(resultsWithVectorLimit1.hasNext()); - // entity3 should be the nearest neighbor - assertEquals(entity3, resultsWithVectorLimit1.next()); - assertFalse(resultsWithVectorLimit1.hasNext()); - datastore.delete(entity1.getKey(), entity2.getKey(), entity3.getKey()); + @Test + public void testVectorSearchQueryWithDistanceThreshold() { + datastore.put(VECTOR_ENTITY_1, VECTOR_ENTITY_2, VECTOR_ENTITY_3); + + VectorValue vectorValue = VectorValue.newBuilder(1.78, 2.56, 3.88).build(); + FindNearest vectorQuery = + new FindNearest( + "embedding_field", vectorValue, FindNearest.DistanceMeasure.COSINE, 1, "distance"); + + Query query = Query.newEntityQueryBuilder().setFindNearest(vectorQuery).build(); + + + // Test FindNearest query with distanceThreshold + FindNearest findNearestQueryWithThreshold = + new FindNearest( + "embedding_field", + VectorValue.newBuilder(1, 9, 11.1).build(), + FindNearest.DistanceMeasure.EUCLIDEAN, + 3, "vector_distance", 2.0); + Query queryWithWithThreshold = + Query.newEntityQueryBuilder().setKind(VECTOR_KIND).setFindNearest(findNearestQueryWithThreshold).build(); + QueryResults resultWithThreshold = datastore.run(queryWithWithThreshold); + List resultsCopyWithThreshold = makeResultsCopy(resultWithThreshold); + // Verify threshold was applied regardless of limit + assertEquals(2, resultsCopyWithThreshold.size()); + // Verify qualified EUCLIDEAN distance: d((1, 9, 11.1), (1, 9, 11.1)) = 0.0, d((1, 9, 11.1), (1, 7, 11.1)) = 2.0 + assertEquals(DoubleValue.of(0.0), resultsCopyWithThreshold.get(0).getValue("vector_distance")); + assertEquals(DoubleValue.of(2.0), resultsCopyWithThreshold.get(1).getValue("vector_distance")); + } + + @Test + public void testQueryWithVectorSearchWithDistanceField() { + datastore.put(VECTOR_ENTITY_1, VECTOR_ENTITY_2, VECTOR_ENTITY_3); + // Test FindNearest query with distanceField + FindNearest findNearestQueryWithDistanceField = + new FindNearest( + "embedding_field", + VectorValue.newBuilder(1, 9, 11.1).build(), + FindNearest.DistanceMeasure.DOT_PRODUCT, + 3, "vector_distance", 0.0); + Query queryWithWithDistanceField = + Query.newEntityQueryBuilder().setKind(VECTOR_KIND).setFindNearest(findNearestQueryWithDistanceField).build(); + QueryResults resultWithDistanceField = datastore.run(queryWithWithDistanceField); + List resultsCopyWithDistanceField = makeResultsCopy(resultWithDistanceField); + // Verify results count + assertEquals(3, resultsCopyWithDistanceField.size()); + for (int i = 0; i < resultsCopyWithDistanceField.size(); i++) + { + // Verify distance field was not 0 + assertNotEquals(DoubleValue.of(0.0), resultsCopyWithDistanceField.get(i).getValue("vector_distance")); + } } @Test @@ -2224,7 +2238,7 @@ public void testQueryWithReadTime() throws InterruptedException { assertEquals(entity2, withReadTime.next()); assertFalse(withReadTime.hasNext()); } finally { - datastore.delete(entity1.getKey(), entity2.getKey(), entity3.getKey()); + // datastore.delete(entity1.getKey(), entity2.getKey(), entity3.getKey()); } } diff --git a/google-cloud-datastore/src/test/resources/index.yaml b/google-cloud-datastore/src/test/resources/index.yaml index ff1b08626..54f47b764 100644 --- a/google-cloud-datastore/src/test/resources/index.yaml +++ b/google-cloud-datastore/src/test/resources/index.yaml @@ -45,4 +45,18 @@ indexes: properties: - name: done - name: priority - direction: desc \ No newline at end of file + direction: desc + - kind: TaskConcepts + properties: + - name: __key__ + - name: embedding_field + vectorConfig: + dimension: 3 + flat: { } + - kind: CoffeeBean + properties: + - name: __key__ + - name: embedding_field + vectorConfig: + dimension: 3 + flat: { } \ No newline at end of file diff --git a/samples/snippets/src/test/resources/index.yaml b/samples/snippets/src/test/resources/index.yaml index 5f2f0c74a..8c8fa6d94 100644 --- a/samples/snippets/src/test/resources/index.yaml +++ b/samples/snippets/src/test/resources/index.yaml @@ -26,4 +26,11 @@ indexes: - kind: employees properties: - name: salary - - name: experience \ No newline at end of file + - name: experience +- kind: CoffeeBean + properties: + - name: __key__ + - name: embedding_field + vectorConfig: + dimension: 3 + flat: {} \ No newline at end of file From 1806f0a4459c10855e09da060398a28685eb829d Mon Sep 17 00:00:00 2001 From: cloud-java-bot Date: Mon, 2 Dec 2024 20:11:26 +0000 Subject: [PATCH 26/33] chore: generate libraries at Mon Dec 2 20:09:57 UTC 2024 --- .../datastore/snippets/ConceptsTest.java | 8 +- .../cloud/datastore/BaseEntityTest.java | 4 +- .../datastore/it/ITDatastoreConceptsTest.java | 14 +-- .../cloud/datastore/it/ITDatastoreTest.java | 111 ++++++++++-------- 4 files changed, 73 insertions(+), 64 deletions(-) diff --git a/com/google/datastore/snippets/ConceptsTest.java b/com/google/datastore/snippets/ConceptsTest.java index cc0c88751..51dcd4b3f 100644 --- a/com/google/datastore/snippets/ConceptsTest.java +++ b/com/google/datastore/snippets/ConceptsTest.java @@ -409,9 +409,7 @@ private void setUpQueryTests() { "description", StringValue.newBuilder("Learn Cloud Datastore").setExcludeFromIndexes(true).build()) .set("tag", "fun", "l", "programming", "learn") - .set( - "vector_property", - VectorValue.newBuilder(3.0, 1.0, 2.0).build()) + .set("vector_property", VectorValue.newBuilder(3.0, 1.0, 2.0).build()) .build()); } @@ -1204,8 +1202,8 @@ public void testVectorSearch() { // [START datastore_vector_search] VectorValue vectorValue = VectorValue.newBuilder(1.78, 2.56, 3.88).build(); FindNearest vectorQuery = - new FindNearest( - "vector_property", vectorValue, FindNearest.DistanceMeasure.COSINE, 1, "distance"); + new FindNearest( + "vector_property", vectorValue, FindNearest.DistanceMeasure.COSINE, 1, "distance"); Query query = Query.newEntityQueryBuilder().setFindNearest(vectorQuery).build(); // [END datastore_vector_search] diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/BaseEntityTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/BaseEntityTest.java index 4187e7b23..3e01999aa 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/BaseEntityTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/BaseEntityTest.java @@ -37,9 +37,7 @@ public class BaseEntityTest { private static final Timestamp TIMESTAMP = Timestamp.now(); private static final LatLng LAT_LNG = new LatLng(37.422035, -122.084124); private static final VectorValue VECTOR = - VectorValue.newBuilder(1.78, 2.56, 3.88) - .setMeaning(VECTOR_MEANING) - .build(); + VectorValue.newBuilder(1.78, 2.56, 3.88).setMeaning(VECTOR_MEANING).build(); private static final Key KEY = Key.newBuilder("ds1", "k1", "n1").build(); private static final Entity ENTITY = Entity.newBuilder(KEY).set("name", "foo").build(); private static final IncompleteKey INCOMPLETE_KEY = IncompleteKey.newBuilder("ds1", "k1").build(); diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreConceptsTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreConceptsTest.java index 07e59adf4..684730365 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreConceptsTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreConceptsTest.java @@ -156,9 +156,7 @@ private void setUpQueryTests() { "description", StringValue.newBuilder("Learn Cloud Datastore").setExcludeFromIndexes(true).build()) .set("tag", "fun", "l", "programming", "learn") - .set( - "embedding_field", - VectorValue.newBuilder(3.0, 1.0, 2.0).build()) + .set("embedding_field", VectorValue.newBuilder(3.0, 1.0, 2.0).build()) .build()); } @@ -582,8 +580,8 @@ public void testVectorSearch() { new FindNearest( "embedding_field", vectorValue, FindNearest.DistanceMeasure.COSINE, 1, "distance"); - Query query = Query.newEntityQueryBuilder() - .setKind(TASK_CONCEPTS).setFindNearest(vectorQuery).build(); + Query query = + Query.newEntityQueryBuilder().setKind(TASK_CONCEPTS).setFindNearest(vectorQuery).build(); assertValidQuery(query); } @@ -592,7 +590,8 @@ public void testVectorSearchWithEmptyVector() { VectorValue emptyVector = VectorValue.newBuilder().build(); FindNearest vectorQuery = new FindNearest("embedding_field", emptyVector, FindNearest.DistanceMeasure.EUCLIDEAN, 1); - Query query = Query.newEntityQueryBuilder().setKind(TASK_CONCEPTS).setFindNearest(vectorQuery).build(); + Query query = + Query.newEntityQueryBuilder().setKind(TASK_CONCEPTS).setFindNearest(vectorQuery).build(); assertInvalidQuery(query); } @@ -601,7 +600,8 @@ public void testVectorSearchWithUnmatchedVectorSize() { VectorValue vectorValue = VectorValue.newBuilder(1.78, 2.56, 3.88, 4.33).build(); FindNearest vectorQuery = new FindNearest("embedding_field", vectorValue, FindNearest.DistanceMeasure.DOT_PRODUCT, 1); - Query query = Query.newEntityQueryBuilder().setKind(TASK_CONCEPTS).setFindNearest(vectorQuery).build(); + Query query = + Query.newEntityQueryBuilder().setKind(TASK_CONCEPTS).setFindNearest(vectorQuery).build(); assertInvalidQuery(query); } diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreTest.java index 302bbf6d1..76ac0ef3f 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreTest.java @@ -149,7 +149,13 @@ public ITDatastoreTest( PROJECT_ID = this.options.getProjectId(); NAMESPACE = this.options.getNamespace(); - System.out.println("Project: " + PROJECT_ID + ", Namespace: " + NAMESPACE + ", db: " + options.getDatabaseId()); + System.out.println( + "Project: " + + PROJECT_ID + + ", Namespace: " + + NAMESPACE + + ", db: " + + options.getDatabaseId()); ROOT_KEY = Key.newBuilder(PROJECT_ID, "rootkey", "default", options.getDatabaseId()) @@ -172,7 +178,8 @@ public ITDatastoreTest( Key.newBuilder(options.getProjectId(), KIND2, 100, options.getDatabaseId()) .setNamespace(NAMESPACE) .build(); - VECTORKEY = Key.newBuilder(PROJECT_ID, VECTOR_KIND, "bean1", options.getDatabaseId()) + VECTORKEY = + Key.newBuilder(PROJECT_ID, VECTOR_KIND, "bean1", options.getDatabaseId()) .setNamespace(NAMESPACE) .build(); @@ -225,28 +232,21 @@ public ITDatastoreTest( .set("partial2", ENTITY2) .build(); VECTOR_ENTITY_1 = - Entity.newBuilder(VECTORKEY) - .set("name", "Arabica") - .set( - "embedding_field", - VectorValue.newBuilder(1.0, 7.0, 11.1).build()) - .build(); + Entity.newBuilder(VECTORKEY) + .set("name", "Arabica") + .set("embedding_field", VectorValue.newBuilder(1.0, 7.0, 11.1).build()) + .build(); VECTOR_ENTITY_2 = - Entity.newBuilder( - Key.newBuilder(VECTORKEY).setName("bean2").build()) - .set("name", "Robusta") - .set( - "embedding_field", - VectorValue.newBuilder(1.0, 9.0, 11.1).build()).set("vector_distance", 0) - .build(); + Entity.newBuilder(Key.newBuilder(VECTORKEY).setName("bean2").build()) + .set("name", "Robusta") + .set("embedding_field", VectorValue.newBuilder(1.0, 9.0, 11.1).build()) + .set("vector_distance", 0) + .build(); VECTOR_ENTITY_3 = - Entity.newBuilder( - Key.newBuilder(VECTORKEY).setName("bean3").build()) - .set("name", "Excelsa") - .set( - "embedding_field", - VectorValue.newBuilder(4.0, 9.0, 11.1).build()) - .build(); + Entity.newBuilder(Key.newBuilder(VECTORKEY).setName("bean3").build()) + .set("name", "Excelsa") + .set("embedding_field", VectorValue.newBuilder(4.0, 9.0, 11.1).build()) + .build(); Key aggregationKey1 = datastore.newKeyFactory().setKind(MARKS_KIND).newKey(1); Key aggregationKey2 = datastore.newKeyFactory().setKind(MARKS_KIND).newKey(2); @@ -289,7 +289,7 @@ public void tearDown() { @Parameterized.Parameters(name = "database: {2}") public static Iterable data() { return Arrays.asList( - new Object[][] {{OPTIONS_1, DATASTORE_1, "default"}, {OPTIONS_2, DATASTORE_2, "test-db"}}); + new Object[][] {{OPTIONS_1, DATASTORE_1, "default"}, {OPTIONS_2, DATASTORE_2, "test-db"}}); } private Iterator getStronglyConsistentResults(Query scQuery, Query query) @@ -2120,13 +2120,16 @@ public void testVectorSearchQueryWithLimit() { datastore.put(VECTOR_ENTITY_1, VECTOR_ENTITY_2, VECTOR_ENTITY_3); // Test FindNearest query with limit FindNearest findNearestQueryWithLimit = - new FindNearest( - "embedding_field", - VectorValue.newBuilder(1, 9, 11.1).build(), - FindNearest.DistanceMeasure.DOT_PRODUCT, - 3); + new FindNearest( + "embedding_field", + VectorValue.newBuilder(1, 9, 11.1).build(), + FindNearest.DistanceMeasure.DOT_PRODUCT, + 3); Query queryWithLimit = - Query.newEntityQueryBuilder().setKind(VECTOR_KIND).setFindNearest(findNearestQueryWithLimit).build(); + Query.newEntityQueryBuilder() + .setKind(VECTOR_KIND) + .setFindNearest(findNearestQueryWithLimit) + .build(); QueryResults resultWithLimit = datastore.run(queryWithLimit); @@ -2142,26 +2145,31 @@ public void testVectorSearchQueryWithDistanceThreshold() { VectorValue vectorValue = VectorValue.newBuilder(1.78, 2.56, 3.88).build(); FindNearest vectorQuery = - new FindNearest( - "embedding_field", vectorValue, FindNearest.DistanceMeasure.COSINE, 1, "distance"); + new FindNearest( + "embedding_field", vectorValue, FindNearest.DistanceMeasure.COSINE, 1, "distance"); Query query = Query.newEntityQueryBuilder().setFindNearest(vectorQuery).build(); - // Test FindNearest query with distanceThreshold FindNearest findNearestQueryWithThreshold = - new FindNearest( - "embedding_field", - VectorValue.newBuilder(1, 9, 11.1).build(), - FindNearest.DistanceMeasure.EUCLIDEAN, - 3, "vector_distance", 2.0); + new FindNearest( + "embedding_field", + VectorValue.newBuilder(1, 9, 11.1).build(), + FindNearest.DistanceMeasure.EUCLIDEAN, + 3, + "vector_distance", + 2.0); Query queryWithWithThreshold = - Query.newEntityQueryBuilder().setKind(VECTOR_KIND).setFindNearest(findNearestQueryWithThreshold).build(); + Query.newEntityQueryBuilder() + .setKind(VECTOR_KIND) + .setFindNearest(findNearestQueryWithThreshold) + .build(); QueryResults resultWithThreshold = datastore.run(queryWithWithThreshold); List resultsCopyWithThreshold = makeResultsCopy(resultWithThreshold); // Verify threshold was applied regardless of limit assertEquals(2, resultsCopyWithThreshold.size()); - // Verify qualified EUCLIDEAN distance: d((1, 9, 11.1), (1, 9, 11.1)) = 0.0, d((1, 9, 11.1), (1, 7, 11.1)) = 2.0 + // Verify qualified EUCLIDEAN distance: d((1, 9, 11.1), (1, 9, 11.1)) = 0.0, d((1, 9, 11.1), (1, + // 7, 11.1)) = 2.0 assertEquals(DoubleValue.of(0.0), resultsCopyWithThreshold.get(0).getValue("vector_distance")); assertEquals(DoubleValue.of(2.0), resultsCopyWithThreshold.get(1).getValue("vector_distance")); } @@ -2171,21 +2179,26 @@ public void testQueryWithVectorSearchWithDistanceField() { datastore.put(VECTOR_ENTITY_1, VECTOR_ENTITY_2, VECTOR_ENTITY_3); // Test FindNearest query with distanceField FindNearest findNearestQueryWithDistanceField = - new FindNearest( - "embedding_field", - VectorValue.newBuilder(1, 9, 11.1).build(), - FindNearest.DistanceMeasure.DOT_PRODUCT, - 3, "vector_distance", 0.0); + new FindNearest( + "embedding_field", + VectorValue.newBuilder(1, 9, 11.1).build(), + FindNearest.DistanceMeasure.DOT_PRODUCT, + 3, + "vector_distance", + 0.0); Query queryWithWithDistanceField = - Query.newEntityQueryBuilder().setKind(VECTOR_KIND).setFindNearest(findNearestQueryWithDistanceField).build(); + Query.newEntityQueryBuilder() + .setKind(VECTOR_KIND) + .setFindNearest(findNearestQueryWithDistanceField) + .build(); QueryResults resultWithDistanceField = datastore.run(queryWithWithDistanceField); List resultsCopyWithDistanceField = makeResultsCopy(resultWithDistanceField); // Verify results count assertEquals(3, resultsCopyWithDistanceField.size()); - for (int i = 0; i < resultsCopyWithDistanceField.size(); i++) - { + for (int i = 0; i < resultsCopyWithDistanceField.size(); i++) { // Verify distance field was not 0 - assertNotEquals(DoubleValue.of(0.0), resultsCopyWithDistanceField.get(i).getValue("vector_distance")); + assertNotEquals( + DoubleValue.of(0.0), resultsCopyWithDistanceField.get(i).getValue("vector_distance")); } } @@ -2238,7 +2251,7 @@ public void testQueryWithReadTime() throws InterruptedException { assertEquals(entity2, withReadTime.next()); assertFalse(withReadTime.hasNext()); } finally { - // datastore.delete(entity1.getKey(), entity2.getKey(), entity3.getKey()); + // datastore.delete(entity1.getKey(), entity2.getKey(), entity3.getKey()); } } From fca3689a249aa2d546b3bc0316c0a9a29cc807c0 Mon Sep 17 00:00:00 2001 From: Cindy Peng <148148319+cindy-peng@users.noreply.github.com> Date: Mon, 2 Dec 2024 12:39:25 -0800 Subject: [PATCH 27/33] fix initial numer of entities for ITDatastoreTst --- .../java/com/google/cloud/datastore/it/ITDatastoreTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreTest.java b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreTest.java index 302bbf6d1..d57bb69b8 100644 --- a/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreTest.java +++ b/google-cloud-datastore/src/test/java/com/google/cloud/datastore/it/ITDatastoreTest.java @@ -2059,7 +2059,7 @@ public Integer run(DatastoreReaderWriter transaction) { public void testSkippedResults() { Query query = Query.newKeyQueryBuilder().setOffset(Integer.MAX_VALUE).build(); int numberOfEntities = datastore.run(query).getSkippedResults(); - assertEquals(5, numberOfEntities); + assertEquals(2, numberOfEntities); } @Test From 60dd19d122caf7f540aa0d0421cd1a16bd236f67 Mon Sep 17 00:00:00 2001 From: Cindy Peng <148148319+cindy-peng@users.noreply.github.com> Date: Mon, 2 Dec 2024 16:28:36 -0800 Subject: [PATCH 28/33] Add sample code for Java datastore vector search --- .../datastore/filters/OrderFieldsQuery.java | 2 +- .../datastore/vectorsearch/StoreVectors.java | 53 +++++++++++++++ .../vectorsearch/VectorSearchBasic.java | 58 +++++++++++++++++ .../VectorSearchDistanceResultProperty.java | 58 +++++++++++++++++ ...earchDistanceResultPropertyProjection.java | 59 +++++++++++++++++ .../VectorSearchDistanceThreshold.java | 58 +++++++++++++++++ .../VectorSearchLargeResponse.java | 65 +++++++++++++++++++ .../vectorsearch/VectorSearchPrefilter.java | 60 +++++++++++++++++ 8 files changed, 412 insertions(+), 1 deletion(-) create mode 100644 samples/snippets/src/main/java/com/example/datastore/vectorsearch/StoreVectors.java create mode 100644 samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchBasic.java create mode 100644 samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultProperty.java create mode 100644 samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultPropertyProjection.java create mode 100644 samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceThreshold.java create mode 100644 samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchLargeResponse.java create mode 100644 samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchPrefilter.java diff --git a/samples/snippets/src/main/java/com/example/datastore/filters/OrderFieldsQuery.java b/samples/snippets/src/main/java/com/example/datastore/filters/OrderFieldsQuery.java index 24fc7901c..f55f2f7f0 100644 --- a/samples/snippets/src/main/java/com/example/datastore/filters/OrderFieldsQuery.java +++ b/samples/snippets/src/main/java/com/example/datastore/filters/OrderFieldsQuery.java @@ -17,7 +17,7 @@ package com.example.datastore.filters; // sample-metadata: -// title: Queries with order fileds +// title: Queries with order fields // description: The following query order properties // in the decreasing order of query constraint selectivity. diff --git a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/StoreVectors.java b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/StoreVectors.java new file mode 100644 index 000000000..510756577 --- /dev/null +++ b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/StoreVectors.java @@ -0,0 +1,53 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.datastore.vectorsearch; + +// [START datastore_store_vectors] + +import com.google.cloud.datastore.Datastore; +import com.google.cloud.datastore.DatastoreOptions; +import com.google.cloud.datastore.Key; +import com.google.cloud.datastore.Entity; +import com.google.cloud.datastore.VectorValue; + +public class StoreVectors { + public static void invoke() throws Exception { + // Instantiates a client + Datastore datastore = DatastoreOptions.getDefaultInstance().getService(); + + // The Cloud Datastore key for the new entity + Key key = datastore.newKeyFactory().setKind("CoffeeBean").newKey("Kahawa"); + + // Prepares the entity with a vector embedding + Entity entity = + Entity.newBuilder(key) + .set("name", "Kahawa") + .set("description", "Information about the Kahawa coffee beans.") + .set("roast", "dark") + .set("embedding_field", VectorValue.newBuilder(1.0, 7.0, 11.1).build()) + .build(); + + // Saves the entity + datastore.put(entity); + System.out.printf("Saved %s: %s%n", entity.getKey().getName(), entity.getString("description")); + + // Retrieve entity + Entity retrieved = datastore.get(key); + System.out.printf("Retrieved %s: %s%n", key.getName(), retrieved.getString("description")); + } +} +// [END datastore_store_vectors] diff --git a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchBasic.java b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchBasic.java new file mode 100644 index 000000000..1f437101d --- /dev/null +++ b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchBasic.java @@ -0,0 +1,58 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.datastore.vectorsearch; + +// [START datastore_vector_search_basic] + +import com.google.cloud.datastore.Datastore; +import com.google.cloud.datastore.DatastoreOptions; +import com.google.cloud.datastore.Entity; +import com.google.cloud.datastore.Query; +import com.google.cloud.datastore.QueryResults; +import com.google.cloud.datastore.VectorValue; +import com.google.cloud.datastore.FindNearest; + +public class VectorSearchBasic { + public static void invoke() throws Exception { + // Instantiates a client + Datastore datastore = DatastoreOptions.getDefaultInstance().getService(); + + // Create vector search query + Query vectorSearchQuery = + Query.newEntityQueryBuilder() + .setKind("CoffeeBean") + .setFindNearest(new FindNearest( + "embedding_field", + VectorValue.newBuilder(1, 9, 11.1).build(), + FindNearest.DistanceMeasure.DOT_PRODUCT, + 3)) + .build(); + + // Execute vector search query + QueryResults results = datastore.run(vectorSearchQuery); + + if (!results.hasNext()) { + throw new Exception("query yielded no results"); + } + + while (results.hasNext()) { + Entity entity = results.next(); + System.out.printf("Entity: %s%n", entity.getKey().getName()); + } + } +} +// [END datastore_vector_search_basic] diff --git a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultProperty.java b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultProperty.java new file mode 100644 index 000000000..62f335024 --- /dev/null +++ b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultProperty.java @@ -0,0 +1,58 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.datastore.vectorsearch; + +// [START datastore_vector_search_distance_result_property] + +import com.google.cloud.datastore.Datastore; +import com.google.cloud.datastore.DatastoreOptions; +import com.google.cloud.datastore.Entity; +import com.google.cloud.datastore.Query; +import com.google.cloud.datastore.QueryResults; +import com.google.cloud.datastore.VectorValue; +import com.google.cloud.datastore.FindNearest; + +public class VectorSearchDistanceResultProperty { + public static void invoke() throws Exception { + // Instantiates a client + Datastore datastore = DatastoreOptions.getDefaultInstance().getService(); + + // Create vector search query with distance result property + Query vectorSearchQuery = + Query.newEntityQueryBuilder() + .setKind("CoffeeBean") + .setFindNearest(new FindNearest( + "embedding_field", + VectorValue.newBuilder(1, 9, 11.1).build(), + FindNearest.DistanceMeasure.DOT_PRODUCT, + 3, "vector_distance")) + .build(); + + // Execute vector search query + QueryResults results = datastore.run(vectorSearchQuery); + + if (!results.hasNext()) { + throw new Exception("query yielded no results"); + } + + while (results.hasNext()) { + Entity entity = results.next(); + System.out.printf("Entity: %s, Distance: %s%n", entity.getKey().getName(), entity.getDouble("vector_distance")); + } + } +} +// [END datastore_vector_search_distance_result_property] diff --git a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultPropertyProjection.java b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultPropertyProjection.java new file mode 100644 index 000000000..72960ac84 --- /dev/null +++ b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultPropertyProjection.java @@ -0,0 +1,59 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.datastore.vectorsearch; + +// [START datastore_vector_search_distance_result_property_projection] + +import com.google.cloud.datastore.Datastore; +import com.google.cloud.datastore.DatastoreOptions; +import com.google.cloud.datastore.Query; +import com.google.cloud.datastore.QueryResults; +import com.google.cloud.datastore.ProjectionEntity; +import com.google.cloud.datastore.VectorValue; +import com.google.cloud.datastore.FindNearest; + +public class VectorSearchDistanceResultPropertyProjection { + public static void invoke() throws Exception { + // Instantiates a client + Datastore datastore = DatastoreOptions.getDefaultInstance().getService(); + + // Create vector search query with projection + Query vectorSearchQuery = + Query.newProjectionEntityQueryBuilder() + .setKind("CoffeeBean") + .setFindNearest(new FindNearest( + "embedding_field", + VectorValue.newBuilder(1, 9, 11.1).build(), + FindNearest.DistanceMeasure.DOT_PRODUCT, + 3, "vector_distance")) + .setProjection("vector_distance") + .build(); + + // Execute vector search query + QueryResults results = datastore.run(vectorSearchQuery); + + if (!results.hasNext()) { + throw new Exception("query yielded no results"); + } + + while (results.hasNext()) { + ProjectionEntity entity = results.next(); + System.out.printf("Entity: %s, Distance: %s%n", entity.getKey().getName(), entity.getDouble("vector_distance")); + } + } +} +// [END datastore_vector_search_distance_result_property_projection] diff --git a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceThreshold.java b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceThreshold.java new file mode 100644 index 000000000..201e9d5ca --- /dev/null +++ b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceThreshold.java @@ -0,0 +1,58 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.datastore.vectorsearch; + +// [START datastore_vector_search_distance_threshold] + +import com.google.cloud.datastore.Datastore; +import com.google.cloud.datastore.DatastoreOptions; +import com.google.cloud.datastore.Entity; +import com.google.cloud.datastore.Query; +import com.google.cloud.datastore.QueryResults; +import com.google.cloud.datastore.VectorValue; +import com.google.cloud.datastore.FindNearest; + +public class VectorSearchDistanceThreshold { + public static void invoke() throws Exception { + // Instantiates a client + Datastore datastore = DatastoreOptions.getDefaultInstance().getService(); + + // Create vector search query with distance threshold + Query vectorSearchQuery = + Query.newEntityQueryBuilder() + .setKind("CoffeeBean") + .setFindNearest(new FindNearest( + "embedding_field", + VectorValue.newBuilder(1, 9, 11.1).build(), + FindNearest.DistanceMeasure.EUCLIDEAN, + 3, "vector_distance", 2.0)) + .build(); + + // Execute vector search query + QueryResults results = datastore.run(vectorSearchQuery); + + if (!results.hasNext()) { + throw new Exception("query yielded no results"); + } + + while (results.hasNext()) { + Entity entity = results.next(); + System.out.printf("Entity: %s, Distance: %s%n", entity.getKey().getName(), entity.getDouble("vector_distance")); + } + } +} +// [END datastore_vector_search_distance_threshold] diff --git a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchLargeResponse.java b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchLargeResponse.java new file mode 100644 index 000000000..070eb075c --- /dev/null +++ b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchLargeResponse.java @@ -0,0 +1,65 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.datastore.vectorsearch; + +// [START datastore_vector_search_large_response] + +import com.google.common.collect.Iterators; +import java.util.Iterator; +import com.google.cloud.datastore.Datastore; +import com.google.cloud.datastore.DatastoreOptions; +import com.google.cloud.datastore.Entity; +import com.google.cloud.datastore.Query; +import com.google.cloud.datastore.QueryResults; +import com.google.cloud.datastore.Key; +import com.google.cloud.datastore.VectorValue; +import com.google.cloud.datastore.FindNearest; + +public class VectorSearchLargeResponse { + public static void invoke() throws Exception { + // Instantiates a client + Datastore datastore = DatastoreOptions.getDefaultInstance().getService(); + + // Create a keys-only vector search query + Query vectorSearchKeyOnlyQuery = + Query.newKeyQueryBuilder() + .setKind("CoffeeBean") + .setFindNearest(new FindNearest( + "embedding_field", + VectorValue.newBuilder(1, 9, 11.1).build(), + FindNearest.DistanceMeasure.EUCLIDEAN, + 3, "vector_distance", 2.0)) + .build(); + + QueryResults keyResults = datastore.run(vectorSearchKeyOnlyQuery); + Key[] keys = Iterators.toArray(keyResults, Key.class); + + // Next, perform a second query for the remaining data + Iterator entities = datastore.get(keys); + + if (!keyResults.hasNext()) { + throw new Exception("query yielded no results"); + } + + // Combine and print results + while (keyResults.hasNext()) { + Key keyResult = keyResults.next(); + System.out.printf("Entity: %s, Distance: %s%n", keyResult.getName(), entities.next().getDouble("vector_distance")); + } + } +} +// [END datastore_vector_search_large_response] diff --git a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchPrefilter.java b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchPrefilter.java new file mode 100644 index 000000000..e24854aec --- /dev/null +++ b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchPrefilter.java @@ -0,0 +1,60 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.example.datastore.vectorsearch; + +// [START datastore_vector_search_prefilter] + +import com.google.cloud.datastore.Datastore; +import com.google.cloud.datastore.DatastoreOptions; +import com.google.cloud.datastore.Entity; +import com.google.cloud.datastore.Query; +import com.google.cloud.datastore.StructuredQuery.PropertyFilter; +import com.google.cloud.datastore.QueryResults; +import com.google.cloud.datastore.VectorValue; +import com.google.cloud.datastore.FindNearest; + +public class VectorSearchPrefilter { + public static void invoke() throws Exception { + // Instantiates a client + Datastore datastore = DatastoreOptions.getDefaultInstance().getService(); + + // Create vector search query with property filter + Query vectorSearchQuery = + Query.newEntityQueryBuilder() + .setKind("CoffeeBean") + .setFilter(PropertyFilter.eq("roast", "dark")) + .setFindNearest(new FindNearest( + "embedding_field", + VectorValue.newBuilder(1, 9, 11.1).build(), + FindNearest.DistanceMeasure.DOT_PRODUCT, + 3)) + .build(); + + // Execute vector search query + QueryResults results = datastore.run(vectorSearchQuery); + + if (!results.hasNext()) { + throw new Exception("query yielded no results"); + } + + while (results.hasNext()) { + Entity entity = results.next(); + System.out.printf("Entity: %s%n", entity.getKey().getName()); + } + } +} +// [END datastore_vector_search_prefilter] From d7c25593866be8d5c4923bba0773d248cc34b6ed Mon Sep 17 00:00:00 2001 From: cloud-java-bot Date: Tue, 3 Dec 2024 00:31:20 +0000 Subject: [PATCH 29/33] chore: generate libraries at Tue Dec 3 00:29:52 UTC 2024 --- README.md | 7 +++++ .../datastore/vectorsearch/StoreVectors.java | 14 ++++----- .../vectorsearch/VectorSearchBasic.java | 19 ++++++------ .../VectorSearchDistanceResultProperty.java | 24 ++++++++------ ...earchDistanceResultPropertyProjection.java | 28 ++++++++++------- .../VectorSearchDistanceThreshold.java | 25 +++++++++------ .../VectorSearchLargeResponse.java | 31 +++++++++++-------- .../vectorsearch/VectorSearchPrefilter.java | 23 +++++++------- 8 files changed, 99 insertions(+), 72 deletions(-) diff --git a/README.md b/README.md index f059e75ed..c1d264369 100644 --- a/README.md +++ b/README.md @@ -375,6 +375,13 @@ Samples are in the [`samples/`](https://github.com/googleapis/java-datastore/tre | Query Profile Explain Aggregation | [source code](https://github.com/googleapis/java-datastore/blob/main/samples/snippets/src/main/java/com/example/datastore/queryprofile/QueryProfileExplainAggregation.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-datastore&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/datastore/queryprofile/QueryProfileExplainAggregation.java) | | Query Profile Explain Analyze | [source code](https://github.com/googleapis/java-datastore/blob/main/samples/snippets/src/main/java/com/example/datastore/queryprofile/QueryProfileExplainAnalyze.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-datastore&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/datastore/queryprofile/QueryProfileExplainAnalyze.java) | | Query Profile Explain Analyze Aggregation | [source code](https://github.com/googleapis/java-datastore/blob/main/samples/snippets/src/main/java/com/example/datastore/queryprofile/QueryProfileExplainAnalyzeAggregation.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-datastore&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/datastore/queryprofile/QueryProfileExplainAnalyzeAggregation.java) | +| Store Vectors | [source code](https://github.com/googleapis/java-datastore/blob/main/samples/snippets/src/main/java/com/example/datastore/vectorsearch/StoreVectors.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-datastore&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/datastore/vectorsearch/StoreVectors.java) | +| Vector Search Basic | [source code](https://github.com/googleapis/java-datastore/blob/main/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchBasic.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-datastore&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchBasic.java) | +| Vector Search Distance Result Property | [source code](https://github.com/googleapis/java-datastore/blob/main/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultProperty.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-datastore&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultProperty.java) | +| Vector Search Distance Result Property Projection | [source code](https://github.com/googleapis/java-datastore/blob/main/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultPropertyProjection.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-datastore&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultPropertyProjection.java) | +| Vector Search Distance Threshold | [source code](https://github.com/googleapis/java-datastore/blob/main/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceThreshold.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-datastore&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceThreshold.java) | +| Vector Search Large Response | [source code](https://github.com/googleapis/java-datastore/blob/main/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchLargeResponse.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-datastore&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchLargeResponse.java) | +| Vector Search Prefilter | [source code](https://github.com/googleapis/java-datastore/blob/main/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchPrefilter.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-datastore&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchPrefilter.java) | | Task List | [source code](https://github.com/googleapis/java-datastore/blob/main/samples/snippets/src/main/java/com/google/datastore/snippets/TaskList.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-datastore&page=editor&open_in_editor=samples/snippets/src/main/java/com/google/datastore/snippets/TaskList.java) | diff --git a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/StoreVectors.java b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/StoreVectors.java index 510756577..f7cc236ae 100644 --- a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/StoreVectors.java +++ b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/StoreVectors.java @@ -20,8 +20,8 @@ import com.google.cloud.datastore.Datastore; import com.google.cloud.datastore.DatastoreOptions; -import com.google.cloud.datastore.Key; import com.google.cloud.datastore.Entity; +import com.google.cloud.datastore.Key; import com.google.cloud.datastore.VectorValue; public class StoreVectors { @@ -34,12 +34,12 @@ public static void invoke() throws Exception { // Prepares the entity with a vector embedding Entity entity = - Entity.newBuilder(key) - .set("name", "Kahawa") - .set("description", "Information about the Kahawa coffee beans.") - .set("roast", "dark") - .set("embedding_field", VectorValue.newBuilder(1.0, 7.0, 11.1).build()) - .build(); + Entity.newBuilder(key) + .set("name", "Kahawa") + .set("description", "Information about the Kahawa coffee beans.") + .set("roast", "dark") + .set("embedding_field", VectorValue.newBuilder(1.0, 7.0, 11.1).build()) + .build(); // Saves the entity datastore.put(entity); diff --git a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchBasic.java b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchBasic.java index 1f437101d..e32199105 100644 --- a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchBasic.java +++ b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchBasic.java @@ -21,10 +21,10 @@ import com.google.cloud.datastore.Datastore; import com.google.cloud.datastore.DatastoreOptions; import com.google.cloud.datastore.Entity; +import com.google.cloud.datastore.FindNearest; import com.google.cloud.datastore.Query; import com.google.cloud.datastore.QueryResults; import com.google.cloud.datastore.VectorValue; -import com.google.cloud.datastore.FindNearest; public class VectorSearchBasic { public static void invoke() throws Exception { @@ -33,14 +33,15 @@ public static void invoke() throws Exception { // Create vector search query Query vectorSearchQuery = - Query.newEntityQueryBuilder() - .setKind("CoffeeBean") - .setFindNearest(new FindNearest( - "embedding_field", - VectorValue.newBuilder(1, 9, 11.1).build(), - FindNearest.DistanceMeasure.DOT_PRODUCT, - 3)) - .build(); + Query.newEntityQueryBuilder() + .setKind("CoffeeBean") + .setFindNearest( + new FindNearest( + "embedding_field", + VectorValue.newBuilder(1, 9, 11.1).build(), + FindNearest.DistanceMeasure.DOT_PRODUCT, + 3)) + .build(); // Execute vector search query QueryResults results = datastore.run(vectorSearchQuery); diff --git a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultProperty.java b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultProperty.java index 62f335024..ca308828b 100644 --- a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultProperty.java +++ b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultProperty.java @@ -21,10 +21,10 @@ import com.google.cloud.datastore.Datastore; import com.google.cloud.datastore.DatastoreOptions; import com.google.cloud.datastore.Entity; +import com.google.cloud.datastore.FindNearest; import com.google.cloud.datastore.Query; import com.google.cloud.datastore.QueryResults; import com.google.cloud.datastore.VectorValue; -import com.google.cloud.datastore.FindNearest; public class VectorSearchDistanceResultProperty { public static void invoke() throws Exception { @@ -33,14 +33,16 @@ public static void invoke() throws Exception { // Create vector search query with distance result property Query vectorSearchQuery = - Query.newEntityQueryBuilder() - .setKind("CoffeeBean") - .setFindNearest(new FindNearest( - "embedding_field", - VectorValue.newBuilder(1, 9, 11.1).build(), - FindNearest.DistanceMeasure.DOT_PRODUCT, - 3, "vector_distance")) - .build(); + Query.newEntityQueryBuilder() + .setKind("CoffeeBean") + .setFindNearest( + new FindNearest( + "embedding_field", + VectorValue.newBuilder(1, 9, 11.1).build(), + FindNearest.DistanceMeasure.DOT_PRODUCT, + 3, + "vector_distance")) + .build(); // Execute vector search query QueryResults results = datastore.run(vectorSearchQuery); @@ -51,7 +53,9 @@ public static void invoke() throws Exception { while (results.hasNext()) { Entity entity = results.next(); - System.out.printf("Entity: %s, Distance: %s%n", entity.getKey().getName(), entity.getDouble("vector_distance")); + System.out.printf( + "Entity: %s, Distance: %s%n", + entity.getKey().getName(), entity.getDouble("vector_distance")); } } } diff --git a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultPropertyProjection.java b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultPropertyProjection.java index 72960ac84..c9a9ca4a6 100644 --- a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultPropertyProjection.java +++ b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultPropertyProjection.java @@ -20,11 +20,11 @@ import com.google.cloud.datastore.Datastore; import com.google.cloud.datastore.DatastoreOptions; +import com.google.cloud.datastore.FindNearest; +import com.google.cloud.datastore.ProjectionEntity; import com.google.cloud.datastore.Query; import com.google.cloud.datastore.QueryResults; -import com.google.cloud.datastore.ProjectionEntity; import com.google.cloud.datastore.VectorValue; -import com.google.cloud.datastore.FindNearest; public class VectorSearchDistanceResultPropertyProjection { public static void invoke() throws Exception { @@ -33,15 +33,17 @@ public static void invoke() throws Exception { // Create vector search query with projection Query vectorSearchQuery = - Query.newProjectionEntityQueryBuilder() - .setKind("CoffeeBean") - .setFindNearest(new FindNearest( - "embedding_field", - VectorValue.newBuilder(1, 9, 11.1).build(), - FindNearest.DistanceMeasure.DOT_PRODUCT, - 3, "vector_distance")) - .setProjection("vector_distance") - .build(); + Query.newProjectionEntityQueryBuilder() + .setKind("CoffeeBean") + .setFindNearest( + new FindNearest( + "embedding_field", + VectorValue.newBuilder(1, 9, 11.1).build(), + FindNearest.DistanceMeasure.DOT_PRODUCT, + 3, + "vector_distance")) + .setProjection("vector_distance") + .build(); // Execute vector search query QueryResults results = datastore.run(vectorSearchQuery); @@ -52,7 +54,9 @@ public static void invoke() throws Exception { while (results.hasNext()) { ProjectionEntity entity = results.next(); - System.out.printf("Entity: %s, Distance: %s%n", entity.getKey().getName(), entity.getDouble("vector_distance")); + System.out.printf( + "Entity: %s, Distance: %s%n", + entity.getKey().getName(), entity.getDouble("vector_distance")); } } } diff --git a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceThreshold.java b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceThreshold.java index 201e9d5ca..b9e4c658d 100644 --- a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceThreshold.java +++ b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceThreshold.java @@ -21,10 +21,10 @@ import com.google.cloud.datastore.Datastore; import com.google.cloud.datastore.DatastoreOptions; import com.google.cloud.datastore.Entity; +import com.google.cloud.datastore.FindNearest; import com.google.cloud.datastore.Query; import com.google.cloud.datastore.QueryResults; import com.google.cloud.datastore.VectorValue; -import com.google.cloud.datastore.FindNearest; public class VectorSearchDistanceThreshold { public static void invoke() throws Exception { @@ -33,14 +33,17 @@ public static void invoke() throws Exception { // Create vector search query with distance threshold Query vectorSearchQuery = - Query.newEntityQueryBuilder() - .setKind("CoffeeBean") - .setFindNearest(new FindNearest( - "embedding_field", - VectorValue.newBuilder(1, 9, 11.1).build(), - FindNearest.DistanceMeasure.EUCLIDEAN, - 3, "vector_distance", 2.0)) - .build(); + Query.newEntityQueryBuilder() + .setKind("CoffeeBean") + .setFindNearest( + new FindNearest( + "embedding_field", + VectorValue.newBuilder(1, 9, 11.1).build(), + FindNearest.DistanceMeasure.EUCLIDEAN, + 3, + "vector_distance", + 2.0)) + .build(); // Execute vector search query QueryResults results = datastore.run(vectorSearchQuery); @@ -51,7 +54,9 @@ public static void invoke() throws Exception { while (results.hasNext()) { Entity entity = results.next(); - System.out.printf("Entity: %s, Distance: %s%n", entity.getKey().getName(), entity.getDouble("vector_distance")); + System.out.printf( + "Entity: %s, Distance: %s%n", + entity.getKey().getName(), entity.getDouble("vector_distance")); } } } diff --git a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchLargeResponse.java b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchLargeResponse.java index 070eb075c..39a61d861 100644 --- a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchLargeResponse.java +++ b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchLargeResponse.java @@ -18,16 +18,16 @@ // [START datastore_vector_search_large_response] -import com.google.common.collect.Iterators; -import java.util.Iterator; import com.google.cloud.datastore.Datastore; import com.google.cloud.datastore.DatastoreOptions; import com.google.cloud.datastore.Entity; +import com.google.cloud.datastore.FindNearest; +import com.google.cloud.datastore.Key; import com.google.cloud.datastore.Query; import com.google.cloud.datastore.QueryResults; -import com.google.cloud.datastore.Key; import com.google.cloud.datastore.VectorValue; -import com.google.cloud.datastore.FindNearest; +import com.google.common.collect.Iterators; +import java.util.Iterator; public class VectorSearchLargeResponse { public static void invoke() throws Exception { @@ -36,14 +36,17 @@ public static void invoke() throws Exception { // Create a keys-only vector search query Query vectorSearchKeyOnlyQuery = - Query.newKeyQueryBuilder() - .setKind("CoffeeBean") - .setFindNearest(new FindNearest( - "embedding_field", - VectorValue.newBuilder(1, 9, 11.1).build(), - FindNearest.DistanceMeasure.EUCLIDEAN, - 3, "vector_distance", 2.0)) - .build(); + Query.newKeyQueryBuilder() + .setKind("CoffeeBean") + .setFindNearest( + new FindNearest( + "embedding_field", + VectorValue.newBuilder(1, 9, 11.1).build(), + FindNearest.DistanceMeasure.EUCLIDEAN, + 3, + "vector_distance", + 2.0)) + .build(); QueryResults keyResults = datastore.run(vectorSearchKeyOnlyQuery); Key[] keys = Iterators.toArray(keyResults, Key.class); @@ -58,7 +61,9 @@ public static void invoke() throws Exception { // Combine and print results while (keyResults.hasNext()) { Key keyResult = keyResults.next(); - System.out.printf("Entity: %s, Distance: %s%n", keyResult.getName(), entities.next().getDouble("vector_distance")); + System.out.printf( + "Entity: %s, Distance: %s%n", + keyResult.getName(), entities.next().getDouble("vector_distance")); } } } diff --git a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchPrefilter.java b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchPrefilter.java index e24854aec..5338b4049 100644 --- a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchPrefilter.java +++ b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchPrefilter.java @@ -21,11 +21,11 @@ import com.google.cloud.datastore.Datastore; import com.google.cloud.datastore.DatastoreOptions; import com.google.cloud.datastore.Entity; +import com.google.cloud.datastore.FindNearest; import com.google.cloud.datastore.Query; -import com.google.cloud.datastore.StructuredQuery.PropertyFilter; import com.google.cloud.datastore.QueryResults; +import com.google.cloud.datastore.StructuredQuery.PropertyFilter; import com.google.cloud.datastore.VectorValue; -import com.google.cloud.datastore.FindNearest; public class VectorSearchPrefilter { public static void invoke() throws Exception { @@ -34,15 +34,16 @@ public static void invoke() throws Exception { // Create vector search query with property filter Query vectorSearchQuery = - Query.newEntityQueryBuilder() - .setKind("CoffeeBean") - .setFilter(PropertyFilter.eq("roast", "dark")) - .setFindNearest(new FindNearest( - "embedding_field", - VectorValue.newBuilder(1, 9, 11.1).build(), - FindNearest.DistanceMeasure.DOT_PRODUCT, - 3)) - .build(); + Query.newEntityQueryBuilder() + .setKind("CoffeeBean") + .setFilter(PropertyFilter.eq("roast", "dark")) + .setFindNearest( + new FindNearest( + "embedding_field", + VectorValue.newBuilder(1, 9, 11.1).build(), + FindNearest.DistanceMeasure.DOT_PRODUCT, + 3)) + .build(); // Execute vector search query QueryResults results = datastore.run(vectorSearchQuery); From 6679cd2b8d4deb2c6ba073a553cea994f4b7783c Mon Sep 17 00:00:00 2001 From: Cindy Peng <148148319+cindy-peng@users.noreply.github.com> Date: Fri, 13 Dec 2024 17:04:19 -0800 Subject: [PATCH 30/33] Add tests to sample code --- .../datastore/vectorsearch/StoreVectors.java | 2 +- .../vectorsearch/VectorSearchBasic.java | 5 +- ...earchDistanceResultPropertyProjection.java | 4 +- .../VectorSearchLargeResponse.java | 32 ++-- .../vectorsearch/VectorSearchPrefilter.java | 6 +- .../vectorsearch/VectorSearchSampleIT.java | 148 ++++++++++++++++++ 6 files changed, 174 insertions(+), 23 deletions(-) create mode 100644 samples/snippets/src/test/java/com/example/datastore/vectorsearch/VectorSearchSampleIT.java diff --git a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/StoreVectors.java b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/StoreVectors.java index 510756577..5acabc065 100644 --- a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/StoreVectors.java +++ b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/StoreVectors.java @@ -47,7 +47,7 @@ public static void invoke() throws Exception { // Retrieve entity Entity retrieved = datastore.get(key); - System.out.printf("Retrieved %s: %s%n", key.getName(), retrieved.getString("description")); + System.out.printf("Retrieved %s with embedding_field: %s%n", key.getName(), retrieved.getVector("embedding_field")); } } // [END datastore_store_vectors] diff --git a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchBasic.java b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchBasic.java index 1f437101d..fac1c7a2c 100644 --- a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchBasic.java +++ b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchBasic.java @@ -30,7 +30,6 @@ public class VectorSearchBasic { public static void invoke() throws Exception { // Instantiates a client Datastore datastore = DatastoreOptions.getDefaultInstance().getService(); - // Create vector search query Query vectorSearchQuery = Query.newEntityQueryBuilder() @@ -38,8 +37,8 @@ public static void invoke() throws Exception { .setFindNearest(new FindNearest( "embedding_field", VectorValue.newBuilder(1, 9, 11.1).build(), - FindNearest.DistanceMeasure.DOT_PRODUCT, - 3)) + FindNearest.DistanceMeasure.EUCLIDEAN, + 1)) .build(); // Execute vector search query diff --git a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultPropertyProjection.java b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultPropertyProjection.java index 72960ac84..96d6e52f2 100644 --- a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultPropertyProjection.java +++ b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultPropertyProjection.java @@ -38,9 +38,9 @@ public static void invoke() throws Exception { .setFindNearest(new FindNearest( "embedding_field", VectorValue.newBuilder(1, 9, 11.1).build(), - FindNearest.DistanceMeasure.DOT_PRODUCT, + FindNearest.DistanceMeasure.EUCLIDEAN, 3, "vector_distance")) - .setProjection("vector_distance") + .setProjection("roast") .build(); // Execute vector search query diff --git a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchLargeResponse.java b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchLargeResponse.java index 070eb075c..f0668b3da 100644 --- a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchLargeResponse.java +++ b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchLargeResponse.java @@ -18,16 +18,19 @@ // [START datastore_vector_search_large_response] +import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterators; import java.util.Iterator; import com.google.cloud.datastore.Datastore; import com.google.cloud.datastore.DatastoreOptions; import com.google.cloud.datastore.Entity; +import com.google.cloud.datastore.FindNearest; +import com.google.cloud.datastore.Key; +import com.google.cloud.datastore.ProjectionEntity; import com.google.cloud.datastore.Query; import com.google.cloud.datastore.QueryResults; -import com.google.cloud.datastore.Key; +import com.google.cloud.datastore.StructuredQuery; import com.google.cloud.datastore.VectorValue; -import com.google.cloud.datastore.FindNearest; public class VectorSearchLargeResponse { public static void invoke() throws Exception { @@ -35,9 +38,10 @@ public static void invoke() throws Exception { Datastore datastore = DatastoreOptions.getDefaultInstance().getService(); // Create a keys-only vector search query - Query vectorSearchKeyOnlyQuery = - Query.newKeyQueryBuilder() + StructuredQuery keyOnlyVectorQuery = + Query.newProjectionEntityQueryBuilder() .setKind("CoffeeBean") + .setProjection("__key__") .setFindNearest(new FindNearest( "embedding_field", VectorValue.newBuilder(1, 9, 11.1).build(), @@ -45,20 +49,20 @@ public static void invoke() throws Exception { 3, "vector_distance", 2.0)) .build(); - QueryResults keyResults = datastore.run(vectorSearchKeyOnlyQuery); - Key[] keys = Iterators.toArray(keyResults, Key.class); + QueryResults keyOnlyResults = datastore.run(keyOnlyVectorQuery); + ProjectionEntity[] keyEntities = Iterators.toArray(keyOnlyResults, ProjectionEntity.class); + Key[] keys = ImmutableList.copyOf(keyEntities).stream().map(e -> e.getKey()).toArray(Key[]::new); + System.out.printf("Key query result size: %s%n", keys.length); - // Next, perform a second query for the remaining data + // Lookup the full entities using the result of the keys only query. Iterator entities = datastore.get(keys); - - if (!keyResults.hasNext()) { - throw new Exception("query yielded no results"); - } + Entity[] entitiesArray = Iterators.toArray(entities, Entity.class); + System.out.printf("Entity query result size: %s%n", entitiesArray.length); // Combine and print results - while (keyResults.hasNext()) { - Key keyResult = keyResults.next(); - System.out.printf("Entity: %s, Distance: %s%n", keyResult.getName(), entities.next().getDouble("vector_distance")); + for (int i = 0; i < keyEntities.length; i++) + { + System.out.printf("Entity: %s, Distance: %s, Roast: %s%n", keyEntities[i].getKey().getName(), keyEntities[i].getDouble("vector_distance"), entitiesArray[i].getString("roast")); } } } diff --git a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchPrefilter.java b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchPrefilter.java index e24854aec..5979c2be7 100644 --- a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchPrefilter.java +++ b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchPrefilter.java @@ -40,8 +40,8 @@ public static void invoke() throws Exception { .setFindNearest(new FindNearest( "embedding_field", VectorValue.newBuilder(1, 9, 11.1).build(), - FindNearest.DistanceMeasure.DOT_PRODUCT, - 3)) + FindNearest.DistanceMeasure.EUCLIDEAN, + 3, "vector_distance", 3.0)) .build(); // Execute vector search query @@ -53,7 +53,7 @@ public static void invoke() throws Exception { while (results.hasNext()) { Entity entity = results.next(); - System.out.printf("Entity: %s%n", entity.getKey().getName()); + System.out.printf("Entity: %s, Distance: %s%n", entity.getKey().getName(), entity.getDouble("vector_distance")); } } } diff --git a/samples/snippets/src/test/java/com/example/datastore/vectorsearch/VectorSearchSampleIT.java b/samples/snippets/src/test/java/com/example/datastore/vectorsearch/VectorSearchSampleIT.java new file mode 100644 index 000000000..276a714a3 --- /dev/null +++ b/samples/snippets/src/test/java/com/example/datastore/vectorsearch/VectorSearchSampleIT.java @@ -0,0 +1,148 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.example.datastore.vectorsearch; + +import com.google.cloud.datastore.*; +import com.rule.SystemsOutRule; +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import com.google.cloud.datastore.Datastore; +import com.google.cloud.datastore.DatastoreOptions; + +@RunWith(JUnit4.class) +@SuppressWarnings("checkstyle:abbreviationaswordinname") +public class VectorSearchSampleIT { + private final Datastore datastore = DatastoreOptions.getDefaultInstance().getService(); + + private Key coffeeBeanKey1; + private Key coffeeBeanKey2; + private Key coffeeBeanKey3; + + @Rule + public final SystemsOutRule systemsOutRule = new SystemsOutRule(); + + @Before + public void setUp() { + //DatastoreOptions.getDefaultHttpTransportOptions() + coffeeBeanKey1 = datastore.newKeyFactory().setKind("CoffeeBean").newKey("Kahawa"); + // Prepares the entity with a vector embedding + Entity entity1 = + Entity.newBuilder(coffeeBeanKey1) + .set("name", "Arabica") + .set("description", "Information about the Arabica coffee beans.") + .set("roast", "dark") + .set("embedding_field", VectorValue.newBuilder(1.0, 7.0, 11.1).build()) + .build(); + + coffeeBeanKey2 = datastore.newKeyFactory().setKind("CoffeeBean").newKey("Robusta"); + Entity entity2 = + Entity.newBuilder(coffeeBeanKey2) + .set("name", "Robusta") + .set("description", "Information about the Robusta coffee beans.") + .set("roast", "light") + .set("embedding_field", VectorValue.newBuilder(1.0, 9.0, 11.1).build()) + .build(); + + coffeeBeanKey3 = datastore.newKeyFactory().setKind("CoffeeBean").newKey("Excelsa"); + Entity entity3 = + Entity.newBuilder(coffeeBeanKey3) + .set("name", "Excelsa") + .set("description", "Information about the Excelsa coffee beans.") + .set("roast", "dark") + .set("embedding_field", VectorValue.newBuilder(4.0, 9.0, 11.1).build()) + .build(); + + datastore.put(entity1); + datastore.put(entity2); + datastore.put(entity3); + } + + @After + public void tearDown() { + datastore.delete(coffeeBeanKey1); + datastore.delete(coffeeBeanKey2); + datastore.delete(coffeeBeanKey3); + } + + @Test + public void testStoreVectors() throws Exception { + // Act + StoreVectors.invoke(); + // Assert + systemsOutRule.assertContains("Retrieved Kahawa with embedding_field"); + } + + @Test + public void testVectorSearchBasic() throws Exception { + // Act + VectorSearchBasic.invoke(); + // Assert + systemsOutRule.assertContains("Entity: Robusta"); + } + + @Test + public void testVectorSearchDistanceResultProperty() throws Exception { + // Act + VectorSearchDistanceResultProperty.invoke(); + // Assert + systemsOutRule.assertContains("Entity: Excelsa, Distance: 208"); + systemsOutRule.assertContains("Entity: Robusta, Distance: 205"); + systemsOutRule.assertContains("Entity: Kahawa, Distance: 187"); + } + + @Test + public void testVectorSearchDistanceResultPropertyProjection() throws Exception { + // Act + VectorSearchDistanceResultPropertyProjection.invoke(); + // Assert + systemsOutRule.assertContains("Entity: Robusta, Distance: 0.0"); + systemsOutRule.assertContains("Entity: Kahawa, Distance: 2.0"); + systemsOutRule.assertContains("Entity: Excelsa, Distance: 3.0"); + } + + @Test + public void testVectorSearchDistanceThreshold() throws Exception { + // Act + VectorSearchDistanceThreshold.invoke(); + // Assert + systemsOutRule.assertContains("Entity: Robusta, Distance: 0.0"); + systemsOutRule.assertContains("Entity: Kahawa, Distance: 2.0"); + } + + @Test + public void testVectorSearchLargeResponse() throws Exception { + // Act + VectorSearchLargeResponse.invoke(); + // Assert + systemsOutRule.assertContains("Key query result size: 2"); + systemsOutRule.assertContains("Entity query result size: 2"); + systemsOutRule.assertContains("Entity: Robusta, Distance: 0.0, Roast: dark"); + systemsOutRule.assertContains("Entity: Kahawa, Distance: 2.0, Roast: light"); + } + + @Test + public void testVectorSearchPrefilter() throws Exception { + // Act + VectorSearchPrefilter.invoke(); + // Assert + systemsOutRule.assertContains("Entity: Kahawa, Distance: 2.0"); + systemsOutRule.assertContains("Entity: Excelsa, Distance: 3.0"); + } +} From cf1bb19acd41b25dd72759d31e2e58dc9aa4340f Mon Sep 17 00:00:00 2001 From: Cindy Peng <148148319+cindy-peng@users.noreply.github.com> Date: Fri, 13 Dec 2024 17:13:27 -0800 Subject: [PATCH 31/33] Fix added interface method warning --- google-cloud-datastore/clirr-ignored-differences.xml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/google-cloud-datastore/clirr-ignored-differences.xml b/google-cloud-datastore/clirr-ignored-differences.xml index 6275f05c6..e3cc028d8 100644 --- a/google-cloud-datastore/clirr-ignored-differences.xml +++ b/google-cloud-datastore/clirr-ignored-differences.xml @@ -14,7 +14,7 @@ com/google/cloud/datastore/DatastoreReader - com.google.cloud.datastore.AggregationResults runAggregation(com.google.cloud.datastore.AggregationQuery, com.google.cloud.datastore.models.ExplainOptions) + com.google.cloud.datastore.AggregationResults runAggregation(com.google.cloud.datastore.AggregationQuery, com.google.cloud.datastore.models.ExplainOptions) 7012 @@ -27,8 +27,13 @@ com.google.cloud.datastore.QueryResults run(com.google.cloud.datastore.Query, com.google.cloud.datastore.models.ExplainOptions) 7012 + + com/google/cloud/datastore/StructuredQuery* + com.google.cloud.datastore.StructuredQuery* setFindNearest(com.google.cloud.datastore.FindNearest) + 7012 + - + com/google/cloud/datastore/ReadOption$QueryConfig com.google.cloud.datastore.ReadOption$QueryConfig create(com.google.cloud.datastore.Query, java.util.List) From 7c46b40903e7f4f7a3961f77b8d723172eae01b6 Mon Sep 17 00:00:00 2001 From: Cindy Peng <148148319+cindy-peng@users.noreply.github.com> Date: Fri, 13 Dec 2024 17:45:48 -0800 Subject: [PATCH 32/33] Fix mvn lint formatting --- .../datastore/vectorsearch/StoreVectors.java | 16 ++++--- ...earchDistanceResultPropertyProjection.java | 4 +- .../VectorSearchLargeResponse.java | 17 ++++--- .../vectorsearch/VectorSearchPrefilter.java | 4 +- .../vectorsearch/VectorSearchSampleIT.java | 48 +++++++++---------- 5 files changed, 49 insertions(+), 40 deletions(-) diff --git a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/StoreVectors.java b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/StoreVectors.java index d58df807e..60d8624d1 100644 --- a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/StoreVectors.java +++ b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/StoreVectors.java @@ -34,12 +34,12 @@ public static void invoke() throws Exception { // Prepares the entity with a vector embedding Entity entity = - Entity.newBuilder(key) - .set("name", "Kahawa") - .set("description", "Information about the Kahawa coffee beans.") - .set("roast", "dark") - .set("embedding_field", VectorValue.newBuilder(1.0, 7.0, 11.1).build()) - .build(); + Entity.newBuilder(key) + .set("name", "Kahawa") + .set("description", "Information about the Kahawa coffee beans.") + .set("roast", "dark") + .set("embedding_field", VectorValue.newBuilder(1.0, 7.0, 11.1).build()) + .build(); // Saves the entity datastore.put(entity); @@ -47,7 +47,9 @@ public static void invoke() throws Exception { // Retrieve entity Entity retrieved = datastore.get(key); - System.out.printf("Retrieved %s with embedding_field: %s%n", key.getName(), retrieved.getVector("embedding_field")); + System.out.printf("Retrieved %s with embedding_field: %s%n", + key.getName(), + retrieved.getVector("embedding_field")); } } // [END datastore_store_vectors] diff --git a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultPropertyProjection.java b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultPropertyProjection.java index 9ceee2c7e..6c391cc5b 100644 --- a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultPropertyProjection.java +++ b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultPropertyProjection.java @@ -52,7 +52,9 @@ public static void invoke() throws Exception { while (results.hasNext()) { ProjectionEntity entity = results.next(); - System.out.printf("Entity: %s, Distance: %s%n", entity.getKey().getName(), entity.getDouble("vector_distance")); + System.out.printf("Entity: %s, Distance: %s%n", + entity.getKey().getName(), + entity.getDouble("vector_distance")); } } } diff --git a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchLargeResponse.java b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchLargeResponse.java index f0668b3da..c92c20bbd 100644 --- a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchLargeResponse.java +++ b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchLargeResponse.java @@ -18,9 +18,6 @@ // [START datastore_vector_search_large_response] -import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterators; -import java.util.Iterator; import com.google.cloud.datastore.Datastore; import com.google.cloud.datastore.DatastoreOptions; import com.google.cloud.datastore.Entity; @@ -31,6 +28,9 @@ import com.google.cloud.datastore.QueryResults; import com.google.cloud.datastore.StructuredQuery; import com.google.cloud.datastore.VectorValue; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Iterators; +import java.util.Iterator; public class VectorSearchLargeResponse { public static void invoke() throws Exception { @@ -51,7 +51,8 @@ public static void invoke() throws Exception { QueryResults keyOnlyResults = datastore.run(keyOnlyVectorQuery); ProjectionEntity[] keyEntities = Iterators.toArray(keyOnlyResults, ProjectionEntity.class); - Key[] keys = ImmutableList.copyOf(keyEntities).stream().map(e -> e.getKey()).toArray(Key[]::new); + Key[] keys = + ImmutableList.copyOf(keyEntities).stream().map(e -> e.getKey()).toArray(Key[]::new); System.out.printf("Key query result size: %s%n", keys.length); // Lookup the full entities using the result of the keys only query. @@ -60,9 +61,11 @@ public static void invoke() throws Exception { System.out.printf("Entity query result size: %s%n", entitiesArray.length); // Combine and print results - for (int i = 0; i < keyEntities.length; i++) - { - System.out.printf("Entity: %s, Distance: %s, Roast: %s%n", keyEntities[i].getKey().getName(), keyEntities[i].getDouble("vector_distance"), entitiesArray[i].getString("roast")); + for (int i = 0; i < keyEntities.length; i++) { + System.out.printf("Entity: %s, Distance: %s, Roast: %s%n", + keyEntities[i].getKey().getName(), + keyEntities[i].getDouble("vector_distance"), + entitiesArray[i].getString("roast")); } } } diff --git a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchPrefilter.java b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchPrefilter.java index 6e9f36864..5872f2b63 100644 --- a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchPrefilter.java +++ b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchPrefilter.java @@ -53,7 +53,9 @@ public static void invoke() throws Exception { while (results.hasNext()) { Entity entity = results.next(); - System.out.printf("Entity: %s, Distance: %s%n", entity.getKey().getName(), entity.getDouble("vector_distance")); + System.out.printf("Entity: %s, Distance: %s%n", + entity.getKey().getName(), + entity.getDouble("vector_distance")); } } } diff --git a/samples/snippets/src/test/java/com/example/datastore/vectorsearch/VectorSearchSampleIT.java b/samples/snippets/src/test/java/com/example/datastore/vectorsearch/VectorSearchSampleIT.java index 276a714a3..a51243045 100644 --- a/samples/snippets/src/test/java/com/example/datastore/vectorsearch/VectorSearchSampleIT.java +++ b/samples/snippets/src/test/java/com/example/datastore/vectorsearch/VectorSearchSampleIT.java @@ -13,9 +13,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package com.example.datastore.vectorsearch; -import com.google.cloud.datastore.*; +import com.google.cloud.datastore.Datastore; +import com.google.cloud.datastore.DatastoreOptions; +import com.google.cloud.datastore.Entity; +import com.google.cloud.datastore.Key; +import com.google.cloud.datastore.VectorValue; import com.rule.SystemsOutRule; import org.junit.After; import org.junit.Before; @@ -23,8 +28,6 @@ import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; -import com.google.cloud.datastore.Datastore; -import com.google.cloud.datastore.DatastoreOptions; @RunWith(JUnit4.class) @SuppressWarnings("checkstyle:abbreviationaswordinname") @@ -43,31 +46,28 @@ public void setUp() { //DatastoreOptions.getDefaultHttpTransportOptions() coffeeBeanKey1 = datastore.newKeyFactory().setKind("CoffeeBean").newKey("Kahawa"); // Prepares the entity with a vector embedding - Entity entity1 = - Entity.newBuilder(coffeeBeanKey1) - .set("name", "Arabica") - .set("description", "Information about the Arabica coffee beans.") - .set("roast", "dark") - .set("embedding_field", VectorValue.newBuilder(1.0, 7.0, 11.1).build()) - .build(); + Entity entity1 = Entity.newBuilder(coffeeBeanKey1) + .set("name", "Arabica") + .set("description", "Information about the Arabica coffee beans.") + .set("roast", "dark") + .set("embedding_field", VectorValue.newBuilder(1.0, 7.0, 11.1).build()) + .build(); coffeeBeanKey2 = datastore.newKeyFactory().setKind("CoffeeBean").newKey("Robusta"); - Entity entity2 = - Entity.newBuilder(coffeeBeanKey2) - .set("name", "Robusta") - .set("description", "Information about the Robusta coffee beans.") - .set("roast", "light") - .set("embedding_field", VectorValue.newBuilder(1.0, 9.0, 11.1).build()) - .build(); + Entity entity2 = Entity.newBuilder(coffeeBeanKey2) + .set("name", "Robusta") + .set("description", "Information about the Robusta coffee beans.") + .set("roast", "light") + .set("embedding_field", VectorValue.newBuilder(1.0, 9.0, 11.1).build()) + .build(); coffeeBeanKey3 = datastore.newKeyFactory().setKind("CoffeeBean").newKey("Excelsa"); - Entity entity3 = - Entity.newBuilder(coffeeBeanKey3) - .set("name", "Excelsa") - .set("description", "Information about the Excelsa coffee beans.") - .set("roast", "dark") - .set("embedding_field", VectorValue.newBuilder(4.0, 9.0, 11.1).build()) - .build(); + Entity entity3 = Entity.newBuilder(coffeeBeanKey3) + .set("name", "Excelsa") + .set("description", "Information about the Excelsa coffee beans.") + .set("roast", "dark") + .set("embedding_field", VectorValue.newBuilder(4.0, 9.0, 11.1).build()) + .build(); datastore.put(entity1); datastore.put(entity2); From 1c73ec9531416f30ea07af87078633d211d500dd Mon Sep 17 00:00:00 2001 From: cloud-java-bot Date: Sat, 14 Dec 2024 01:47:46 +0000 Subject: [PATCH 33/33] chore: generate libraries at Sat Dec 14 01:46:19 UTC 2024 --- README.md | 7 ++++ .../datastore/vectorsearch/StoreVectors.java | 18 +++++------ .../vectorsearch/VectorSearchBasic.java | 17 +++++----- ...earchDistanceResultPropertyProjection.java | 26 ++++++++------- .../VectorSearchLargeResponse.java | 32 +++++++++++-------- .../vectorsearch/VectorSearchPrefilter.java | 27 +++++++++------- .../vectorsearch/VectorSearchSampleIT.java | 14 ++++---- 7 files changed, 80 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index 52bd677cf..4efd4bec7 100644 --- a/README.md +++ b/README.md @@ -375,6 +375,13 @@ Samples are in the [`samples/`](https://github.com/googleapis/java-datastore/tre | Query Profile Explain Aggregation | [source code](https://github.com/googleapis/java-datastore/blob/main/samples/snippets/src/main/java/com/example/datastore/queryprofile/QueryProfileExplainAggregation.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-datastore&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/datastore/queryprofile/QueryProfileExplainAggregation.java) | | Query Profile Explain Analyze | [source code](https://github.com/googleapis/java-datastore/blob/main/samples/snippets/src/main/java/com/example/datastore/queryprofile/QueryProfileExplainAnalyze.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-datastore&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/datastore/queryprofile/QueryProfileExplainAnalyze.java) | | Query Profile Explain Analyze Aggregation | [source code](https://github.com/googleapis/java-datastore/blob/main/samples/snippets/src/main/java/com/example/datastore/queryprofile/QueryProfileExplainAnalyzeAggregation.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-datastore&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/datastore/queryprofile/QueryProfileExplainAnalyzeAggregation.java) | +| Store Vectors | [source code](https://github.com/googleapis/java-datastore/blob/main/samples/snippets/src/main/java/com/example/datastore/vectorsearch/StoreVectors.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-datastore&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/datastore/vectorsearch/StoreVectors.java) | +| Vector Search Basic | [source code](https://github.com/googleapis/java-datastore/blob/main/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchBasic.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-datastore&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchBasic.java) | +| Vector Search Distance Result Property | [source code](https://github.com/googleapis/java-datastore/blob/main/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultProperty.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-datastore&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultProperty.java) | +| Vector Search Distance Result Property Projection | [source code](https://github.com/googleapis/java-datastore/blob/main/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultPropertyProjection.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-datastore&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultPropertyProjection.java) | +| Vector Search Distance Threshold | [source code](https://github.com/googleapis/java-datastore/blob/main/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceThreshold.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-datastore&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceThreshold.java) | +| Vector Search Large Response | [source code](https://github.com/googleapis/java-datastore/blob/main/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchLargeResponse.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-datastore&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchLargeResponse.java) | +| Vector Search Prefilter | [source code](https://github.com/googleapis/java-datastore/blob/main/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchPrefilter.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-datastore&page=editor&open_in_editor=samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchPrefilter.java) | | Task List | [source code](https://github.com/googleapis/java-datastore/blob/main/samples/snippets/src/main/java/com/google/datastore/snippets/TaskList.java) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/java-datastore&page=editor&open_in_editor=samples/snippets/src/main/java/com/google/datastore/snippets/TaskList.java) | diff --git a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/StoreVectors.java b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/StoreVectors.java index 60d8624d1..a1db1ca80 100644 --- a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/StoreVectors.java +++ b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/StoreVectors.java @@ -34,12 +34,12 @@ public static void invoke() throws Exception { // Prepares the entity with a vector embedding Entity entity = - Entity.newBuilder(key) - .set("name", "Kahawa") - .set("description", "Information about the Kahawa coffee beans.") - .set("roast", "dark") - .set("embedding_field", VectorValue.newBuilder(1.0, 7.0, 11.1).build()) - .build(); + Entity.newBuilder(key) + .set("name", "Kahawa") + .set("description", "Information about the Kahawa coffee beans.") + .set("roast", "dark") + .set("embedding_field", VectorValue.newBuilder(1.0, 7.0, 11.1).build()) + .build(); // Saves the entity datastore.put(entity); @@ -47,9 +47,9 @@ public static void invoke() throws Exception { // Retrieve entity Entity retrieved = datastore.get(key); - System.out.printf("Retrieved %s with embedding_field: %s%n", - key.getName(), - retrieved.getVector("embedding_field")); + System.out.printf( + "Retrieved %s with embedding_field: %s%n", + key.getName(), retrieved.getVector("embedding_field")); } } // [END datastore_store_vectors] diff --git a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchBasic.java b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchBasic.java index 06dd71212..a0ca8b953 100644 --- a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchBasic.java +++ b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchBasic.java @@ -33,14 +33,15 @@ public static void invoke() throws Exception { // Create vector search query Query vectorSearchQuery = - Query.newEntityQueryBuilder() - .setKind("CoffeeBean") - .setFindNearest(new FindNearest( - "embedding_field", - VectorValue.newBuilder(1, 9, 11.1).build(), - FindNearest.DistanceMeasure.EUCLIDEAN, - 1)) - .build(); + Query.newEntityQueryBuilder() + .setKind("CoffeeBean") + .setFindNearest( + new FindNearest( + "embedding_field", + VectorValue.newBuilder(1, 9, 11.1).build(), + FindNearest.DistanceMeasure.EUCLIDEAN, + 1)) + .build(); // Execute vector search query QueryResults results = datastore.run(vectorSearchQuery); diff --git a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultPropertyProjection.java b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultPropertyProjection.java index 6c391cc5b..96ce2ccbb 100644 --- a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultPropertyProjection.java +++ b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchDistanceResultPropertyProjection.java @@ -33,15 +33,17 @@ public static void invoke() throws Exception { // Create vector search query with projection Query vectorSearchQuery = - Query.newProjectionEntityQueryBuilder() - .setKind("CoffeeBean") - .setFindNearest(new FindNearest( - "embedding_field", - VectorValue.newBuilder(1, 9, 11.1).build(), - FindNearest.DistanceMeasure.EUCLIDEAN, - 3, "vector_distance")) - .setProjection("roast") - .build(); + Query.newProjectionEntityQueryBuilder() + .setKind("CoffeeBean") + .setFindNearest( + new FindNearest( + "embedding_field", + VectorValue.newBuilder(1, 9, 11.1).build(), + FindNearest.DistanceMeasure.EUCLIDEAN, + 3, + "vector_distance")) + .setProjection("roast") + .build(); // Execute vector search query QueryResults results = datastore.run(vectorSearchQuery); @@ -52,9 +54,9 @@ public static void invoke() throws Exception { while (results.hasNext()) { ProjectionEntity entity = results.next(); - System.out.printf("Entity: %s, Distance: %s%n", - entity.getKey().getName(), - entity.getDouble("vector_distance")); + System.out.printf( + "Entity: %s, Distance: %s%n", + entity.getKey().getName(), entity.getDouble("vector_distance")); } } } diff --git a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchLargeResponse.java b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchLargeResponse.java index c92c20bbd..0744ca6cb 100644 --- a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchLargeResponse.java +++ b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchLargeResponse.java @@ -39,20 +39,23 @@ public static void invoke() throws Exception { // Create a keys-only vector search query StructuredQuery keyOnlyVectorQuery = - Query.newProjectionEntityQueryBuilder() - .setKind("CoffeeBean") - .setProjection("__key__") - .setFindNearest(new FindNearest( - "embedding_field", - VectorValue.newBuilder(1, 9, 11.1).build(), - FindNearest.DistanceMeasure.EUCLIDEAN, - 3, "vector_distance", 2.0)) - .build(); + Query.newProjectionEntityQueryBuilder() + .setKind("CoffeeBean") + .setProjection("__key__") + .setFindNearest( + new FindNearest( + "embedding_field", + VectorValue.newBuilder(1, 9, 11.1).build(), + FindNearest.DistanceMeasure.EUCLIDEAN, + 3, + "vector_distance", + 2.0)) + .build(); QueryResults keyOnlyResults = datastore.run(keyOnlyVectorQuery); ProjectionEntity[] keyEntities = Iterators.toArray(keyOnlyResults, ProjectionEntity.class); Key[] keys = - ImmutableList.copyOf(keyEntities).stream().map(e -> e.getKey()).toArray(Key[]::new); + ImmutableList.copyOf(keyEntities).stream().map(e -> e.getKey()).toArray(Key[]::new); System.out.printf("Key query result size: %s%n", keys.length); // Lookup the full entities using the result of the keys only query. @@ -62,10 +65,11 @@ public static void invoke() throws Exception { // Combine and print results for (int i = 0; i < keyEntities.length; i++) { - System.out.printf("Entity: %s, Distance: %s, Roast: %s%n", - keyEntities[i].getKey().getName(), - keyEntities[i].getDouble("vector_distance"), - entitiesArray[i].getString("roast")); + System.out.printf( + "Entity: %s, Distance: %s, Roast: %s%n", + keyEntities[i].getKey().getName(), + keyEntities[i].getDouble("vector_distance"), + entitiesArray[i].getString("roast")); } } } diff --git a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchPrefilter.java b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchPrefilter.java index 5872f2b63..c00fc5f0f 100644 --- a/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchPrefilter.java +++ b/samples/snippets/src/main/java/com/example/datastore/vectorsearch/VectorSearchPrefilter.java @@ -34,15 +34,18 @@ public static void invoke() throws Exception { // Create vector search query with property filter Query vectorSearchQuery = - Query.newEntityQueryBuilder() - .setKind("CoffeeBean") - .setFilter(PropertyFilter.eq("roast", "dark")) - .setFindNearest(new FindNearest( - "embedding_field", - VectorValue.newBuilder(1, 9, 11.1).build(), - FindNearest.DistanceMeasure.EUCLIDEAN, - 3, "vector_distance", 3.0)) - .build(); + Query.newEntityQueryBuilder() + .setKind("CoffeeBean") + .setFilter(PropertyFilter.eq("roast", "dark")) + .setFindNearest( + new FindNearest( + "embedding_field", + VectorValue.newBuilder(1, 9, 11.1).build(), + FindNearest.DistanceMeasure.EUCLIDEAN, + 3, + "vector_distance", + 3.0)) + .build(); // Execute vector search query QueryResults results = datastore.run(vectorSearchQuery); @@ -53,9 +56,9 @@ public static void invoke() throws Exception { while (results.hasNext()) { Entity entity = results.next(); - System.out.printf("Entity: %s, Distance: %s%n", - entity.getKey().getName(), - entity.getDouble("vector_distance")); + System.out.printf( + "Entity: %s, Distance: %s%n", + entity.getKey().getName(), entity.getDouble("vector_distance")); } } } diff --git a/samples/snippets/src/test/java/com/example/datastore/vectorsearch/VectorSearchSampleIT.java b/samples/snippets/src/test/java/com/example/datastore/vectorsearch/VectorSearchSampleIT.java index a51243045..792ab3c56 100644 --- a/samples/snippets/src/test/java/com/example/datastore/vectorsearch/VectorSearchSampleIT.java +++ b/samples/snippets/src/test/java/com/example/datastore/vectorsearch/VectorSearchSampleIT.java @@ -38,15 +38,15 @@ public class VectorSearchSampleIT { private Key coffeeBeanKey2; private Key coffeeBeanKey3; - @Rule - public final SystemsOutRule systemsOutRule = new SystemsOutRule(); + @Rule public final SystemsOutRule systemsOutRule = new SystemsOutRule(); @Before public void setUp() { - //DatastoreOptions.getDefaultHttpTransportOptions() + // DatastoreOptions.getDefaultHttpTransportOptions() coffeeBeanKey1 = datastore.newKeyFactory().setKind("CoffeeBean").newKey("Kahawa"); // Prepares the entity with a vector embedding - Entity entity1 = Entity.newBuilder(coffeeBeanKey1) + Entity entity1 = + Entity.newBuilder(coffeeBeanKey1) .set("name", "Arabica") .set("description", "Information about the Arabica coffee beans.") .set("roast", "dark") @@ -54,7 +54,8 @@ public void setUp() { .build(); coffeeBeanKey2 = datastore.newKeyFactory().setKind("CoffeeBean").newKey("Robusta"); - Entity entity2 = Entity.newBuilder(coffeeBeanKey2) + Entity entity2 = + Entity.newBuilder(coffeeBeanKey2) .set("name", "Robusta") .set("description", "Information about the Robusta coffee beans.") .set("roast", "light") @@ -62,7 +63,8 @@ public void setUp() { .build(); coffeeBeanKey3 = datastore.newKeyFactory().setKind("CoffeeBean").newKey("Excelsa"); - Entity entity3 = Entity.newBuilder(coffeeBeanKey3) + Entity entity3 = + Entity.newBuilder(coffeeBeanKey3) .set("name", "Excelsa") .set("description", "Information about the Excelsa coffee beans.") .set("roast", "dark")