Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[core] Add benchmark for reader and writer with bloom filter in sort file store #4173

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

[core] Add benchmark for reader and writer with bloom filter in sort file store #4173

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 25 additions & 2 deletions ...-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/AbstractLookupBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,13 @@
import org.apache.paimon.options.MemorySize;
import org.apache.paimon.types.IntType;
import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.BloomFilter;
import org.apache.paimon.utils.Pair;

import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.UUID;
Expand All @@ -47,6 +49,15 @@ abstract class AbstractLookupBenchmark {
new RowCompactedSerializer(RowType.of(new IntType()));
private final GenericRow reusedKey = new GenericRow(1);

protected static List<List<Object>> getCountBloomList() {
List<List<Object>> countBloomList = new ArrayList<>();
for (Integer recordCount : RECORD_COUNT_LIST) {
countBloomList.add(Arrays.asList(recordCount, false));
countBloomList.add(Arrays.asList(recordCount, true));
}
return countBloomList;
}

protected byte[][] generateSequenceInputs(int start, int end) {
int count = end - start;
byte[][] result = new byte[count][6];
Expand Down Expand Up @@ -74,7 +85,12 @@ protected byte[] intToByteArray(int value) {
}

protected Pair<String, LookupStoreFactory.Context> writeData(
Path tempDir, CoreOptions options, byte[][] inputs, int valueLength, boolean sameValue)
Path tempDir,
CoreOptions options,
byte[][] inputs,
int valueLength,
boolean sameValue,
boolean bloomFilterEnabled)
throws IOException {
byte[] value1 = new byte[valueLength];
byte[] value2 = new byte[valueLength];
Expand All @@ -87,7 +103,7 @@ protected Pair<String, LookupStoreFactory.Context> writeData(
keySerializer.createSliceComparator());

File file = new File(tempDir.toFile(), UUID.randomUUID().toString());
LookupStoreWriter writer = factory.createWriter(file, null);
LookupStoreWriter writer = factory.createWriter(file, createBloomFiler(bloomFilterEnabled));
int i = 0;
for (byte[] input : inputs) {
if (sameValue) {
Expand All @@ -104,4 +120,11 @@ protected Pair<String, LookupStoreFactory.Context> writeData(
LookupStoreFactory.Context context = writer.close();
return Pair.of(file.getAbsolutePath(), context);
}

private BloomFilter.Builder createBloomFiler(boolean enabled) {
if (!enabled) {
return null;
}
return BloomFilter.builder(100, 0.01);
}
}
14 changes: 8 additions & 6 deletions ...ro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/LookupReaderBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,15 +49,17 @@
public class LookupReaderBenchmark extends AbstractLookupBenchmark {
private static final int QUERY_KEY_COUNT = 10000;
private final int recordCount;
private final boolean bloomFilterEnabled;
@TempDir Path tempDir;

public LookupReaderBenchmark(int recordCount) {
this.recordCount = recordCount;
public LookupReaderBenchmark(List<Object> countBloomList) {
this.recordCount = (Integer) countBloomList.get(0);
this.bloomFilterEnabled = (Boolean) countBloomList.get(1);
}

@Parameters(name = "record-count-{0}")
public static List<Integer> getVarSeg() {
return RECORD_COUNT_LIST;
@Parameters(name = "countBloom-{0}")
public static List<List<Object>> getVarSeg() {
return getCountBloomList();
}

@TestTemplate
Expand All @@ -81,7 +83,7 @@ private void readLookupDataBenchmark(byte[][] inputs, byte[][] randomInputs)
Collections.singletonMap(
LOOKUP_LOCAL_FILE_TYPE.key(), fileType.name()));
Pair<String, LookupStoreFactory.Context> pair =
writeData(tempDir, options, inputs, valueLength, false);
writeData(tempDir, options, inputs, valueLength, false, bloomFilterEnabled);
benchmark.addCase(
String.format(
"%s-read-%dB-value-%d-num",
Expand Down
20 changes: 14 additions & 6 deletions ...ro-benchmarks/src/test/java/org/apache/paimon/benchmark/lookup/LookupWriterBenchmark.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,15 +38,17 @@
@ExtendWith(ParameterizedTestExtension.class)
public class LookupWriterBenchmark extends AbstractLookupBenchmark {
private final int recordCount;
private final boolean bloomFilterEnabled;
@TempDir Path tempDir;

public LookupWriterBenchmark(int recordCount) {
this.recordCount = recordCount;
public LookupWriterBenchmark(List<Object> countBloomList) {
this.recordCount = (Integer) countBloomList.get(0);
this.bloomFilterEnabled = (Boolean) countBloomList.get(1);
}

@Parameters(name = "record-count-{0}")
public static List<Integer> getVarSeg() {
return RECORD_COUNT_LIST;
@Parameters(name = "countBloom-{0}")
public static List<List<Object>> getVarSeg() {
return getCountBloomList();
}

@TestTemplate
Expand Down Expand Up @@ -78,7 +80,13 @@ private void writeLookupDataBenchmark(byte[][] inputs, boolean sameValue) {
5,
() -> {
try {
writeData(tempDir, options, inputs, valueLength, sameValue);
writeData(
tempDir,
options,
inputs,
valueLength,
sameValue,
bloomFilterEnabled);
} catch (IOException e) {
throw new RuntimeException(e);
}
Expand Down
Loading