From 1b78995fcd9911364922854a17177461a1ecf1b1 Mon Sep 17 00:00:00 2001 From: Vyacheslav Lukianov Date: Wed, 30 Jan 2019 15:01:07 +0300 Subject: [PATCH] #XD-718 fixed --- .../PersistentEntityStoreImpl.java | 8 +- .../java/jetbrains/exodus/log/LogConfig.java | 3 + .../exodus/io/FileDataReaderWriterProvider.kt | 6 +- .../replication/S3DataReaderWriterProvider.kt | 6 +- .../exodus/env/EnvironmentConfig.java | 8 +- .../main/java/jetbrains/exodus/io/Block.java | 47 +++++++++- .../java/jetbrains/exodus/io/DataReader.java | 40 +++++++- .../exodus/io/DataReaderWriterProvider.java | 63 ++++++++++++- .../java/jetbrains/exodus/io/DataWriter.java | 94 +++++++++++++++++-- .../jetbrains/exodus/io/RemoveBlockType.java | 16 +++- 10 files changed, 262 insertions(+), 29 deletions(-) diff --git a/entity-store/src/main/java/jetbrains/exodus/entitystore/PersistentEntityStoreImpl.java b/entity-store/src/main/java/jetbrains/exodus/entitystore/PersistentEntityStoreImpl.java index 3d33a9414..fda9ac375 100644 --- a/entity-store/src/main/java/jetbrains/exodus/entitystore/PersistentEntityStoreImpl.java +++ b/entity-store/src/main/java/jetbrains/exodus/entitystore/PersistentEntityStoreImpl.java @@ -172,8 +172,12 @@ public PersistentEntityStoreImpl(@NotNull final PersistentEntityStoreConfig conf location = environment.getLocation(); // if database is in-memory then never create blobs in BlobVault - final String provider = environment.getEnvironmentConfig().getLogDataReaderWriterProvider(); - readerWriterProvider = DataReaderWriterProvider.getProvider(provider); + final String providerName = environment.getEnvironmentConfig().getLogDataReaderWriterProvider(); + final DataReaderWriterProvider provider = DataReaderWriterProvider.getProvider(providerName); + if (provider == null) { + throw new InvalidSettingException("Unknown DataReaderWriterProvider: " + providerName); + } + readerWriterProvider = provider; if (readerWriterProvider.isInMemory()) { config.setMaxInPlaceBlobSize(Integer.MAX_VALUE); } diff --git a/environment/src/main/java/jetbrains/exodus/log/LogConfig.java b/environment/src/main/java/jetbrains/exodus/log/LogConfig.java index 30b786027..dccd009fc 100644 --- a/environment/src/main/java/jetbrains/exodus/log/LogConfig.java +++ b/environment/src/main/java/jetbrains/exodus/log/LogConfig.java @@ -297,6 +297,9 @@ public static LogConfig create(@NotNull final DataReader reader, @NotNull final public DataReaderWriterProvider getReaderWriterProvider() { if (readerWriterProviderInstance == null) { readerWriterProviderInstance = DataReaderWriterProvider.getProvider(readerWriterProvider); + if (readerWriterProviderInstance == null) { + throw new InvalidSettingException("Unknown DataReaderWriterProvider: " + readerWriterProvider); + } } return readerWriterProviderInstance; } diff --git a/environment/src/main/kotlin/jetbrains/exodus/io/FileDataReaderWriterProvider.kt b/environment/src/main/kotlin/jetbrains/exodus/io/FileDataReaderWriterProvider.kt index 92ebe7755..c80f6f1b5 100644 --- a/environment/src/main/kotlin/jetbrains/exodus/io/FileDataReaderWriterProvider.kt +++ b/environment/src/main/kotlin/jetbrains/exodus/io/FileDataReaderWriterProvider.kt @@ -30,9 +30,9 @@ open class FileDataReaderWriterProvider : DataReaderWriterProvider() { return Pair(reader, newFileDataWriter(location, reader)) } - override fun onEnvironmentCreated(env: Environment) { - super.onEnvironmentCreated(env) - this.env = env as EnvironmentImpl + override fun onEnvironmentCreated(environment: Environment) { + super.onEnvironmentCreated(environment) + this.env = environment as EnvironmentImpl } protected open fun newFileDataReader(location: String): DataReader { diff --git a/multinode/src/main/kotlin/jetbrains/exodus/log/replication/S3DataReaderWriterProvider.kt b/multinode/src/main/kotlin/jetbrains/exodus/log/replication/S3DataReaderWriterProvider.kt index b9c63f879..8e532131a 100644 --- a/multinode/src/main/kotlin/jetbrains/exodus/log/replication/S3DataReaderWriterProvider.kt +++ b/multinode/src/main/kotlin/jetbrains/exodus/log/replication/S3DataReaderWriterProvider.kt @@ -38,9 +38,9 @@ class S3DataReaderWriterProvider @JvmOverloads constructor( private var env: EnvironmentImpl? = null - override fun onEnvironmentCreated(env: Environment) { - super.onEnvironmentCreated(env) - this.env = env as EnvironmentImpl + override fun onEnvironmentCreated(environment: Environment) { + super.onEnvironmentCreated(environment) + this.env = environment as EnvironmentImpl } override fun newReaderWriter(location: String): Pair { diff --git a/openAPI/src/main/java/jetbrains/exodus/env/EnvironmentConfig.java b/openAPI/src/main/java/jetbrains/exodus/env/EnvironmentConfig.java index cccf1cad2..2142146d4 100644 --- a/openAPI/src/main/java/jetbrains/exodus/env/EnvironmentConfig.java +++ b/openAPI/src/main/java/jetbrains/exodus/env/EnvironmentConfig.java @@ -1256,9 +1256,9 @@ public EnvironmentConfig setFullFileReadonly(final boolean readonly) { /** * Returns fully-qualified name of the {@linkplain DataReaderWriterProvider} service provide interface implementation which * will be used to create {@linkplain DataReader} and {@linkplain DataWriter} instances. This setting can be used - * to customize storageL define in-memory one, in-cloud, etc. + * to customize storage: define in-memory one, in-cloud, etc. * Default value is {@linkplain DataReaderWriterProvider#DEFAULT_READER_WRITER_PROVIDER} which means that file system must be - * used as a storage. Several settings are applicable only to FileDataReaderWriterProvider used: + * used as a storage. Several settings are applicable only if FileDataReaderWriterProvider is used: * {@linkplain #LOG_DURABLE_WRITE}, {@linkplain #LOG_SYNC_PERIOD}, {@linkplain #LOG_CACHE_OPEN_FILES}, * {@linkplain #LOG_FULL_FILE_READ_ONLY}, {@linkplain #LOG_CACHE_USE_NIO}, {@linkplain #LOG_CACHE_FREE_PHYSICAL_MEMORY_THRESHOLD}. *

Mutable at runtime: no @@ -1272,9 +1272,9 @@ public String getLogDataReaderWriterProvider() { /** * Sets fully-qualified name of the {@linkplain DataReaderWriterProvider} service provide interface implementation which * will be used to create {@linkplain DataReader} and {@linkplain DataWriter} instances. This setting can be used - * to customize storageL define in-memory one, in-cloud, etc. + * to customize storage: define in-memory one, in-cloud, etc. * Default value is {@linkplain DataReaderWriterProvider#DEFAULT_READER_WRITER_PROVIDER} which means that file system must be - * used as a storage. Several settings are applicable only to FileDataReaderWriterProvider used: + * used as a storage. Several settings are applicable only if FileDataReaderWriterProvider is used: * {@linkplain #LOG_DURABLE_WRITE}, {@linkplain #LOG_SYNC_PERIOD}, {@linkplain #LOG_CACHE_OPEN_FILES}, * {@linkplain #LOG_FULL_FILE_READ_ONLY}, {@linkplain #LOG_CACHE_USE_NIO}, {@linkplain #LOG_CACHE_FREE_PHYSICAL_MEMORY_THRESHOLD}. *

Mutable at runtime: no diff --git a/openAPI/src/main/java/jetbrains/exodus/io/Block.java b/openAPI/src/main/java/jetbrains/exodus/io/Block.java index 8066270fb..e8def3245 100644 --- a/openAPI/src/main/java/jetbrains/exodus/io/Block.java +++ b/openAPI/src/main/java/jetbrains/exodus/io/Block.java @@ -15,14 +15,59 @@ */ package jetbrains.exodus.io; -// TODO: document +import jetbrains.exodus.env.EnvironmentConfig; + +/** + * {@code Block} represents single {@code .xd} file in {@code Log} on storage device - disk, memory, network file + * system. etc. {@code Block} is identified in the log by its {@linkplain #getAddress() address}. {@code Block} + * has {@linkplain #length() length} in bytes, which is not greater than {@linkplain EnvironmentConfig#getLogFileSize() + * maximum log block size}. {@code Block} implementation defines the way data is + * {@linkplain #read(byte[], long, int, int) read} from storage device. + * + * {@code Log} blocks can be mutable and immutable. All blocks having length equal to {@linkplain EnvironmentConfig#getLogFileSize() + * maximum log block size} are immutable. In any moment, only one block can be mutable. This has maximum {@linkplain + * #getAddress() address}. + * + * @see DataReader + * @see DataWriter + * @see DataReaderWriterProvider + * @see EnvironmentConfig#getLogFileSize() + * @since 1.3.0 + */ public interface Block { + /** + * Returns address of the block in the {@code Log}. This address is always constant for the block. + * + * @return address of the block in the {@code Log} + */ long getAddress(); + /** + * Returns length of the block in bytes. Block length is always not greater + * than {@linkplain EnvironmentConfig#getLogFileSize() maximum log file size}. + * + * @return length of the block in bytes + * @see EnvironmentConfig#getLogFileSize() + */ long length(); + /** + * Reads data from the underlying store device. + * + * @param output array to copy data + * @param position starting position in the {@code .xd} file + * @param offset starting offset in the array + * @param count number of bytes to read + * @return actual number of bytes read + */ int read(byte[] output, long position, int offset, int count); + /** + * For immutable {@code Block} implementations, this method returns fresh representation of the same {@code .xd} + * file. For mutable {@code Block} implementations, this method just returns this {@code Block} instance. + * + * @return fresh representation of the same {@code .xd} file + */ Block refresh(); } diff --git a/openAPI/src/main/java/jetbrains/exodus/io/DataReader.java b/openAPI/src/main/java/jetbrains/exodus/io/DataReader.java index 42412af53..fda773a8a 100644 --- a/openAPI/src/main/java/jetbrains/exodus/io/DataReader.java +++ b/openAPI/src/main/java/jetbrains/exodus/io/DataReader.java @@ -17,20 +17,56 @@ import org.jetbrains.annotations.NotNull; -// TODO: document +/** + * {@code DataReader} defines basic structure of {@code Log}. {@code DataReader} provides access to all {@linkplain + * Block blocks} or groups of successive {@linkplain Block blocks} in the log. + * + * @see Block + * @see DataWriter + * @see DataReaderWriterProvider + * @since 1.3.0 + */ public interface DataReader { + /** + * Returns {@code Log} location how it was passed as a parameter to {@linkplain DataReaderWriterProvider#newReaderWriter(String)}. + * + * @return location how it was passed as a parameter to {@linkplain DataReaderWriterProvider#newReaderWriter(String)} + * @see DataReaderWriterProvider#newReaderWriter(String) + */ @NotNull String getLocation(); /** - * @return array of blocks sorted by address. + * Returns all {@code Log} {@linkplain Block blocks} sorted by {@linkplain Block#getAddress() address}. + * + * @return {@linkplain Block blocks} sorted by {@linkplain Block#getAddress() address} + * @see Block + * @see Block#getAddress() */ @NotNull Iterable getBlocks(); + /** + * Returns {@linkplain Block blocks} sorted by {@linkplain Block#getAddress() address} with address greater than + * or equal to specified {@code fromAddress}. + * + * @param fromAddress starting block address + * @return {@linkplain Block blocks} sorted by {@linkplain Block#getAddress() address} + * @see Block + * @see Block#getAddress() + */ @NotNull Iterable getBlocks(long fromAddress); + /** + * Closes {@code DataReader} and all open resources associated with it (files, connections, etc.). After the + * {@code DataReader} is closed, any {@linkplain Block block} got using {@linkplain #getBlocks()} or + * {@linkplain #getBlocks(long)} method would be no longer accessible. + * + * @see Block + * @see #getBlocks() + * @see #getBlocks(long) + */ void close(); } diff --git a/openAPI/src/main/java/jetbrains/exodus/io/DataReaderWriterProvider.java b/openAPI/src/main/java/jetbrains/exodus/io/DataReaderWriterProvider.java index be0a86d6d..5605ad6a3 100644 --- a/openAPI/src/main/java/jetbrains/exodus/io/DataReaderWriterProvider.java +++ b/openAPI/src/main/java/jetbrains/exodus/io/DataReaderWriterProvider.java @@ -15,37 +15,92 @@ */ package jetbrains.exodus.io; -import jetbrains.exodus.InvalidSettingException; import jetbrains.exodus.core.dataStructures.Pair; import jetbrains.exodus.env.Environment; +import jetbrains.exodus.env.EnvironmentConfig; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import java.util.ServiceLoader; +/** + * Service provider interface for creation instances of {@linkplain DataReader} and {@linkplain DataWriter}. + * {@linkplain DataReader} and {@linkplain DataWriter} are used by {@code Log} implementation to perform basic + * operations with {@linkplain Block blocks} ({@code .xd} files) and basic read/write/delete operations. + * + * Service provider interface is identified by a fully-qualified name of its implementation. When opening an + * {@linkplain Environment}, {@linkplain #DEFAULT_READER_WRITER_PROVIDER} is used as default provide name. To use a + * custom I/O provider, specify its fully-qualified name as a parameter of {@linkplain EnvironmentConfig#setLogDataReaderWriterProvider}. + * + * On {@linkplain Environment} creation new instance of {@code DataReaderWriterProvider} is created. + * + * @see Block + * @see DataReader + * @see DataWriter + * @see EnvironmentConfig#getLogDataReaderWriterProvider + * @see EnvironmentConfig#setLogDataReaderWriterProvider + * @since 1.3.0 + */ public abstract class DataReaderWriterProvider { + /** + * Fully-qualified name of default {@code }DataReaderWriteProvider}. + */ public static final String DEFAULT_READER_WRITER_PROVIDER = "jetbrains.exodus.io.FileDataReaderWriterProvider"; + /** + * Creates pair of new instances of {@linkplain DataReader} and {@linkplain DataWriter} by specified location. + * What is location depends on the implementation of {@code DataReaderWriterProvider}, e.g. for {@code FileDataReaderWriterProvider} + * location is a full path on local file system where the database is located. + * + * @param location identifies the database in this {@code DataReaderWriterProvider} + * @return pair of new instances of {@linkplain DataReader} and {@linkplain DataWriter} + */ public abstract Pair newReaderWriter(@NotNull final String location); + /** + * Returns {@code true} if the {@code DataReaderWriterProvider} creates in-memory {@linkplain DataReader} and {@linkplain DataWriter}. + * + * @return {@code true} if the {@code DataReaderWriterProvider} creates in-memory {@linkplain DataReader} and {@linkplain DataWriter} + */ public boolean isInMemory() { return false; } + /** + * Returns {@code true} if the {@code DataReaderWriterProvider} creates read-only {@linkplain DataWriter}. + * + * @return {@code true} if the {@code DataReaderWriterProvider} creates read-only {@linkplain DataWriter} + */ public boolean isReadonly() { return false; } - public void onEnvironmentCreated(@NotNull final Environment env) { + /** + * Callback method which is called when an environment is been opened/created. Can be used in implementation of + * the {@code DataReaderWriterProvider} to access directly an {@linkplain Environment} instance, + * its {@linkplain Environment#getEnvironmentConfig() config}, etc. Creation of {@code environment} is not + * completed when the method is called. + * + * @param environment {@linkplain Environment} instance which is been opened/created using this + * {@code DataReaderWriterProvider} + */ + public void onEnvironmentCreated(@NotNull final Environment environment) { } - @NotNull + /** + * Gets a {@code DataReaderWriterProvider} implementation by specified provider name. + * + * @param providerName fully-qualified name of {@code DataReaderWriterProvider} implementation + * @return {@code DataReaderWriterProvider} implementation of {@code null} if the service could not be loaded + */ + @Nullable public static DataReaderWriterProvider getProvider(@NotNull final String providerName) { for (DataReaderWriterProvider provider : ServiceLoader.load(DataReaderWriterProvider.class)) { if (provider.getClass().getCanonicalName().equalsIgnoreCase(providerName)) { return provider; } } - throw new InvalidSettingException("Unknown DataReaderWriterProvider: " + providerName); + return null; } } diff --git a/openAPI/src/main/java/jetbrains/exodus/io/DataWriter.java b/openAPI/src/main/java/jetbrains/exodus/io/DataWriter.java index d94438750..3182daed0 100644 --- a/openAPI/src/main/java/jetbrains/exodus/io/DataWriter.java +++ b/openAPI/src/main/java/jetbrains/exodus/io/DataWriter.java @@ -15,51 +15,127 @@ */ package jetbrains.exodus.io; +import jetbrains.exodus.ExodusException; +import jetbrains.exodus.env.EnvironmentConfig; import org.jetbrains.annotations.NotNull; import java.io.Closeable; -// TODO: document +/** + * {@code DataWriter} defines the way how data is written to {@code Log}, how {@linkplain Block blocks} appear in + * the log and how they are removed from the log. + * + * {@code Log} blocks can be mutable and immutable. All blocks having length equal to {@linkplain EnvironmentConfig#getLogFileSize() + * maximum log block size} are immutable. In any moment, only one block can be mutable. This has maximum {@linkplain + * Block#getAddress() address}. {@code DataWriter} always writes to a single mutable block, that makes {@code log} + * appendable. + * + * @see Block + * @see DataReader + * @see DataReaderWriterProvider + * @since 1.3.0 + */ public interface DataWriter extends Closeable { + /** + * Returns {@code true} is the DataWriter is open. i.e. there is an incomplete mutable {@linkplain Block block} + * having length less than {@linkplain EnvironmentConfig#getLogFileSize() maximum log block size}. + * + * @return {@code true} is the DataWriter is open + */ boolean isOpen(); + /** + * Writes (appends) binary data to {@code Log}. Returns new {@linkplain Block} instance representing mutable block. + * + * @param b binary data array + * @param off starting offset in the array + * @param len number of byte to write + * @return mutable {@linkplain Block} instance + * @see Block + */ Block write(byte[] b, int off, int len); + /** + * If applicable, forces flush of written data to underlying storage device. Transaction durability depends + * on the implementation of this method. + */ void sync(); + /** + * If applicable, forces flush of changes in directory structure to underlying storage device. + */ void syncDirectory(); - @Override + /** + * Closes the data writer. {@code Log} closes the data writer each time when mutable {@linkplain Block block} + * becomes immutable, i.e. reaches its {@linkplain EnvironmentConfig#getLogFileSize() maximum log block size}. + */ void close(); + /** + * Clears {@code Log} by location specified as a parameter to {@linkplain DataReaderWriterProvider#newReaderWriter(String)}. + * The database becomes empty. + */ void clear(); + /** + * Open existing {@linkplain Block block} for writing or creates the new one by specified address with specified length. + * This method is used in the log recovery procedure, so {@code DataWriter} should truncate existing block + * if its physical length is greater than specified one. This method is always applied to a mutable {@linkplain Block block}. + * + * @param address address of {@linkplain Block block} to open or create + * @param length valid {@linkplain Block block} length + * @return mutable {@linkplain Block} instance + */ Block openOrCreateBlock(long address, long length); + /** + * Removes existing immutable {@linkplain Block block}. If applicable, this method can rename + * {@linkplain Block block} (file on a file system) instead of removing it if specified + * {@code RemoveBlockType rbt} is equal to {@linkplain RemoveBlockType#Rename}. + * + * @param blockAddress address of {@linkplain Block block} to remove + * @param rbt {@linkplain RemoveBlockType#Rename} to rename {@linkplain Block block} instead of removing it. + * @see RemoveBlockType + */ void removeBlock(long blockAddress, @NotNull RemoveBlockType rbt); + /** + * Truncates existing {@linkplain Block block} to specified length. Does nothing is specified length is greater + * than or equal to physical {@linkplain Block block} length. + * + * @param blockAddress address of {@linkplain Block block} to truncate + * @param length {@linkplain Block block} length + */ void truncateBlock(long blockAddress, long length); /** - * Try to lock writer during specified time. + * If applicable, tries to acquire writer's lock in specified time. If the lock is acquired, returns {@code true}. + * Successfully acquired lock guarantees that {@code Log} cannot be opened in parallel (within same + * JVM or not) unless it is released by the writer. * - * @param timeout - if writer is already locked try to lock it during specified timeout. - * @return true if locked successfully, false otherwise + * @param timeout - time to wait for lock acquisition + * @return {@code true} if the lock is acquired + * @see #release() + * @see #lockInfo() */ boolean lock(long timeout); /** - * Releases writer + * Releases writer's lock. * - * @return true if released successfully, false otherwise + * @return {@code true} if released successfully + * @see #lock(long) */ boolean release(); /** - * For debug purposes, returns detailed information about current lock owner. Can be used if lock() failed. + * For debugging purposes, returns detailed information about current lock owner. If {@linkplain #lock(long)} + * return {@code false}, {@code Log} throws an {@linkplain ExodusException} with the lock info in its message. * - * @return Human-readable information about lock owner. + * @return Human-readable information about lock owner + * @see #lock(long) */ String lockInfo(); } diff --git a/openAPI/src/main/java/jetbrains/exodus/io/RemoveBlockType.java b/openAPI/src/main/java/jetbrains/exodus/io/RemoveBlockType.java index f8a3aad7a..7d8c87e9d 100644 --- a/openAPI/src/main/java/jetbrains/exodus/io/RemoveBlockType.java +++ b/openAPI/src/main/java/jetbrains/exodus/io/RemoveBlockType.java @@ -15,7 +15,21 @@ */ package jetbrains.exodus.io; -// TODO: document +import jetbrains.exodus.env.EnvironmentConfig; + +/** + * Type of action applied to the {@linkplain Block blocks} processed by database GC. GC moves accessible data + * from old blocks to new ones, so old blocks can finally be removed using {@linkplain DataWriter#removeBlock(long, RemoveBlockType)}. + * For debugging purposes, for supporting custom backup procedures and so on, blocks can also be renamed, not only removed. + * Actual value of the second parameter passed to {@linkplain DataWriter#removeBlock(long, RemoveBlockType)} is controlled + * by the {@linkplain EnvironmentConfig#GC_RENAME_FILES} setting. + * + * @see Block + * @see DataWriter#removeBlock(long, RemoveBlockType) + * @see EnvironmentConfig#getGcRenameFiles() + * @see EnvironmentConfig#setGcRenameFiles(boolean) + * @since 1.3.0 + */ public enum RemoveBlockType { Delete, Rename