Skip to content

Commit

Permalink
#XD-718 fixed
Browse files Browse the repository at this point in the history
  • Loading branch information
penemue committed Jan 30, 2019
1 parent 654f47a commit 1b78995
Show file tree
Hide file tree
Showing 10 changed files with 262 additions and 29 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -172,8 +172,12 @@ public PersistentEntityStoreImpl(@NotNull final PersistentEntityStoreConfig conf
location = environment.getLocation();

// if database is in-memory then never create blobs in BlobVault
final String provider = environment.getEnvironmentConfig().getLogDataReaderWriterProvider();
readerWriterProvider = DataReaderWriterProvider.getProvider(provider);
final String providerName = environment.getEnvironmentConfig().getLogDataReaderWriterProvider();
final DataReaderWriterProvider provider = DataReaderWriterProvider.getProvider(providerName);
if (provider == null) {
throw new InvalidSettingException("Unknown DataReaderWriterProvider: " + providerName);
}
readerWriterProvider = provider;
if (readerWriterProvider.isInMemory()) {
config.setMaxInPlaceBlobSize(Integer.MAX_VALUE);
}
Expand Down
3 changes: 3 additions & 0 deletions environment/src/main/java/jetbrains/exodus/log/LogConfig.java
Original file line number Diff line number Diff line change
Expand Up @@ -297,6 +297,9 @@ public static LogConfig create(@NotNull final DataReader reader, @NotNull final
public DataReaderWriterProvider getReaderWriterProvider() {
if (readerWriterProviderInstance == null) {
readerWriterProviderInstance = DataReaderWriterProvider.getProvider(readerWriterProvider);
if (readerWriterProviderInstance == null) {
throw new InvalidSettingException("Unknown DataReaderWriterProvider: " + readerWriterProvider);
}
}
return readerWriterProviderInstance;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,9 @@ open class FileDataReaderWriterProvider : DataReaderWriterProvider() {
return Pair(reader, newFileDataWriter(location, reader))
}

override fun onEnvironmentCreated(env: Environment) {
super.onEnvironmentCreated(env)
this.env = env as EnvironmentImpl
override fun onEnvironmentCreated(environment: Environment) {
super.onEnvironmentCreated(environment)
this.env = environment as EnvironmentImpl
}

protected open fun newFileDataReader(location: String): DataReader {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,9 @@ class S3DataReaderWriterProvider @JvmOverloads constructor(

private var env: EnvironmentImpl? = null

override fun onEnvironmentCreated(env: Environment) {
super.onEnvironmentCreated(env)
this.env = env as EnvironmentImpl
override fun onEnvironmentCreated(environment: Environment) {
super.onEnvironmentCreated(environment)
this.env = environment as EnvironmentImpl
}

override fun newReaderWriter(location: String): Pair<DataReader, DataWriter> {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1256,9 +1256,9 @@ public EnvironmentConfig setFullFileReadonly(final boolean readonly) {
/**
* Returns fully-qualified name of the {@linkplain DataReaderWriterProvider} service provide interface implementation which
* will be used to create {@linkplain DataReader} and {@linkplain DataWriter} instances. This setting can be used
* to customize storageL define in-memory one, in-cloud, etc.
* to customize storage: define in-memory one, in-cloud, etc.
* Default value is {@linkplain DataReaderWriterProvider#DEFAULT_READER_WRITER_PROVIDER} which means that file system must be
* used as a storage. Several settings are applicable only to FileDataReaderWriterProvider used:
* used as a storage. Several settings are applicable only if FileDataReaderWriterProvider is used:
* {@linkplain #LOG_DURABLE_WRITE}, {@linkplain #LOG_SYNC_PERIOD}, {@linkplain #LOG_CACHE_OPEN_FILES},
* {@linkplain #LOG_FULL_FILE_READ_ONLY}, {@linkplain #LOG_CACHE_USE_NIO}, {@linkplain #LOG_CACHE_FREE_PHYSICAL_MEMORY_THRESHOLD}.
* <p>Mutable at runtime: no
Expand All @@ -1272,9 +1272,9 @@ public String getLogDataReaderWriterProvider() {
/**
* Sets fully-qualified name of the {@linkplain DataReaderWriterProvider} service provide interface implementation which
* will be used to create {@linkplain DataReader} and {@linkplain DataWriter} instances. This setting can be used
* to customize storageL define in-memory one, in-cloud, etc.
* to customize storage: define in-memory one, in-cloud, etc.
* Default value is {@linkplain DataReaderWriterProvider#DEFAULT_READER_WRITER_PROVIDER} which means that file system must be
* used as a storage. Several settings are applicable only to FileDataReaderWriterProvider used:
* used as a storage. Several settings are applicable only if FileDataReaderWriterProvider is used:
* {@linkplain #LOG_DURABLE_WRITE}, {@linkplain #LOG_SYNC_PERIOD}, {@linkplain #LOG_CACHE_OPEN_FILES},
* {@linkplain #LOG_FULL_FILE_READ_ONLY}, {@linkplain #LOG_CACHE_USE_NIO}, {@linkplain #LOG_CACHE_FREE_PHYSICAL_MEMORY_THRESHOLD}.
* <p>Mutable at runtime: no
Expand Down
47 changes: 46 additions & 1 deletion openAPI/src/main/java/jetbrains/exodus/io/Block.java
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,59 @@
*/
package jetbrains.exodus.io;

// TODO: document
import jetbrains.exodus.env.EnvironmentConfig;

/**
* {@code Block} represents single {@code .xd} file in {@code Log} on storage device - disk, memory, network file
* system. etc. {@code Block} is identified in the log by its {@linkplain #getAddress() address}. {@code Block}
* has {@linkplain #length() length} in bytes, which is not greater than {@linkplain EnvironmentConfig#getLogFileSize()
* maximum log block size}. {@code Block} implementation defines the way data is
* {@linkplain #read(byte[], long, int, int) read} from storage device.
*
* {@code Log} blocks can be mutable and immutable. All blocks having length equal to {@linkplain EnvironmentConfig#getLogFileSize()
* maximum log block size} are immutable. In any moment, only one block can be mutable. This has maximum {@linkplain
* #getAddress() address}.
*
* @see DataReader
* @see DataWriter
* @see DataReaderWriterProvider
* @see EnvironmentConfig#getLogFileSize()
* @since 1.3.0
*/
public interface Block {

/**
* Returns address of the block in the {@code Log}. This address is always constant for the block.
*
* @return address of the block in the {@code Log}
*/
long getAddress();

/**
* Returns length of the block in bytes. Block length is always not greater
* than {@linkplain EnvironmentConfig#getLogFileSize() maximum log file size}.
*
* @return length of the block in bytes
* @see EnvironmentConfig#getLogFileSize()
*/
long length();

/**
* Reads data from the underlying store device.
*
* @param output array to copy data
* @param position starting position in the {@code .xd} file
* @param offset starting offset in the array
* @param count number of bytes to read
* @return actual number of bytes read
*/
int read(byte[] output, long position, int offset, int count);

/**
* For immutable {@code Block} implementations, this method returns fresh representation of the same {@code .xd}
* file. For mutable {@code Block} implementations, this method just returns this {@code Block} instance.
*
* @return fresh representation of the same {@code .xd} file
*/
Block refresh();
}
40 changes: 38 additions & 2 deletions openAPI/src/main/java/jetbrains/exodus/io/DataReader.java
Original file line number Diff line number Diff line change
Expand Up @@ -17,20 +17,56 @@

import org.jetbrains.annotations.NotNull;

// TODO: document
/**
* {@code DataReader} defines basic structure of {@code Log}. {@code DataReader} provides access to all {@linkplain
* Block blocks} or groups of successive {@linkplain Block blocks} in the log.
*
* @see Block
* @see DataWriter
* @see DataReaderWriterProvider
* @since 1.3.0
*/
public interface DataReader {

/**
* Returns {@code Log} location how it was passed as a parameter to {@linkplain DataReaderWriterProvider#newReaderWriter(String)}.
*
* @return location how it was passed as a parameter to {@linkplain DataReaderWriterProvider#newReaderWriter(String)}
* @see DataReaderWriterProvider#newReaderWriter(String)
*/
@NotNull
String getLocation();

/**
* @return array of blocks sorted by address.
* Returns all {@code Log} {@linkplain Block blocks} sorted by {@linkplain Block#getAddress() address}.
*
* @return {@linkplain Block blocks} sorted by {@linkplain Block#getAddress() address}
* @see Block
* @see Block#getAddress()
*/
@NotNull
Iterable<Block> getBlocks();

/**
* Returns {@linkplain Block blocks} sorted by {@linkplain Block#getAddress() address} with address greater than
* or equal to specified {@code fromAddress}.
*
* @param fromAddress starting block address
* @return {@linkplain Block blocks} sorted by {@linkplain Block#getAddress() address}
* @see Block
* @see Block#getAddress()
*/
@NotNull
Iterable<Block> getBlocks(long fromAddress);

/**
* Closes {@code DataReader} and all open resources associated with it (files, connections, etc.). After the
* {@code DataReader} is closed, any {@linkplain Block block} got using {@linkplain #getBlocks()} or
* {@linkplain #getBlocks(long)} method would be no longer accessible.
*
* @see Block
* @see #getBlocks()
* @see #getBlocks(long)
*/
void close();
}
Original file line number Diff line number Diff line change
Expand Up @@ -15,37 +15,92 @@
*/
package jetbrains.exodus.io;

import jetbrains.exodus.InvalidSettingException;
import jetbrains.exodus.core.dataStructures.Pair;
import jetbrains.exodus.env.Environment;
import jetbrains.exodus.env.EnvironmentConfig;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;

import java.util.ServiceLoader;

/**
* Service provider interface for creation instances of {@linkplain DataReader} and {@linkplain DataWriter}.
* {@linkplain DataReader} and {@linkplain DataWriter} are used by {@code Log} implementation to perform basic
* operations with {@linkplain Block blocks} ({@code .xd} files) and basic read/write/delete operations.
*
* Service provider interface is identified by a fully-qualified name of its implementation. When opening an
* {@linkplain Environment}, {@linkplain #DEFAULT_READER_WRITER_PROVIDER} is used as default provide name. To use a
* custom I/O provider, specify its fully-qualified name as a parameter of {@linkplain EnvironmentConfig#setLogDataReaderWriterProvider}.
*
* On {@linkplain Environment} creation new instance of {@code DataReaderWriterProvider} is created.
*
* @see Block
* @see DataReader
* @see DataWriter
* @see EnvironmentConfig#getLogDataReaderWriterProvider
* @see EnvironmentConfig#setLogDataReaderWriterProvider
* @since 1.3.0
*/
public abstract class DataReaderWriterProvider {

/**
* Fully-qualified name of default {@code }DataReaderWriteProvider}.
*/
public static final String DEFAULT_READER_WRITER_PROVIDER = "jetbrains.exodus.io.FileDataReaderWriterProvider";

/**
* Creates pair of new instances of {@linkplain DataReader} and {@linkplain DataWriter} by specified location.
* What is location depends on the implementation of {@code DataReaderWriterProvider}, e.g. for {@code FileDataReaderWriterProvider}
* location is a full path on local file system where the database is located.
*
* @param location identifies the database in this {@code DataReaderWriterProvider}
* @return pair of new instances of {@linkplain DataReader} and {@linkplain DataWriter}
*/
public abstract Pair<DataReader, DataWriter> newReaderWriter(@NotNull final String location);

/**
* Returns {@code true} if the {@code DataReaderWriterProvider} creates in-memory {@linkplain DataReader} and {@linkplain DataWriter}.
*
* @return {@code true} if the {@code DataReaderWriterProvider} creates in-memory {@linkplain DataReader} and {@linkplain DataWriter}
*/
public boolean isInMemory() {
return false;
}

/**
* Returns {@code true} if the {@code DataReaderWriterProvider} creates read-only {@linkplain DataWriter}.
*
* @return {@code true} if the {@code DataReaderWriterProvider} creates read-only {@linkplain DataWriter}
*/
public boolean isReadonly() {
return false;
}

public void onEnvironmentCreated(@NotNull final Environment env) {
/**
* Callback method which is called when an environment is been opened/created. Can be used in implementation of
* the {@code DataReaderWriterProvider} to access directly an {@linkplain Environment} instance,
* its {@linkplain Environment#getEnvironmentConfig() config}, etc. Creation of {@code environment} is not
* completed when the method is called.
*
* @param environment {@linkplain Environment} instance which is been opened/created using this
* {@code DataReaderWriterProvider}
*/
public void onEnvironmentCreated(@NotNull final Environment environment) {
}

@NotNull
/**
* Gets a {@code DataReaderWriterProvider} implementation by specified provider name.
*
* @param providerName fully-qualified name of {@code DataReaderWriterProvider} implementation
* @return {@code DataReaderWriterProvider} implementation of {@code null} if the service could not be loaded
*/
@Nullable
public static DataReaderWriterProvider getProvider(@NotNull final String providerName) {
for (DataReaderWriterProvider provider : ServiceLoader.load(DataReaderWriterProvider.class)) {
if (provider.getClass().getCanonicalName().equalsIgnoreCase(providerName)) {
return provider;
}
}
throw new InvalidSettingException("Unknown DataReaderWriterProvider: " + providerName);
return null;
}
}
Loading

0 comments on commit 1b78995

Please sign in to comment.