748 background fsync (#766)

* max dirty bytes config param

* update changelog

* [748] Fix build error, use human readable format for `max_dirty_bytes_before_sync`

* [748] Build fix

CHANGELOG.md

@@ -4,7 +4,7 @@ Bob versions changelog
 
 ## [Unreleased]
 #### Added
-
+- Blob performes fsync if buffered bytes are larger than max_dirty_bytes_before_sync config param (#748)
 
 #### Changed
 - Use cargo workspace to declare dependencies to avoid their duplication (#821)

bob-backend/src/pearl/holder.rs

@@ -487,6 +487,7 @@ impl Holder {
         let prefix = self.inner.config.blob_file_name_prefix();
         let max_data = self.inner.config.max_data_in_blob();
         let max_blob_size = self.inner.config.max_blob_size();
+        let max_dirty_bytes_before_sync = self.inner.config.max_dirty_bytes_before_sync();
         let mut filter_config = BloomConfig::default();
         let validate_data_during_index_regen = self.inner.config.validate_data_checksum_during_index_regen();
         if let Some(count) = self.inner.config.max_buf_bits_count() {
@@ -497,6 +498,7 @@ impl Holder {
             .blob_file_name_prefix(prefix)
             .max_data_in_blob(max_data)
             .max_blob_size(max_blob_size)
+            .set_max_dirty_bytes_before_sync(max_dirty_bytes_before_sync)
             .set_filter_config(filter_config)
             .set_validate_data_during_index_regen(validate_data_during_index_regen)
             .set_dump_sem(self.inner.pearl_creation_context.dump_sem.clone())

bob-common/src/configs/node.rs

@@ -273,13 +273,19 @@ pub struct Pearl {
     /// This optimization is unsafe, value should be at least 2 times the maximum value of 'timestamp_period' that was used throughout the lifetime of the cluster
     #[serde(default)]
     skip_holders_by_timestamp_step_when_reading: Option<String>,
+    #[serde(default = "Pearl::default_max_dirty_bytes_before_sync")]
+    max_dirty_bytes_before_sync: ByteUnit,
 }
 
 impl Pearl {
     pub fn max_buf_bits_count(&self) -> Option<usize> {
         self.bloom_filter_max_buf_bits_count
     }
 
+    pub fn max_dirty_bytes_before_sync(&self) -> u64 {
+        self.max_dirty_bytes_before_sync.as_u64()
+    }
+
     pub fn alien_disk(&self) -> Option<&str> {
         self.alien_disk.as_deref()
     }
@@ -330,6 +336,11 @@ impl Pearl {
         true
     }
 
+    fn default_max_dirty_bytes_before_sync() -> ByteUnit {
+        // 10Mb
+        10 * ByteUnit::MiB
+    }
+
     pub fn allow_duplicates(&self) -> bool {
         self.allow_duplicates
     }
@@ -481,7 +492,8 @@ impl Pearl {
             disks_events_logfile: Pearl::default_disks_events_logfile(),
             bloom_filter_max_buf_bits_count: Some(1_000_000),
             validate_data_checksum_during_index_regen: Pearl::default_validate_data_checksum_during_index_regen(),
-            skip_holders_by_timestamp_step_when_reading: None
+            skip_holders_by_timestamp_step_when_reading: None,
+            max_dirty_bytes_before_sync: Pearl::default_max_dirty_bytes_before_sync(),
         }
     }
 }

config-examples/node.yaml

@@ -86,6 +86,8 @@ pearl:
   #skip_holders_by_timestamp_step_when_reading: 2h
   # [size] required for 'pearl'
   max_blob_size: 100 mb
+  # optional, sets the number of bytes, after writing which sync to disk occurs
+  max_dirty_bytes_before_sync: 10 mb
   # optional, required for 'pearl', disables search for existing keys before write
   allow_duplicates: true
   # optional