Merge pull request #22 from mtak-/bloom-filteer

Bloom filter fallback

Author: mtak-

Cargo.toml

@@ -31,6 +31,7 @@ stats = []
 [dependencies]
 cfg-if = "0.1.9"
 crossbeam-utils = "0.6.5"
+fxhash = "0.2.1"
 lazy_static = "1.3.0"
 lock_api = "0.2.0"
 parking_lot = "0.8.0"

benches/single_threaded_scaling.rs

@@ -11,7 +11,7 @@ mod single_threaded_scaling {
     use swym::{tcell::TCell, thread_key, tx::Ordering};
     use test::Bencher;
 
-    /// this demonstrates issues with the writelog
+    /// This should reveal performance cliffs and regressions in the write log.
     macro_rules! write_count {
         ($name:ident, $lock_name:ident, $atomic_name:ident, $amount:expr) => {
             #[bench]
@@ -75,16 +75,17 @@ mod single_threaded_scaling {
         write_008, lock_write_008, atomic_write_008, 8;
         write_016, lock_write_016, atomic_write_016, 16;
         write_032, lock_write_032, atomic_write_032, 32;
-        write_064, lock_write_064, atomic_write_064, 64;
+        write_063, lock_write_063, atomic_write_063, 63;
 
         // start to hit bloom filter failure here
+        write_064, lock_write_064, atomic_write_064, 64;
         write_065, lock_write_065, atomic_write_065, 65;
         write_066, lock_write_066, atomic_write_066, 66;
         write_067, lock_write_067, atomic_write_067, 67;
         write_068, lock_write_068, atomic_write_068, 68;
 
         write_128, lock_write_128, atomic_write_128, 128;
-
-        write_256, lock_write_256, atomic_write_256, 256
+        write_256, lock_write_256, atomic_write_256, 256;
+        write_512, lock_write_512, atomic_write_512, 512
     }
 }

src/internal.rs

@@ -7,10 +7,12 @@ pub mod alloc;
 #[macro_use]
 pub mod phoenix_tls;
 
-pub mod commit;
+pub mod bloom;
+mod commit;
+mod gc;
+mod parking;
+
 pub mod epoch;
-pub mod gc;
-pub mod parking;
 pub mod read_log;
 pub mod tcell_erased;
 pub mod thread;

src/internal/bloom.rs

@@ -0,0 +1,156 @@
+//! A simple 64bit bloom filter that falls back to an actual HashMap.
+//!
+//!
+//!
+//! Potentially relevant paper: http://www.eecg.toronto.edu/~steffan/papers/jeffrey_spaa11.pdf
+
+use core::{
+    cell::{Cell, UnsafeCell},
+    marker::PhantomData,
+    mem,
+    num::NonZeroUsize,
+};
+use fxhash::FxHashMap;
+use std::collections::hash_map::Entry;
+
+#[derive(Copy, Clone, PartialEq, Eq)]
+pub enum Contained {
+    No,
+    Maybe,
+}
+
+type Filter = usize;
+
+const OVERFLOWED: Filter = !0;
+
+#[derive(Debug)]
+pub struct Bloom<'tcell, K> {
+    filter:   Cell<Filter>,
+    overflow: UnsafeCell<FxHashMap<*const K, usize>>,
+    phantom:  PhantomData<&'tcell K>,
+}
+
+impl<'tcell, K> Bloom<'tcell, K> {
+    #[inline]
+    pub fn new() -> Self {
+        Bloom {
+            filter:   Cell::new(0),
+            overflow: Default::default(),
+            phantom:  PhantomData,
+        }
+    }
+
+    fn overflow(&self) -> &mut FxHashMap<*const K, usize> {
+        unsafe { &mut *self.overflow.get() }
+    }
+
+    #[inline]
+    fn has_overflowed(&self) -> bool {
+        self.filter.get() == OVERFLOWED
+    }
+
+    #[inline]
+    pub fn clear(&mut self) {
+        let filter = *self.filter.get_mut();
+        if filter == OVERFLOWED {
+            self.overflow().clear()
+        }
+        *self.filter.get_mut() = 0;
+        debug_assert!(
+            self.overflow().is_empty(),
+            "`clear` failed to empty the container"
+        );
+        debug_assert!(self.is_empty(), "`clear` failed to empty the container");
+        debug_assert!(
+            !self.has_overflowed(),
+            "`clear` failed to reset to `Inline` storage"
+        );
+    }
+
+    #[inline]
+    pub fn is_empty(&self) -> bool {
+        self.filter.get() == 0
+    }
+
+    #[inline]
+    pub fn to_overflow(&self, offsets: impl Iterator<Item = (&'tcell K, usize)>) {
+        if self.filter.get() != OVERFLOWED {
+            self.run_overflow(offsets)
+        }
+    }
+
+    #[inline(never)]
+    #[cold]
+    fn run_overflow(&self, offsets: impl Iterator<Item = (&'tcell K, usize)>) {
+        self.filter.set(OVERFLOWED);
+        let overflow = self.overflow();
+        overflow.extend(offsets.map(|(k, v)| (k as *const K, v)));
+    }
+
+    #[inline]
+    pub fn contained(&self, key: &K) -> Contained {
+        let bit = bloom_bit(key);
+
+        if unlikely!(self.filter.get() & bit.0.get() != 0) {
+            Contained::Maybe
+        } else {
+            Contained::No
+        }
+    }
+
+    // If this returns Maybe, then there's no guarantee the value was inserted. At that time,
+    // overflowing is required.
+    #[inline]
+    pub fn insert_inline(&self, key: &'tcell K) -> Contained {
+        let filter = self.filter.get();
+        let bit = bloom_bit(key);
+
+        if unlikely!(filter & bit.0.get() != 0) {
+            Contained::Maybe
+        } else {
+            let new_filter = filter | bit.0.get();
+            if new_filter != OVERFLOWED {
+                self.filter.set(new_filter);
+                Contained::No
+            } else {
+                Contained::Maybe
+            }
+        }
+    }
+
+    #[inline]
+    pub fn overflow_get(&self, key: &K) -> Option<usize> {
+        debug_assert!(self.has_overflowed());
+        self.overflow().get(&(key as _)).cloned()
+    }
+
+    #[inline]
+    pub fn overflow_entry(&mut self, key: &K) -> Entry<'_, *const K, usize> {
+        debug_assert!(self.has_overflowed());
+        self.overflow().entry(key as _)
+    }
+}
+
+#[inline]
+const fn calc_shift<T>() -> usize {
+    (mem::align_of::<T>() > 1) as usize
+        + (mem::align_of::<T>() > 2) as usize
+        + (mem::align_of::<T>() > 4) as usize
+        + (mem::align_of::<T>() > 8) as usize
+        + 1 // In practice this +1 results in less failures, however it's not "correct". Any TCell with a
+            // meaningful value happens to have a minimum size of mem::size_of::<usize>() * 2 which might
+            // explain why the +1 is helpful for certain workloads.
+}
+
+#[inline]
+fn bloom_bit<T>(value: *const T) -> BloomBit {
+    let shift = calc_shift::<T>();
+    let raw_hash: usize = value as usize >> shift;
+    let result = 1 << (raw_hash & (mem::size_of::<NonZeroUsize>() * 8 - 1));
+    debug_assert!(result > 0, "bloom_hash should not return 0");
+    let hash = unsafe { NonZeroUsize::new_unchecked(result) };
+    BloomBit(hash)
+}
+
+#[derive(Copy, Clone, PartialEq, Eq)]
+struct BloomBit(NonZeroUsize);