diff --git a/Cargo.lock b/Cargo.lock index 9074b456b6..6224270f95 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3818,6 +3818,7 @@ dependencies = [ "reqwest 0.12.24", "rkyv", "rmp-serde", + "roaring", "seq-macro", "serde", "serde_json", @@ -3829,6 +3830,7 @@ dependencies = [ "tempfile", "textwrap", "thiserror 2.0.17", + "tikv-jemallocator", "time", "tokio", "tracing", diff --git a/Cargo.toml b/Cargo.toml index 9672bb0a9f..1f4181165d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -216,6 +216,7 @@ reqwest-websocket = "0.5.0" rkyv = { version = "0.7.45", default-features = false } rmp-serde = "1.3.0" rmpv = "1.3.0" +roaring = "0.11.3" rstest = "0.15" # Make sure this is the same rustls version used by the `tonic` crate. # See the `ensure_default_crypto_provider` function. diff --git a/crates/dbsp/Cargo.toml b/crates/dbsp/Cargo.toml index 4d1e7d2dad..a40dc4f9ab 100644 --- a/crates/dbsp/Cargo.toml +++ b/crates/dbsp/Cargo.toml @@ -118,6 +118,8 @@ indicatif = { workspace = true } reqwest = { workspace = true, features = ["blocking"] } chrono = { workspace = true, features = ["serde"] } tracing-subscriber = { version = "0.3.20", features = ["env-filter"] } +roaring = { workspace = true } +tikv-jemallocator = { workspace = true } [[bench]] name = "galen" @@ -155,6 +157,10 @@ harness = false name = "star_join" harness = false +[[bench]] +name = "filter_bitmap" +harness = false + [[example]] name = "orgchart" diff --git a/crates/dbsp/benches/filter_bitmap.rs b/crates/dbsp/benches/filter_bitmap.rs new file mode 100644 index 0000000000..c0184b3850 --- /dev/null +++ b/crates/dbsp/benches/filter_bitmap.rs @@ -0,0 +1,665 @@ +//! Membership benchmark for `fastbloom` vs `roaring`. +//! +//! Example: +//! `cargo bench -p dbsp --bench filter_bitmap -- --csv-output filter_bitmap.csv` + +use clap::{Parser, ValueEnum}; +use csv::Writer; +use dbsp::storage::file::BLOOM_FILTER_FALSE_POSITIVE_RATE; +use fastbloom::BloomFilter; +use rand::{RngCore, SeedableRng}; +use rand_chacha::ChaCha8Rng; +use roaring::RoaringBitmap; +use serde::Serialize; +use std::{ + fmt::{Display, Formatter}, + fs::File, + mem::size_of, + path::PathBuf, + time::Instant, +}; +use tikv_jemallocator::Jemalloc; + +#[global_allocator] +static ALLOC: Jemalloc = Jemalloc; + +const DEFAULT_BLOOM_SEED: u128 = 42; +const MIN_BLOOM_EXPECTED_ITEMS: u64 = 64; +const U32_KEY_SPACE_SIZE: u64 = u32::MAX as u64 + 1; +const DEFAULT_LOOKUP_LIMIT: u64 = 50_000_000; + +// Mirror the spine_async size bands, capped to what fits in the u32 keyspace. +const DEFAULT_SPINE_LEVEL_SIZES: [u64; 6] = [ + 14_999, + 99_999, + 999_999, + 9_999_999, + 99_999_999, + 999_999_999, +]; + +fn main() { + let args = Args::parse(); + let num_elements_list = args.num_elements(); + args.validate(&num_elements_list); + + let csv_file = File::create(&args.csv_output) + .unwrap_or_else(|error| panic!("failed to create {}: {error}", args.csv_output.display())); + let mut csv_writer = Writer::from_writer(csv_file); + + println!("benchmark=filter_bitmap"); + println!( + "num_elements={}", + num_elements_list + .iter() + .map(u64::to_string) + .collect::>() + .join(",") + ); + println!("repetitions={}", args.repetitions); + println!("insert_order={}", args.insert_order); + println!("lookup_order={}", args.lookup_order); + println!("insert_seed={}", args.insert_seed); + println!("lookup_seed={}", args.lookup_seed); + println!( + "structures={}", + args.structures + .iter() + .map(ToString::to_string) + .collect::>() + .join(",") + ); + println!( + "bloom_false_positive_rate={}", + args.bloom_false_positive_rate + ); + println!("bloom_seed={}", args.bloom_seed); + println!("csv_output={}", args.csv_output.display()); + println!(); + + for &num_elements in &num_elements_list { + let lookup_count = args + .lookup_count + .unwrap_or(num_elements.min(DEFAULT_LOOKUP_LIMIT)); + let max_false_positive_lookup_count = U32_KEY_SPACE_SIZE - num_elements; + let false_positive_lookup_count = args + .false_positive_lookup_count + .unwrap_or(lookup_count.min(max_false_positive_lookup_count)); + let bloom_expected_items = args + .bloom_expected_items + .unwrap_or(num_elements) + .max(MIN_BLOOM_EXPECTED_ITEMS); + + for structure in &args.structures { + let result = match structure { + Structure::Bloom => benchmark_bloom( + &args, + num_elements, + lookup_count, + false_positive_lookup_count, + bloom_expected_items, + ), + Structure::Roaring => benchmark_roaring(&args, num_elements, lookup_count), + }; + + print_report( + *structure, + &result, + num_elements, + lookup_count, + false_positive_lookup_count, + ); + + csv_writer + .serialize(CsvRow::from_result( + *structure, + &args, + num_elements, + lookup_count, + false_positive_lookup_count, + bloom_expected_items, + &result, + )) + .expect("failed to write CSV row"); + csv_writer.flush().expect("failed to flush CSV writer"); + } + } +} + +#[derive(Parser, Debug, Clone)] +#[command(name = "filter_bitmap")] +#[command(about = "Benchmark fastbloom against roaring bitmap for u32 membership queries")] +struct Args { + /// Comma-separated input sizes. Defaults to spine_async level-cap sizes that fit in u32. + #[arg(long, value_name = "CSV")] + num_elements: Option, + + /// Number of successful lookups to benchmark for each input size. + /// Defaults to min(num_elements, 50_000_000). + #[arg(long)] + lookup_count: Option, + + /// Number of negative lookups used to measure bloom false positives for each input size. + #[arg(long)] + false_positive_lookup_count: Option, + + /// Number of repeated benchmark runs used to compute min/avg/max/std. + #[arg(long, default_value_t = 5)] + repetitions: usize, + + /// Structures to benchmark. + #[arg(long, value_delimiter = ',', default_value = "bloom,roaring")] + structures: Vec, + + /// Insert order over the chosen keyset. + #[arg(long, default_value_t = Order::Random)] + insert_order: Order, + + /// Lookup order over the chosen keyset or sampled subset. + #[arg(long, default_value_t = Order::Random)] + lookup_order: Order, + + /// Seed used when `insert-order=random`. + #[arg(long, default_value_t = 0)] + insert_seed: u64, + + /// Seed used when `lookup-order=random`. + #[arg(long, default_value_t = 1)] + lookup_seed: u64, + + /// Bloom filter false-positive rate. Defaults to DBSP storage default. + #[arg(long, default_value_t = BLOOM_FILTER_FALSE_POSITIVE_RATE)] + bloom_false_positive_rate: f64, + + /// Bloom filter seed. Defaults to DBSP storage seed. + #[arg(long, default_value_t = DEFAULT_BLOOM_SEED)] + bloom_seed: u128, + + /// Expected number of items passed to the bloom filter builder for each input size. + #[arg(long)] + bloom_expected_items: Option, + + /// Output CSV path. + #[arg(long, default_value = "filter_bitmap.csv")] + csv_output: PathBuf, + + // When running with `cargo bench` the binary gets the `--bench` flag, so we + // have to parse and ignore it so clap doesn't reject it. + #[doc(hidden)] + #[arg(long = "bench", hide = true)] + __bench: bool, +} + +impl Args { + fn num_elements(&self) -> Vec { + match &self.num_elements { + Some(csv) => parse_u64_csv(csv), + None => DEFAULT_SPINE_LEVEL_SIZES.to_vec(), + } + } + + fn validate(&self, num_elements_list: &[u64]) { + assert!( + !num_elements_list.is_empty(), + "--num-elements must select at least one size" + ); + assert!( + self.repetitions > 0, + "--repetitions must be greater than zero" + ); + assert!( + !self.structures.is_empty(), + "--structures must select at least one structure" + ); + assert!( + self.bloom_false_positive_rate > 0.0 && self.bloom_false_positive_rate < 1.0, + "--bloom-false-positive-rate must be between 0 and 1" + ); + + for &num_elements in num_elements_list { + assert!( + num_elements > 0, + "--num-elements values must be greater than zero" + ); + assert!( + num_elements <= u32::MAX as u64, + "--num-elements values must be <= {}", + u32::MAX + ); + assert!( + self.lookup_count + .unwrap_or(num_elements.min(DEFAULT_LOOKUP_LIMIT)) + <= num_elements, + "--lookup-count must be <= each --num-elements value" + ); + let max_false_positive_lookup_count = U32_KEY_SPACE_SIZE - num_elements; + assert!( + self.false_positive_lookup_count + .unwrap_or(self.lookup_count.unwrap_or(num_elements).min(max_false_positive_lookup_count)) + <= max_false_positive_lookup_count, + "--false-positive-lookup-count must be <= {} for num_elements={num_elements}", + max_false_positive_lookup_count + ); + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +enum Structure { + Bloom, + Roaring, +} + +impl Display for Structure { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Self::Bloom => f.write_str("bloom"), + Self::Roaring => f.write_str("roaring"), + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +enum Order { + Sequential, + Random, +} + +impl Display for Order { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Self::Sequential => f.write_str("sequential"), + Self::Random => f.write_str("random"), + } + } +} + +#[derive(Clone, Copy, Debug)] +struct AffinePermutation { + len: u64, + multiplier: u64, + offset: u64, +} + +impl AffinePermutation { + fn sequential(len: u64) -> Self { + Self { + len, + multiplier: 1, + offset: 0, + } + } + + fn random(len: u64, seed: u64) -> Self { + let mut rng = ChaCha8Rng::seed_from_u64(seed); + let mut multiplier = (rng.next_u64() % len) | 1; + while gcd(multiplier, len) != 1 { + multiplier = (multiplier + 2) % len; + if multiplier == 0 { + multiplier = 1; + } + } + let offset = rng.next_u64() % len; + Self { + len, + multiplier, + offset, + } + } + + fn for_order(len: u64, order: Order, seed: u64) -> Self { + match order { + Order::Sequential => Self::sequential(len), + Order::Random => Self::random(len, seed), + } + } + + fn index_at(&self, position: u64) -> u64 { + debug_assert!(position < self.len); + (self + .multiplier + .wrapping_mul(position) + .wrapping_add(self.offset)) + % self.len + } + + fn key_at(&self, position: u64) -> u32 { + self.index_at(position) as u32 + } + + fn absent_key_at(&self, position: u64, inserted_keys: u64) -> u32 { + (inserted_keys + self.index_at(position)) as u32 + } +} + +fn gcd(mut lhs: u64, mut rhs: u64) -> u64 { + while rhs != 0 { + let next = lhs % rhs; + lhs = rhs; + rhs = next; + } + lhs +} + +fn parse_u64_csv(csv: &str) -> Vec { + let mut out: Vec = csv + .split(',') + .filter(|entry| !entry.trim().is_empty()) + .map(|entry| { + entry + .trim() + .parse::() + .unwrap_or_else(|error| panic!("invalid u64 in --num-elements: {entry} ({error})")) + }) + .collect(); + out.sort_unstable(); + out.dedup(); + out +} + +#[derive(Debug, Clone, Copy)] +struct SummaryStats { + min: f64, + avg: f64, + max: f64, + stddev: f64, +} + +impl SummaryStats { + fn from_samples(samples: &[f64]) -> Self { + let min = samples.iter().copied().fold(f64::INFINITY, f64::min); + let max = samples.iter().copied().fold(f64::NEG_INFINITY, f64::max); + let avg = samples.iter().sum::() / samples.len() as f64; + let variance = samples + .iter() + .map(|sample| { + let delta = *sample - avg; + delta * delta + }) + .sum::() + / samples.len() as f64; + Self { + min, + avg, + max, + stddev: variance.sqrt(), + } + } +} + +#[derive(Debug, Clone, Copy)] +struct BenchmarkResult { + insert_ns_per_element: SummaryStats, + lookup_ns_per_element: SummaryStats, + bytes_used: usize, + false_positive_rate_percent: Option, +} + +#[derive(Debug, Serialize)] +struct CsvRow { + structure: &'static str, + num_elements: u64, + lookup_count: u64, + false_positive_lookup_count: u64, + repetitions: usize, + insert_order: &'static str, + lookup_order: &'static str, + insert_seed: u64, + lookup_seed: u64, + bloom_false_positive_rate_target_percent: f64, + bloom_seed: u128, + bloom_expected_items: u64, + bytes_used: usize, + bytes_per_element: f64, + bits_per_element: Option, + insert_ns_per_element_min: f64, + insert_ns_per_element_avg: f64, + insert_ns_per_element_max: f64, + insert_ns_per_element_stddev: f64, + lookup_ns_per_element_min: f64, + lookup_ns_per_element_avg: f64, + lookup_ns_per_element_max: f64, + lookup_ns_per_element_stddev: f64, + false_positive_rate_percent_min: Option, + false_positive_rate_percent_avg: Option, + false_positive_rate_percent_max: Option, + false_positive_rate_percent_stddev: Option, +} + +impl CsvRow { + fn from_result( + structure: Structure, + args: &Args, + num_elements: u64, + lookup_count: u64, + false_positive_lookup_count: u64, + bloom_expected_items: u64, + result: &BenchmarkResult, + ) -> Self { + let bits_per_element = (structure == Structure::Bloom) + .then_some((result.bytes_used as f64 * 8.0) / num_elements as f64); + let false_positive_stats = result.false_positive_rate_percent; + + Self { + structure: structure.as_str(), + num_elements, + lookup_count, + false_positive_lookup_count, + repetitions: args.repetitions, + insert_order: args.insert_order.as_str(), + lookup_order: args.lookup_order.as_str(), + insert_seed: args.insert_seed, + lookup_seed: args.lookup_seed, + bloom_false_positive_rate_target_percent: args.bloom_false_positive_rate * 100.0, + bloom_seed: args.bloom_seed, + bloom_expected_items, + bytes_used: result.bytes_used, + bytes_per_element: result.bytes_used as f64 / num_elements as f64, + bits_per_element, + insert_ns_per_element_min: result.insert_ns_per_element.min, + insert_ns_per_element_avg: result.insert_ns_per_element.avg, + insert_ns_per_element_max: result.insert_ns_per_element.max, + insert_ns_per_element_stddev: result.insert_ns_per_element.stddev, + lookup_ns_per_element_min: result.lookup_ns_per_element.min, + lookup_ns_per_element_avg: result.lookup_ns_per_element.avg, + lookup_ns_per_element_max: result.lookup_ns_per_element.max, + lookup_ns_per_element_stddev: result.lookup_ns_per_element.stddev, + false_positive_rate_percent_min: false_positive_stats.map(|stats| stats.min), + false_positive_rate_percent_avg: false_positive_stats.map(|stats| stats.avg), + false_positive_rate_percent_max: false_positive_stats.map(|stats| stats.max), + false_positive_rate_percent_stddev: false_positive_stats.map(|stats| stats.stddev), + } + } +} + +impl Structure { + fn as_str(self) -> &'static str { + match self { + Self::Bloom => "bloom", + Self::Roaring => "roaring", + } + } +} + +impl Order { + fn as_str(self) -> &'static str { + match self { + Self::Sequential => "sequential", + Self::Random => "random", + } + } +} + +fn benchmark_bloom( + args: &Args, + num_elements: u64, + lookup_count: u64, + false_positive_lookup_count: u64, + bloom_expected_items: u64, +) -> BenchmarkResult { + let mut insert_samples = Vec::with_capacity(args.repetitions); + let mut lookup_samples = Vec::with_capacity(args.repetitions); + let mut false_positive_rate_percent_samples = Vec::with_capacity(args.repetitions); + let mut bytes_used = 0; + let expected_items = + usize::try_from(bloom_expected_items).expect("bloom expected items must fit in usize"); + + for repetition in 0..args.repetitions { + let insert_permutation = AffinePermutation::for_order( + num_elements, + args.insert_order, + args.insert_seed.wrapping_add(repetition as u64), + ); + let lookup_permutation = AffinePermutation::for_order( + num_elements, + args.lookup_order, + args.lookup_seed.wrapping_add(repetition as u64), + ); + let false_positive_permutation = AffinePermutation::for_order( + U32_KEY_SPACE_SIZE - num_elements, + args.lookup_order, + args.lookup_seed.wrapping_add(repetition as u64), + ); + + let mut bloom = BloomFilter::with_false_pos(args.bloom_false_positive_rate) + .seed(&args.bloom_seed) + .expected_items(expected_items.max(MIN_BLOOM_EXPECTED_ITEMS as usize)); + + let insert_started = Instant::now(); + for index in 0..num_elements { + let key = insert_permutation.key_at(index); + bloom.insert(&key); + } + let insert_elapsed = insert_started.elapsed(); + + let lookup_started = Instant::now(); + let mut hits = 0u64; + for index in 0..lookup_count { + let key = lookup_permutation.key_at(index); + hits += u64::from(bloom.contains(&key)); + } + let lookup_elapsed = lookup_started.elapsed(); + + assert_eq!(hits, lookup_count, "expected all lookup keys to be present"); + + let mut false_positives = 0u64; + for index in 0..false_positive_lookup_count { + let key = false_positive_permutation.absent_key_at(index, num_elements); + false_positives += u64::from(bloom.contains(&key)); + } + + bytes_used = bloom.as_slice().len() * size_of::(); + insert_samples.push(insert_elapsed.as_nanos() as f64 / num_elements as f64); + lookup_samples.push(lookup_elapsed.as_nanos() as f64 / lookup_count as f64); + false_positive_rate_percent_samples + .push((false_positives as f64 / false_positive_lookup_count as f64) * 100.0); + } + + BenchmarkResult { + insert_ns_per_element: SummaryStats::from_samples(&insert_samples), + lookup_ns_per_element: SummaryStats::from_samples(&lookup_samples), + bytes_used, + false_positive_rate_percent: Some(SummaryStats::from_samples( + &false_positive_rate_percent_samples, + )), + } +} + +fn benchmark_roaring(args: &Args, num_elements: u64, lookup_count: u64) -> BenchmarkResult { + let mut insert_samples = Vec::with_capacity(args.repetitions); + let mut lookup_samples = Vec::with_capacity(args.repetitions); + let mut bytes_used = 0; + + for repetition in 0..args.repetitions { + let insert_permutation = AffinePermutation::for_order( + num_elements, + args.insert_order, + args.insert_seed.wrapping_add(repetition as u64), + ); + let lookup_permutation = AffinePermutation::for_order( + num_elements, + args.lookup_order, + args.lookup_seed.wrapping_add(repetition as u64), + ); + + let mut bitmap = RoaringBitmap::new(); + + let insert_started = Instant::now(); + for index in 0..num_elements { + let key = insert_permutation.key_at(index); + let _ = bitmap.insert(key); + } + let insert_elapsed = insert_started.elapsed(); + + let lookup_started = Instant::now(); + let mut hits = 0u64; + for index in 0..lookup_count { + let key = lookup_permutation.key_at(index); + hits += u64::from(bitmap.contains(key)); + } + let lookup_elapsed = lookup_started.elapsed(); + + assert_eq!(hits, lookup_count, "expected all lookup keys to be present"); + bytes_used = bitmap.serialized_size(); + insert_samples.push(insert_elapsed.as_nanos() as f64 / num_elements as f64); + lookup_samples.push(lookup_elapsed.as_nanos() as f64 / lookup_count as f64); + } + + BenchmarkResult { + insert_ns_per_element: SummaryStats::from_samples(&insert_samples), + lookup_ns_per_element: SummaryStats::from_samples(&lookup_samples), + bytes_used, + false_positive_rate_percent: None, + } +} + +fn print_report( + structure: Structure, + result: &BenchmarkResult, + num_elements: u64, + lookup_count: u64, + false_positive_lookup_count: u64, +) { + println!("structure={structure}"); + println!("num_elements={num_elements}"); + println!("bytes_used={}", result.bytes_used); + println!("bytes_used_human={}", format_bytes(result.bytes_used as f64)); + println!( + "bytes_per_element={}", + format_bytes(result.bytes_used as f64 / num_elements as f64) + ); + if structure == Structure::Bloom { + println!( + "bits_per_element={:.6}", + (result.bytes_used as f64 * 8.0) / num_elements as f64 + ); + } + print_stats("insert_ns_per_element", result.insert_ns_per_element); + print_stats("lookup_ns_per_element", result.lookup_ns_per_element); + println!("lookup_count={lookup_count}"); + if let Some(stats) = result.false_positive_rate_percent { + println!("false_positive_lookup_count={false_positive_lookup_count}"); + print_stats("false_positive_rate_percent", stats); + } + println!(); +} + +fn print_stats(label: &str, stats: SummaryStats) { + println!("{label}.min={:.6}", stats.min); + println!("{label}.avg={:.6}", stats.avg); + println!("{label}.max={:.6}", stats.max); + println!("{label}.stddev={:.6}", stats.stddev); +} + +fn format_bytes(bytes: f64) -> String { + const UNITS: [&str; 5] = ["B", "KiB", "MiB", "GiB", "TiB"]; + + let mut value = bytes; + let mut unit_index = 0; + while value >= 1024.0 && unit_index + 1 < UNITS.len() { + value /= 1024.0; + unit_index += 1; + } + + format!("{value:.6} {}", UNITS[unit_index]) +} diff --git a/scripts/plot_filter_bitmap.py b/scripts/plot_filter_bitmap.py new file mode 100644 index 0000000000..9e6ea5af39 --- /dev/null +++ b/scripts/plot_filter_bitmap.py @@ -0,0 +1,242 @@ +#!/usr/bin/env -S uv run --script +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "pandas>=2.2", +# "plotly>=5.24", +# "kaleido>=0.2.1", +# ] +# /// + +from __future__ import annotations + +import argparse +from pathlib import Path + +import pandas as pd +import plotly.graph_objects as go +from plotly.subplots import make_subplots + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Plot filter_bitmap.csv comparisons for bloom vs roaring." + ) + parser.add_argument( + "--input", + type=Path, + default=Path("filter_bitmap.csv"), + help="Input CSV produced by crates/dbsp/benches/filter_bitmap.rs", + ) + parser.add_argument( + "--output-dir", + type=Path, + default=Path("filter_bitmap_plots"), + help="Directory to write plots into", + ) + return parser.parse_args() + + +def format_structure(name: str) -> str: + return { + "bloom": "fastbloom", + "roaring": "roaring", + }.get(name, name) + + +def format_num_elements(value: int) -> str: + return f"{value:,}" + + +def format_bytes(value: float) -> str: + units = ["B", "KiB", "MiB", "GiB", "TiB"] + unit_index = 0 + while value >= 1024.0 and unit_index + 1 < len(units): + value /= 1024.0 + unit_index += 1 + return f"{value:.2f} {units[unit_index]}" + + +def format_ns_per_element(value: float) -> str: + return f"{value:.2f} ns" + + +def build_metric_figure( + frame: pd.DataFrame, + y_column: str, + y_label: str, + title: str, + formatter, +) -> go.Figure: + fig = go.Figure() + colors = { + "bloom": "#0f766e", + "roaring": "#c2410c", + } + + ordered_sizes = sorted(frame["num_elements"].unique()) + x_labels = [format_num_elements(value) for value in ordered_sizes] + + for structure in ["bloom", "roaring"]: + structure_frame = ( + frame[frame["structure"] == structure] + .sort_values("num_elements") + .set_index("num_elements") + ) + if structure_frame.empty: + continue + + y_values = [float(structure_frame.loc[size, y_column]) for size in ordered_sizes] + text_values = [formatter(value) for value in y_values] + fig.add_trace( + go.Bar( + name=format_structure(structure), + x=x_labels, + y=y_values, + text=text_values, + textposition="outside", + cliponaxis=False, + marker_color=colors[structure], + ) + ) + + fig.update_layout( + title=title, + barmode="group", + template="plotly_white", + width=max(950, 180 * len(ordered_sizes)), + height=650, + legend_title_text="Structure", + margin=dict(t=90, r=30, b=80, l=80), + ) + fig.update_xaxes(title_text="Input Size (num_elements)") + fig.update_yaxes(title_text=y_label, type="log") + return fig + + +def build_summary_figure(frame: pd.DataFrame) -> go.Figure: + fig = make_subplots( + rows=1, + cols=3, + subplot_titles=("Insert Time", "Lookup Time", "Memory Usage"), + horizontal_spacing=0.08, + ) + + colors = { + "bloom": "#0f766e", + "roaring": "#c2410c", + } + ordered_sizes = sorted(frame["num_elements"].unique()) + x_labels = [format_num_elements(value) for value in ordered_sizes] + metrics = [ + ("insert_ns_per_element_avg", format_ns_per_element, "Insert Time (ns/element)"), + ("lookup_ns_per_element_avg", format_ns_per_element, "Lookup Time (ns/element)"), + ("bytes_used", format_bytes, "Memory Usage (bytes)"), + ] + + for col_index, (metric, formatter, y_label) in enumerate(metrics, start=1): + for structure in ["bloom", "roaring"]: + structure_frame = ( + frame[frame["structure"] == structure] + .sort_values("num_elements") + .set_index("num_elements") + ) + if structure_frame.empty: + continue + + y_values = [float(structure_frame.loc[size, metric]) for size in ordered_sizes] + text_values = [formatter(value) for value in y_values] + fig.add_trace( + go.Bar( + name=format_structure(structure), + x=x_labels, + y=y_values, + text=text_values, + textposition="outside", + cliponaxis=False, + marker_color=colors[structure], + showlegend=col_index == 1, + ), + row=1, + col=col_index, + ) + fig.update_yaxes(title_text=y_label, type="log", row=1, col=col_index) + fig.update_xaxes(title_text="Input Size (num_elements)", row=1, col=col_index) + + fig.update_layout( + title="filter_bitmap Summary", + barmode="group", + template="plotly_white", + width=max(1600, 420 * len(ordered_sizes)), + height=700, + legend_title_text="Structure", + margin=dict(t=100, r=30, b=90, l=70), + ) + return fig + + +def write_figure(fig: go.Figure, base_path: Path) -> None: + fig.write_html(base_path.with_suffix(".html")) + fig.write_image(base_path.with_suffix(".png"), scale=2) + + +def main() -> None: + args = parse_args() + if not args.input.exists(): + raise SystemExit(f"input CSV not found: {args.input}") + + frame = pd.read_csv(args.input) + if frame.empty: + raise SystemExit(f"input CSV is empty: {args.input}") + + required_columns = { + "structure", + "num_elements", + "insert_ns_per_element_avg", + "lookup_ns_per_element_avg", + "bytes_used", + } + missing_columns = sorted(required_columns - set(frame.columns)) + if missing_columns: + raise SystemExit( + f"input CSV is missing required columns: {', '.join(missing_columns)}" + ) + + frame = frame.copy().sort_values(["num_elements", "structure"]) + args.output_dir.mkdir(parents=True, exist_ok=True) + + insert_figure = build_metric_figure( + frame, + "insert_ns_per_element_avg", + "Insert Time (ns/element)", + "filter_bitmap: Insert Time", + format_ns_per_element, + ) + write_figure(insert_figure, args.output_dir / "filter_bitmap_insert_ns_per_element") + + lookup_figure = build_metric_figure( + frame, + "lookup_ns_per_element_avg", + "Lookup Time (ns/element)", + "filter_bitmap: Lookup Time", + format_ns_per_element, + ) + write_figure(lookup_figure, args.output_dir / "filter_bitmap_lookup_ns_per_element") + + memory_figure = build_metric_figure( + frame, + "bytes_used", + "Memory Usage (bytes)", + "filter_bitmap: Memory Usage", + format_bytes, + ) + write_figure(memory_figure, args.output_dir / "filter_bitmap_memory_bytes") + + summary_figure = build_summary_figure(frame) + write_figure(summary_figure, args.output_dir / "filter_bitmap_summary") + + print(f"wrote plots to {args.output_dir}") + + +if __name__ == "__main__": + main()