etl_core/checkpoint/mod.rs
1//! Checkpointing: acknowledgement tracking and watermark commits.
2//!
3//! One [`AckRef`] is created per source poll batch (by an [`AckIssuer`] on
4//! the polling thread) and cloned into every record derived from that batch
5//! — filter drops, `flat_map` fan-out, and multi-sink routing all compose
6//! through plain `Clone`/`Drop`. When the last clone drops, the batch
7//! resolves with the worst status observed and a message flows to the
8//! [`Checkpointer`], which advances per-partition contiguous watermarks
9//! ([`PartitionTracker`]) and hands `Source::commit` its positions on an
10//! interval.
11//!
12//! Invariants (see `docs/DESIGN.md` and `CLAUDE.md`):
13//! - the tracker stays synchronous and tokio-free (loom-tested);
14//! - the ack path never blocks (unbounded channels, atomics only);
15//! - a watermark never advances past an unacknowledged or failed batch,
16//! including across rebalances (assignment epochs make stale
17//! acknowledgements harmless).
18//!
19//! The concurrency-bearing primitives are model-checked with
20//! [loom](https://docs.rs/loom):
21//!
22//! ```text
23//! RUSTFLAGS="--cfg loom" cargo test -p etl-core --release checkpoint::loom_tests
24//! ```
25
26mod ack;
27#[cfg(not(loom))]
28mod checkpointer;
29#[cfg(all(test, loom))]
30mod loom_tests;
31mod sync;
32mod tracker;
33
34pub use ack::{AckMsg, AckRef, AckSet, AckStatus, BatchId};
35#[cfg(not(loom))]
36pub use checkpointer::{AckIssuer, Checkpointer, DrainStats};
37pub use tracker::{PartitionTracker, ResolveOutcome};