//! # Dynamic Rate Limit Distribution System
//!
//! This module implements a sophisticated rate limiting system that dynamically
//! redistributes unused capacity to high-demand channels, maximizing overall
//! system utilization while maintaining fairness and guaranteed minimums.
//!
//! ## Design Philosophy`Preformatted text`
//!
//! Traditional rate limiters waste capacity when some channels are underutilized.
//! This system addresses that inefficiency by continuously monitoring usage patterns
//! and reallocating unused capacity to channels that need it most.
//!
//! ## Key Concepts
//!
//! - **Base Limits**: Guaranteed minimum rate limits that cannot be taken away
//! - **Bonus Capacity**: Additional capacity allocated from unused portions
//! - **Usage Tracking**: Historical usage patterns inform redistribution decisions
//! - **Demand Scoring**: Channels are prioritized based on their usage patterns
//!
//! ## Algorithm Overview
//!
//! 1. Track usage patterns over a sliding window
//! 2. Identify high-demand channels (>80% usage) and low-demand channels (<50% usage)
//! 3. Calculate available bonus capacity from system headroom and underutilized channels
//! 4. Redistribute bonus capacity to high-demand channels proportionally
//! 5. Repeat periodically to adapt to changing traffic patterns
use std::collections::{HashMap, BinaryHeap};
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use std::cmp::Ordering;
use tokio::time::interval;
use tokio::task;
/// Represents a single usage event in the system.
///
/// We track individual events rather than aggregated counts to enable
/// fine-grained analysis of usage patterns over time. This allows us to
/// accurately calculate rates over arbitrary time windows.
#[derive(Debug, Clone)]
struct UsageEvent {
/// When this usage occurred. Using Instant for monotonic time guarantees.
timestamp: Instant,
/// Number of requests in this event. Usually 1, but batching is supported.
count: u32,
}
/// Comprehensive statistics for a single channel.
///
/// This struct maintains both real-time usage counts and historical data,
/// enabling both immediate rate limiting decisions and long-term pattern
analysis.
#[derive(Debug, Default)]
struct ChannelStats {
/// Historical usage events within the tracking window.
///
/// We maintain a vector rather than a ring buffer because:
/// 1. We need to scan all events for rate calculations anyway
/// 2. The window is typically small (5 minutes = ~300 events at 1 req/sec)
/// 3. Cleanup happens during normal operations, preventing unbounded growth
usage_history: Vec<UsageEvent>,
/// Current usage count within the rate limit period.
///
/// This is reset at each redistribution interval to ensure accurate
/// rate limiting. It represents "consumed" capacity, not historical usage.
current_usage: u32,
}
/// Priority score for demand-based capacity allocation.
///
/// The scoring algorithm considers both usage rate and absolute capacity
/// to ensure fair distribution. High-capacity channels with high usage
/// get priority, but smaller channels aren't starved.
#[derive(Debug)]
struct DemandScore {
/// Composite score: usage_rate * base_capacity.
///
/// This formula ensures that:
/// - A 500 req/s channel at 90% usage (450 req/s actual) gets more bonus than
/// - A 100 req/s channel at 90% usage (90 req/s actual)
///
/// This reflects the absolute impact of additional capacity.
score: f64,
/// The channel identifier for reverse lookup after heap operations.
channel_id: String,
}
// Implement comparison traits for BinaryHeap (max heap by default).
// We use total ordering with a fallback to Equal for NaN cases,
// though our algorithm should never produce NaN values.
impl Eq for DemandScore {}
impl PartialEq for DemandScore {
fn eq(&self, other: &Self) -> bool {
self.score == other.score
}
}
impl Ord for DemandScore {
fn cmp(&self, other: &Self) -> Ordering {
// Handle potential NaN values by treating them as equal
self.score.partial_cmp(&other.score).unwrap_or(Ordering::Equal)
}
}
impl PartialOrd for DemandScore {
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
self.score.partial_cmp(&other.score)
}
}
/// Configuration parameters for the rate limit distributor.
///
/// These parameters control the behavior of the redistribution algorithm
/// and can be tuned based on your specific use case and traffic patterns.
#[derive(Debug, Clone)]
pub struct DistributorConfig {
/// Total system capacity across all channels.
///
/// This should be set based on your infrastructure limits (e.g., database
/// connections, API quota, bandwidth). It must be >= sum of all base limits
/// to ensure base guarantees can be met.
pub total_capacity: u32,
/// How often to recalculate and redistribute capacity.
///
/// Shorter intervals mean faster adaptation to traffic changes but higher
/// CPU usage. Typical values: 10-60 seconds. Too short (<5s) may cause
/// thrashing; too long (>5min) reduces responsiveness.
pub redistribution_interval: Duration,
/// Time window for usage pattern analysis.
///
/// This should be several times longer than redistribution_interval to
/// smooth out short-term spikes. Typical value: 5 minutes. Longer windows
/// provide more stable patterns but slower adaptation.
pub usage_window: Duration,
/// Usage rate threshold to classify a channel as "high demand".
///
/// Channels using more than this percentage of their limit are eligible
/// for bonus capacity. Default: 0.8 (80%). Lower values distribute bonus
/// more liberally; higher values reserve it for truly constrained channels.
pub high_usage_threshold: f64,
/// Usage rate threshold to classify a channel as "low demand".
///
/// Channels using less than this percentage may have capacity reclaimed
/// for redistribution. Default: 0.5 (50%). This prevents penalizing
/// channels with bursty traffic patterns.
pub low_usage_threshold: f64,
/// Maximum bonus capacity as a ratio of base limit.
///
/// Prevents any single channel from hoarding capacity. Default: 0.5 (50%).
/// A channel with 100 req/s base can get at most 50 req/s bonus.
/// This ensures fairness and prevents starvation.
pub max_bonus_ratio: f64,
}
impl Default for DistributorConfig {
fn default() -> Self {
Self {
total_capacity: 1000,
redistribution_interval: Duration::from_secs(60),
usage_window: Duration::from_secs(300),
high_usage_threshold: 0.8,
low_usage_threshold: 0.5,
max_bonus_ratio: 0.5,
}
}
}
/// The core rate limit distributor that manages dynamic capacity allocation.
///
/// This struct orchestrates the entire system, tracking usage patterns,
/// calculating optimal distributions, and coordinating with rate limiters.
/// It's designed to be shared across threads using Arc.
pub struct RateLimitDistributor {
/// Immutable base limits that serve as guaranteed minimums.
///
/// These are never modified after initialization, providing a stable
/// foundation for capacity planning. Channels always get at least this much.
base_limits: HashMap<String, u32>,
/// Current effective limits including base + bonus allocations.
///
/// Protected by a Mutex for thread-safe updates. Read-heavy workloads
/// might benefit from RwLock, but redistribution is infrequent enough
/// that Mutex is simpler and sufficient.
current_limits: Arc<Mutex<HashMap<String, u32>>>,
/// Per-channel usage statistics and history.
///
/// Tracks both real-time usage for rate limiting and historical patterns
/// for redistribution decisions. Mutex-protected for concurrent access.
channel_stats: Arc<Mutex<HashMap<String, ChannelStats>>>,
/// Configuration parameters for the distribution algorithm.
config: DistributorConfig,
/// System start time for uptime tracking and diagnostics.
start_time: Instant,
}
impl RateLimitDistributor {
/// Creates a new rate limit distributor with specified base limits and configuration.
///
/// # Arguments
///
/// * `base_limits` - HashMap of channel_id -> base rate limit
/// * `config` - Configuration parameters for the distribution algorithm
///
/// # Panics
///
/// Panics if total_capacity < sum of base_limits, as this would make it
/// impossible to guarantee base limits for all channels.
pub fn new(base_limits: HashMap<String, u32>, config: DistributorConfig) -> Self {
// Validate configuration
let total_base: u32 = base_limits.values().sum();
assert!(
config.total_capacity >= total_base,
"Total capacity ({}) must be >= sum of base limits ({})",
config.total_capacity, total_base
);
let current_limits = Arc::new(Mutex::new(base_limits.clone()));
let channel_stats = Arc::new(Mutex::new(HashMap::new()));
Self {
base_limits,
current_limits,
channel_stats,
config,
start_time: Instant::now(),
}
}
/// Starts the background redistribution task.
///
/// This spawns a Tokio task that periodically recalculates and redistributes
/// capacity based on observed usage patterns. The task runs indefinitely
/// until the distributor is dropped.
///
/// # Why a separate task?
///
/// Running redistribution in a separate task decouples it from request
/// processing, preventing redistribution latency from affecting request paths.
/// It also allows redistribution to happen even during quiet periods.
pub fn start(self: Arc<Self>) {
let distributor = Arc::clone(&self);
tokio::spawn(async move {
let mut interval = interval(distributor.config.redistribution_interval);
// Skip the immediate first tick to allow initial usage data to accumulate
interval.tick().await;
loop {
interval.tick().await;
distributor.redistribute_capacity();
}
});
}
/// Checks if a request from a channel is within the current rate limit.
///
/// This is the hot path for request processing and is optimized for:
/// 1. Minimal lock contention (single mutex acquisition)
/// 2. Fast path for allowed requests
/// 3. Automatic usage tracking for redistribution decisions
///
/// # Returns
///
/// * `Ok((true, limit))` - Request allowed, with current limit
/// * `Ok((false, limit))` - Request denied due to rate limit
/// * `Err(msg)` - Channel not found or lock poisoned
pub fn check_rate_limit(&self, channel_id: &str) -> Result<(bool, u32), String> {
// We need to lock both maps atomically to ensure consistency
// between limit checking and stats updating
let mut limits = self.current_limits.lock()
.map_err(|e| format!("Failed to acquire limits lock: {}", e))?;
let mut stats = self.channel_stats.lock()
.map_err(|e| format!("Failed to acquire stats lock: {}", e))?;
// Fail fast for unknown channels rather than auto-creating them
// This prevents typos or attacks from creating unlimited channels
let current_limit = *limits.get(channel_id)
.ok_or_else(|| format!("Unknown channel: {}", channel_id))?;
// Get or create stats for this channel
// We create on demand to avoid pre-allocating for unused channels
let channel_stat = stats.entry(channel_id.to_string())
.or_insert_with(ChannelStats::default);
if channel_stat.current_usage < current_limit {
// Request allowed - update stats
channel_stat.current_usage += 1;
channel_stat.usage_history.push(UsageEvent {
timestamp: Instant::now(),
count: 1,
});
// Opportunistic cleanup of old events to prevent unbounded growth
// We do this here rather than in a separate task to avoid additional
// lock contention and ensure cleanup happens proportionally to usage
let cutoff = Instant::now() - self.config.usage_window;
channel_stat.usage_history.retain(|event| event.timestamp > cutoff);
Ok((true, current_limit))
} else {
// Request denied - still record the attempt for accurate demand tracking
// This ensures high-demand channels are recognized even when rate-limited
Ok((false, current_limit))
}
}
/// Calculates the recent usage rate for a channel.
///
/// Uses a 60-second window for rate calculation rather than the full
/// usage_window to provide more responsive measurements. This shorter
/// window better reflects current demand while the longer usage_window
/// provides historical context for pattern analysis.
///
/// # Why 60 seconds?
///
/// This balances responsiveness with stability. Shorter windows (e.g., 10s)
/// would be too noisy; longer windows (e.g., 5min) would be too slow to
/// detect traffic changes.
fn calculate_usage_rate(&self, channel_id: &str) -> Result<f64, String> {
let limits = self.current_limits.lock()
.map_err(|e| format!("Failed to acquire limits lock: {}", e))?;
let stats = self.channel_stats.lock()
.map_err(|e| format!("Failed to acquire stats lock: {}", e))?;
// Guard against divide-by-zero and missing channels
let current_limit = match limits.get(channel_id) {
Some(&limit) if limit > 0 => limit,
_ => return Ok(0.0),
};
let channel_stat = match stats.get(channel_id) {
Some(stat) => stat,
None => return Ok(0.0), // No usage history means 0% usage
};
// Count recent usage with early termination for efficiency
let cutoff = Instant::now() - Duration::from_secs(60);
let recent_usage: u32 = channel_stat.usage_history
.iter()
.rev() // Most recent events first for early termination
.take_while(|event| event.timestamp > cutoff)
.map(|event| event.count)
.sum();
Ok(recent_usage as f64 / current_limit as f64)
}
/// Redistributes unused capacity to high-demand channels.
///
/// This is the core of the dynamic allocation algorithm. It runs
/// periodically and performs these steps:
///
/// 1. Reset usage counters (start fresh each period)
/// 2. Calculate usage rates for all channels
/// 3. Identify high-demand and low-demand channels
/// 4. Calculate total redistributable capacity
/// 5. Allocate extra capacity to high-demand channels
/// 6. Update effective limits
///
/// The algorithm is designed to be stable (small traffic changes produce
/// small allocation changes) and fair (capacity is distributed proportionally).
fn redistribute_capacity(&self) {
// Use a separate method for the actual logic to enable proper error handling
// without panicking in the background task
if let Err(e) = self.try_redistribute_capacity() {
eprintln!("Failed to redistribute capacity: {}", e);
// Continue running despite errors - better to keep current limits
// than to crash the redistribution task
}
}
/// Internal implementation of redistribute_capacity with proper error handling.
///
/// Separated from the public method to enable testing and error propagation
/// while keeping the public API simple.
fn try_redistribute_capacity(&self) -> Result<(), String> {
// Acquire locks once for the entire redistribution operation
// This ensures consistency but may block requests briefly
let mut limits = self.current_limits.lock()
.map_err(|e| format!("Failed to acquire limits lock: {}", e))?;
let mut stats = self.channel_stats.lock()
.map_err(|e| format!("Failed to acquire stats lock: {}", e))?;
// Reset usage counters for the next period
// This ensures rate limits are enforced per-period, not cumulatively
for stat in stats.values_mut() {
stat.current_usage = 0;
}
// Phase 1: Analyze current usage patterns
let mut usage_rates = HashMap::new();
let mut demand_heap = BinaryHeap::new();
for channel_id in self.base_limits.keys() {
let usage_rate = self.calculate_usage_rate(channel_id)?;
usage_rates.insert(channel_id.clone(), usage_rate);
// Identify high-demand channels that could benefit from extra capacity
if usage_rate > self.config.high_usage_threshold {
let base_limit = self.base_limits[channel_id];
// Score reflects absolute demand, not just percentage
// This prevents tiny channels from dominating redistribution
let score = usage_rate * base_limit as f64;
demand_heap.push(DemandScore {
score,
channel_id: channel_id.clone(),
});
}
}
// Phase 2: Calculate available capacity for redistribution
// Start with system headroom (total - sum of base)
let total_base: u32 = self.base_limits.values().sum();
let mut redistributable =
self.config.total_capacity.saturating_sub(total_base);
// Add capacity reclaimed from underutilized channels
// We only reclaim the portion they're definitely not using
for (channel_id, &usage_rate) in &usage_rates {
if usage_rate < self.config.low_usage_threshold {
let base_limit = self.base_limits[channel_id];
// Only reclaim the unused portion below the threshold
// E.g., if threshold is 50% and usage is 30%, reclaim 20% of base
let reclaimable = (base_limit as f64 *
(self.config.low_usage_threshold - usage_rate)) as u32;
redistributable += reclaimable;
}
}
// Phase 3: Reset to base limits before redistribution
// This ensures we start fresh and don't accumulate bonuses
*limits = self.base_limits.clone();
// Phase 4: Distribute extra capacity to high-demand channels
// Using a heap ensures we prioritize the neediest channels
while redistributable > 0 && !demand_heap.is_empty() {
if let Some(demand) = demand_heap.pop() {
let base_limit = self.base_limits[&demand.channel_id];
// Cap bonus to prevent any channel from hoarding capacity
let max_bonus = (base_limit as f64 * self.config.max_bonus_ratio) as u32;
let bonus = max_bonus.min(redistributable);
// Update the limit for this channel
if let Some(limit) = limits.get_mut(&demand.channel_id) {
*limit += bonus;
redistributable = redistributable.saturating_sub(bonus);
}
}
}
// Log the redistribution results for monitoring
self.log_redistribution(&limits, &usage_rates);
Ok(())
}
/// Logs the current distribution state for monitoring and debugging.
///
/// In production, this should be integrated with your metrics system
/// (Prometheus, StatsD, etc.) rather than printing to stdout.
fn log_redistribution(&self,
current_limits: &HashMap<String, u32>,
usage_rates: &HashMap<String, f64>) {
println!("\n=== Rate Limit Redistribution ===");
println!("Timestamp: {:?}", SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs());
println!("Uptime: {:?}", self.start_time.elapsed());
// Sort channels for consistent output
let mut channels: Vec<_> = self.base_limits.keys().collect();
channels.sort();
for channel_id in channels {
let base = self.base_limits[channel_id];
let current = current_limits.get(channel_id).copied().unwrap_or(base);
let bonus = current.saturating_sub(base);
let usage_rate = usage_rates.get(channel_id).copied().unwrap_or(0.0);
println!("Channel {}: Base={}, Current={}, Bonus={}, Usage={:.1}%",
channel_id, base, current, bonus, usage_rate * 100.0);
}
}
}
/// Token bucket implementation with dynamic capacity adjustment.
///
/// Token buckets provide smooth rate limiting by accumulating tokens over time
/// and consuming them for requests. This implementation adds dynamic capacity
/// adjustment to work with the redistribution system.
///
/// # Why token buckets?
///
/// Token buckets allow burst capacity while maintaining average rate limits.
/// This is more user-friendly than hard per-second limits and works well
/// with real-world traffic patterns that tend to be bursty.
pub struct TokenBucket {
/// Current token count (can be fractional for smooth refill).
tokens: f64,
/// Last refill timestamp for calculating accumulated tokens.
last_refill: Instant,
/// Current capacity (dynamically adjusted by the distributor).
capacity: u32,
}
impl TokenBucket {
/// Creates a new token bucket with the specified capacity.
///
/// Starts full to allow immediate requests without waiting for refill.
pub fn new(capacity: u32) -> Self {
Self {
tokens: capacity as f64,
last_refill: Instant::now(),
capacity,
}
}
/// Attempts to consume tokens from the bucket.
///
/// # Algorithm
///
/// 1. Calculate elapsed time since last refill
/// 2. Add accumulated tokens based on refill rate
/// 3. Cap tokens at current capacity
/// 4. Try to consume requested tokens
/// 5. Update capacity if changed by distributor
///
/// This provides smooth rate limiting with burst capacity up to the
/// full bucket size.
pub fn try_consume(&mut self, tokens: f64, new_capacity: u32) -> bool {
let now = Instant::now();
let elapsed = now.duration_since(self.last_refill).as_secs_f64();
// Update capacity if changed by the distributor
// This allows dynamic adjustment without resetting the bucket
if new_capacity != self.capacity {
// Scale existing tokens proportionally to prevent gaming
// the system by requesting right after capacity increases
let scale = new_capacity as f64 / self.capacity as f64;
self.tokens *= scale;
self.capacity = new_capacity;
}
// Refill tokens based on time elapsed
// We use capacity/60 as the per-second rate to align with
// typical rate limit specifications (requests per minute)
let refill_rate = self.capacity as f64 / 60.0;
self.tokens += elapsed * refill_rate;
// Cap at capacity to prevent unlimited accumulation
self.tokens = self.tokens.min(self.capacity as f64);
self.last_refill = now;
// Try to consume the requested tokens
if self.tokens >= tokens {
self.tokens -= tokens;
true
} else {
false
}
}
}
/// High-level rate limiter that combines token buckets with dynamic limits.
///
/// This is the main interface for request processing. It coordinates between
/// the distributor (which sets limits) and token buckets (which enforce them).
pub struct TokenBucketRateLimiter {
/// Reference to the distributor for limit updates and usage tracking.
distributor: Arc<RateLimitDistributor>,
/// Per-channel token buckets for smooth rate limiting.
///
/// We use a separate lock from the distributor to minimize contention
/// between request processing and redistribution.
buckets: Arc<Mutex<HashMap<String, TokenBucket>>>,
}
impl TokenBucketRateLimiter {
/// Creates a new rate limiter backed by the specified distributor.
pub fn new(distributor: Arc<RateLimitDistributor>) -> Self {
Self {
distributor,
buckets: Arc::new(Mutex::new(HashMap::new())),
}
}
/// Checks if a request from the specified channel should be allowed.
///
/// This is the main entry point for request processing. It:
/// 1. Checks with the distributor for current limits and usage tracking
/// 2. Applies token bucket smoothing for burst tolerance
/// 3. Returns the final allow/deny decision
///
/// # Why two levels of rate limiting?
///
/// The distributor tracks hard limits and usage patterns, while token
/// buckets provide smooth enforcement with burst tolerance. This combination
/// gives us both accurate long-term limits and user-friendly behavior.
pub fn allow_request(&self, channel_id: &str) -> Result<bool, String> {
// First check with distributor for limits and usage tracking
let (allowed, current_limit) = self.distributor.check_rate_limit(channel_id)?;
// If distributor says no, respect that decision
if !allowed {
return Ok(false);
}
// Apply token bucket for smooth rate limiting
let mut buckets = self.buckets.lock()
.map_err(|e| format!("Failed to acquire buckets lock: {}", e))?;
// Create bucket on demand to avoid pre-allocating for all possible channels
let bucket = buckets.entry(channel_id.to_string())
.or_insert_with(|| TokenBucket::new(current_limit));
// Try to consume one token (could be parameterized for batch requests)
Ok(bucket.try_consume(1.0, current_limit))
}
}
/// Example usage and traffic simulation for testing
#[cfg(test)]
mod tests {
use super::*;
use rand::Rng;
/// Simulates realistic traffic patterns to demonstrate the system behavior.
///
/// This test creates channels with different usage patterns and shows how
/// the system redistributes capacity from low-usage to high-usage channels.
#[tokio::test]
async fn test_traffic_simulation() {
// Define base limits representing different service tiers
let mut base_limits = HashMap::new();
base_limits.insert("customer_A".to_string(), 100); // Basic tier
base_limits.insert("customer_B".to_string(), 200); // Standard tier
base_limits.insert("customer_C".to_string(), 500); // Premium tier
base_limits.insert("customer_D".to_string(), 100); // Basic tier
base_limits.insert("customer_E".to_string(), 200); // Standard tier
// Configure distributor with aggressive settings for testing
let config = DistributorConfig {
total_capacity: 1500, // 36% headroom over base total of 1100
redistribution_interval: Duration::from_secs(5), // Fast for testing
usage_window: Duration::from_secs(60),
high_usage_threshold: 0.8,
low_usage_threshold: 0.5,
max_bonus_ratio: 0.5,
};
// Create and start distributor
let distributor = Arc::new(RateLimitDistributor::new(base_limits, config));
distributor.clone().start();
// Create rate limiter
let limiter = TokenBucketRateLimiter::new(distributor);
// Define usage patterns to simulate different customer behaviors
let usage_patterns = vec![
("customer_A", 0.3), // Low usage - likely won't use full capacity
("customer_B", 0.95), // High usage - needs more than base allows
("customer_C", 0.9), // High usage - premium customer under pressure
("customer_D", 0.1), // Very low usage - capacity can be reclaimed
("customer_E", 0.5), // Medium usage - right at the threshold
];
// Run simulation
println!("Starting traffic simulation...");
let mut rng = rand::thread_rng();
let start = Instant::now();
// Track metrics for analysis
let mut request_counts = HashMap::new();
let mut allowed_counts = HashMap::new();
// Simulate 15 seconds of traffic (3 redistribution cycles)
while start.elapsed() < Duration::from_secs(15) {
// Generate requests based on probability patterns
for (customer, probability) in &usage_patterns {
if rng.gen::<f64>() < *probability {
*request_counts.entry(*customer).or_insert(0) += 1;
if limiter.allow_request(customer).unwrap_or(false) {
*allowed_counts.entry(*customer).or_insert(0) += 1;
}
}
}
// Small delay to simulate realistic request spacing
tokio::time::sleep(Duration::from_millis(10)).await;
}
// Print final statistics to verify redistribution worked
println!("\n\n=== Final Statistics ===");
let mut customers: Vec<_> = request_counts.keys().collect();
customers.sort();
for customer in customers {
let total = request_counts[customer];
let allowed = allowed_counts.get(customer).copied().unwrap_or(0);
let rejected = total - allowed;
let success_rate = if total > 0 {
allowed as f64 / total as f64 * 100.0
} else {
0.0
};
println!("{}: Requests={}, Allowed={}, Rejected={}, Success Rate={:.1}%",
customer, total, allowed, rejected, success_rate);
// Verify high-usage customers got bonus capacity
if customer == &"customer_B" || customer == &"customer_C" {
assert!(success_rate > 85.0,
"High-usage {} should have >85% success rate", customer);
}
}
}
}
// Cargo.toml dependencies:
/*
[dependencies]
tokio = { version = "1.35", features = ["full"] }
rand = "0.8"
[dev-dependencies]
tokio-test = "0.4"
*/`Preformatted text`
1 Like
That’s more readable for me.
What’s your opinion on the inline docs and under-lining theory?