Metrics

Prometheus-compatible metrics for production monitoring.

Setup

# Cargo.toml
[dependencies]
metrics = "0.22"
metrics-exporter-prometheus = "0.13"

Initialize Prometheus Exporter

use metrics_exporter_prometheus::PrometheusBuilder;

fn init_metrics() -> Result<()> {
    PrometheusBuilder::new()
        .with_http_listener(([0, 0, 0, 0], 9090))
        .install()?;

    Ok(())
}

// Access metrics at http://localhost:9090/metrics

Counter (Monotonically Increasing)

use metrics::counter;

// Increment by 1
counter!("videos_processed_total").increment(1);

// With labels
counter!("http_requests_total", "method" => "GET", "status" => "200").increment(1);

// In a function
fn record_request(method: &str, status: u16) {
    counter!(
        "http_requests_total",
        "method" => method.to_string(),
        "status" => status.to_string()
    ).increment(1);
}

Gauge (Can Go Up or Down)

use metrics::gauge;

// Set absolute value
gauge!("gpu_vram_used_gb").set(8.5);

// Increment/decrement
gauge!("active_connections").increment(1.0);
gauge!("active_connections").decrement(1.0);

// With labels
gauge!("queue_depth", "queue" => "embedding").set(42.0);

Histogram (Distribution of Values)

use metrics::histogram;

// Record a value
histogram!("request_duration_seconds").record(0.125);

// With labels
histogram!(
    "processing_duration_seconds",
    "model" => "whisper",
    "phase" => "inference"
).record(duration.as_secs_f64());

// Record timing
let start = std::time::Instant::now();
do_work();
histogram!("operation_duration_seconds").record(start.elapsed().as_secs_f64());

Describe Metrics

use metrics::{describe_counter, describe_gauge, describe_histogram, Unit};

fn describe_metrics() {
    describe_counter!(
        "videos_processed_total",
        Unit::Count,
        "Total number of videos processed"
    );

    describe_gauge!(
        "gpu_vram_used_gb",
        Unit::Gigabytes,
        "Current GPU VRAM usage"
    );

    describe_histogram!(
        "request_duration_seconds",
        Unit::Seconds,
        "Request processing time distribution"
    );
}

Metrics Struct Pattern

use metrics::{counter, gauge, histogram};

pub struct ProcessorMetrics;

impl ProcessorMetrics {
    pub fn record_video_processed(model: &str) {
        counter!("videos_processed_total", "model" => model.to_string()).increment(1);
    }

    pub fn record_processing_time(model: &str, duration: std::time::Duration) {
        histogram!(
            "processing_duration_seconds",
            "model" => model.to_string()
        ).record(duration.as_secs_f64());
    }

    pub fn set_queue_depth(queue: &str, depth: usize) {
        gauge!("queue_depth", "queue" => queue.to_string()).set(depth as f64);
    }

    pub fn record_error(error_type: &str) {
        counter!("processing_errors_total", "type" => error_type.to_string()).increment(1);
    }
}

// Usage
ProcessorMetrics::record_video_processed("whisper");
ProcessorMetrics::record_processing_time("whisper", elapsed);

Scheduler Metrics Example

pub struct SchedulerMetrics {
    model_switches: AtomicU64,
    items_processed: DashMap<ModelType, u64>,
    processing_times: DashMap<ModelType, f64>,
}

impl SchedulerMetrics {
    pub fn new() -> Self {
        Self {
            model_switches: AtomicU64::new(0),
            items_processed: DashMap::new(),
            processing_times: DashMap::new(),
        }
    }

    pub fn record_model_switch(&self, from: Option<ModelType>, to: ModelType) {
        self.model_switches.fetch_add(1, Ordering::Relaxed);
        counter!(
            "model_switches_total",
            "to" => format!("{:?}", to)
        ).increment(1);
    }

    pub fn record_batch_processed(&self, model: ModelType, count: u64, duration: Duration) {
        *self.items_processed.entry(model).or_insert(0) += count;

        counter!(
            "items_processed_total",
            "model" => format!("{:?}", model)
        ).increment(count);

        histogram!(
            "batch_processing_seconds",
            "model" => format!("{:?}", model)
        ).record(duration.as_secs_f64());
    }

    pub fn export(&self) -> MetricsSnapshot {
        MetricsSnapshot {
            model_switches: self.model_switches.load(Ordering::Relaxed),
            items_processed: self.items_processed.iter()
                .map(|e| (*e.key(), *e.value()))
                .collect(),
        }
    }
}

HTTP Endpoint Integration

use axum::{routing::get, Router};
use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle};

fn setup_metrics() -> PrometheusHandle {
    PrometheusBuilder::new()
        .set_buckets_for_metric(
            Matcher::Prefix("http_request".to_string()),
            &[0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0],
        )
        .unwrap()
        .install_recorder()
        .unwrap()
}

async fn metrics_handler(handle: axum::Extension<PrometheusHandle>) -> String {
    handle.render()
}

#[tokio::main]
async fn main() {
    let handle = setup_metrics();

    let app = Router::new()
        .route("/metrics", get(metrics_handler))
        .layer(axum::Extension(handle));

    axum::Server::bind(&"0.0.0.0:8080".parse().unwrap())
        .serve(app.into_make_service())
        .await
        .unwrap();
}

Common Metric Patterns

// Request duration with status
fn record_request(method: &str, path: &str, status: u16, duration: Duration) {
    let labels = [
        ("method", method.to_string()),
        ("path", path.to_string()),
        ("status", status.to_string()),
    ];

    counter!("http_requests_total", &labels).increment(1);
    histogram!("http_request_duration_seconds", &labels).record(duration.as_secs_f64());
}

// Error rate tracking
fn record_operation(success: bool, operation: &str) {
    let status = if success { "success" } else { "error" };
    counter!(
        "operations_total",
        "operation" => operation.to_string(),
        "status" => status.to_string()
    ).increment(1);
}

// Resource utilization
fn update_resource_metrics(cpu: f64, memory: f64, disk: f64) {
    gauge!("cpu_usage_percent").set(cpu);
    gauge!("memory_usage_percent").set(memory);
    gauge!("disk_usage_percent").set(disk);
}

Guidelines

• Use counters for things that only increase (requests, errors)
• Use gauges for things that go up and down (connections, memory)
• Use histograms for latency and size distributions
• Include relevant labels but avoid high cardinality
• Describe metrics with units and descriptions
• Expose metrics on standard port (9090 or /metrics endpoint)
• Use consistent naming: <namespace>_<name>_<unit>

Examples

See hercules-local-algo/src/scheduler/metrics.rs for production metrics.