openzeppelin_relayer/metrics/
mod.rs

1//! Metrics module for the application.
2//!
3//! - This module contains the global Prometheus registry.
4//! - Defines specific metrics for the application.
5
6pub mod middleware;
7use lazy_static::lazy_static;
8use prometheus::{
9    CounterVec, Encoder, Gauge, GaugeVec, HistogramOpts, HistogramVec, Opts, Registry, TextEncoder,
10};
11use sysinfo::{Disks, System};
12
13lazy_static! {
14    // Global Prometheus registry.
15    pub static ref REGISTRY: Registry = Registry::new();
16
17    // Counter: Total HTTP requests.
18    pub static ref REQUEST_COUNTER: CounterVec = {
19        let opts = Opts::new("requests_total", "Total number of HTTP requests");
20        let counter_vec = CounterVec::new(opts, &["endpoint", "method", "status"]).unwrap();
21        REGISTRY.register(Box::new(counter_vec.clone())).unwrap();
22        counter_vec
23    };
24
25    // Counter: Total HTTP requests by raw URI.
26    pub static ref RAW_REQUEST_COUNTER: CounterVec = {
27      let opts = Opts::new("raw_requests_total", "Total number of HTTP requests by raw URI");
28      let counter_vec = CounterVec::new(opts, &["raw_uri", "method", "status"]).unwrap();
29      REGISTRY.register(Box::new(counter_vec.clone())).unwrap();
30      counter_vec
31    };
32
33    // Histogram for request latency in seconds.
34    pub static ref REQUEST_LATENCY: HistogramVec = {
35      let histogram_opts = HistogramOpts::new("request_latency_seconds", "Request latency in seconds")
36          .buckets(vec![0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 25.0, 50.0, 100.0]);
37      let histogram_vec = HistogramVec::new(histogram_opts, &["endpoint", "method", "status"]).unwrap();
38      REGISTRY.register(Box::new(histogram_vec.clone())).unwrap();
39      histogram_vec
40    };
41
42    // Counter for error responses.
43    pub static ref ERROR_COUNTER: CounterVec = {
44        let opts = Opts::new("error_requests_total", "Total number of error responses");
45        // Using "status" to record the HTTP status code (or a special label like "service_error")
46        let counter_vec = CounterVec::new(opts, &["endpoint", "method", "status"]).unwrap();
47        REGISTRY.register(Box::new(counter_vec.clone())).unwrap();
48        counter_vec
49    };
50
51    // Gauge for CPU usage percentage.
52    pub static ref CPU_USAGE: Gauge = {
53      let gauge = Gauge::new("cpu_usage_percentage", "Current CPU usage percentage").unwrap();
54      REGISTRY.register(Box::new(gauge.clone())).unwrap();
55      gauge
56    };
57
58    // Gauge for memory usage percentage.
59    pub static ref MEMORY_USAGE_PERCENT: Gauge = {
60      let gauge = Gauge::new("memory_usage_percentage", "Memory usage percentage").unwrap();
61      REGISTRY.register(Box::new(gauge.clone())).unwrap();
62      gauge
63    };
64
65    // Gauge for memory usage in bytes.
66    pub static ref MEMORY_USAGE: Gauge = {
67        let gauge = Gauge::new("memory_usage_bytes", "Memory usage in bytes").unwrap();
68        REGISTRY.register(Box::new(gauge.clone())).unwrap();
69        gauge
70    };
71
72    // Gauge for total memory in bytes.
73    pub static ref TOTAL_MEMORY: Gauge = {
74      let gauge = Gauge::new("total_memory_bytes", "Total memory in bytes").unwrap();
75      REGISTRY.register(Box::new(gauge.clone())).unwrap();
76      gauge
77    };
78
79    // Gauge for available memory in bytes.
80    pub static ref AVAILABLE_MEMORY: Gauge = {
81        let gauge = Gauge::new("available_memory_bytes", "Available memory in bytes").unwrap();
82        REGISTRY.register(Box::new(gauge.clone())).unwrap();
83        gauge
84    };
85
86    // Gauge for used disk space in bytes.
87    pub static ref DISK_USAGE: Gauge = {
88      let gauge = Gauge::new("disk_usage_bytes", "Used disk space in bytes").unwrap();
89      REGISTRY.register(Box::new(gauge.clone())).unwrap();
90      gauge
91    };
92
93    // Gauge for disk usage percentage.
94    pub static ref DISK_USAGE_PERCENT: Gauge = {
95      let gauge = Gauge::new("disk_usage_percentage", "Disk usage percentage").unwrap();
96      REGISTRY.register(Box::new(gauge.clone())).unwrap();
97      gauge
98    };
99
100    // Gauge for in-flight requests.
101    pub static ref IN_FLIGHT_REQUESTS: GaugeVec = {
102        let gauge_vec = GaugeVec::new(
103            Opts::new("in_flight_requests", "Number of in-flight requests"),
104            &["endpoint"]
105        ).unwrap();
106        REGISTRY.register(Box::new(gauge_vec.clone())).unwrap();
107        gauge_vec
108    };
109
110    // Counter for request timeouts.
111    pub static ref TIMEOUT_COUNTER: CounterVec = {
112        let opts = Opts::new("request_timeouts_total", "Total number of request timeouts");
113        let counter_vec = CounterVec::new(opts, &["endpoint", "method", "timeout_type"]).unwrap();
114        REGISTRY.register(Box::new(counter_vec.clone())).unwrap();
115        counter_vec
116    };
117
118    // Gauge for file descriptor count.
119    pub static ref FILE_DESCRIPTORS: Gauge = {
120        let gauge = Gauge::new("file_descriptors_count", "Current file descriptor count").unwrap();
121        REGISTRY.register(Box::new(gauge.clone())).unwrap();
122        gauge
123    };
124
125    // Gauge for CLOSE_WAIT socket count.
126    pub static ref CLOSE_WAIT_SOCKETS: Gauge = {
127        let gauge = Gauge::new("close_wait_sockets_count", "Number of CLOSE_WAIT sockets").unwrap();
128        REGISTRY.register(Box::new(gauge.clone())).unwrap();
129        gauge
130    };
131
132    // Counter for successful transactions (Confirmed status).
133    pub static ref TRANSACTIONS_SUCCESS: CounterVec = {
134        let opts = Opts::new("transactions_success_total", "Total number of successful transactions");
135        let counter_vec = CounterVec::new(opts, &["relayer_id", "network_type"]).unwrap();
136        REGISTRY.register(Box::new(counter_vec.clone())).unwrap();
137        counter_vec
138    };
139
140    // Counter for failed transactions (Failed, Expired, Canceled statuses).
141    // Labels: relayer_id, network_type, failure_reason, previous_status.
142    // Note: `previous_status` label added to track the pipeline stage before the failure
143    // (e.g. "pending", "sent", "submitted"), enabling pre- vs post-submission attribution.
144    pub static ref TRANSACTIONS_FAILED: CounterVec = {
145        let opts = Opts::new("transactions_failed_total", "Total number of failed transactions");
146        let counter_vec = CounterVec::new(opts, &["relayer_id", "network_type", "failure_reason", "previous_status"]).unwrap();
147        REGISTRY.register(Box::new(counter_vec.clone())).unwrap();
148        counter_vec
149    };
150
151    // Counter for RPC failures during API requests (before transaction creation).
152    // This tracks failures that occur during operations like get_status, get_balance, etc.
153    // that happen before a transaction is created.
154    pub static ref API_RPC_FAILURES: CounterVec = {
155        let opts = Opts::new("api_rpc_failures_total", "Total number of RPC failures during API requests (before transaction creation)");
156        let counter_vec = CounterVec::new(opts, &["relayer_id", "network_type", "operation_name", "error_type"]).unwrap();
157        REGISTRY.register(Box::new(counter_vec.clone())).unwrap();
158        counter_vec
159    };
160
161    // Counter for transaction creation (when a transaction is successfully created in the repository).
162    pub static ref TRANSACTIONS_CREATED: CounterVec = {
163        let opts = Opts::new("transactions_created_total", "Total number of transactions created");
164        let counter_vec = CounterVec::new(opts, &["relayer_id", "network_type"]).unwrap();
165        REGISTRY.register(Box::new(counter_vec.clone())).unwrap();
166        counter_vec
167    };
168
169    // Counter for transaction submissions (when status changes to Submitted).
170    pub static ref TRANSACTIONS_SUBMITTED: CounterVec = {
171        let opts = Opts::new("transactions_submitted_total", "Total number of transactions submitted to the network");
172        let counter_vec = CounterVec::new(opts, &["relayer_id", "network_type"]).unwrap();
173        REGISTRY.register(Box::new(counter_vec.clone())).unwrap();
174        counter_vec
175    };
176
177    // Gauge for transaction status distribution (current count of transactions in each status).
178    pub static ref TRANSACTIONS_BY_STATUS: GaugeVec = {
179        let gauge_vec = GaugeVec::new(
180            Opts::new("transactions_by_status", "Current number of transactions by status"),
181            &["relayer_id", "network_type", "status"]
182        ).unwrap();
183        REGISTRY.register(Box::new(gauge_vec.clone())).unwrap();
184        gauge_vec
185    };
186
187    // Histogram for transaction processing times (creation to submission).
188    pub static ref TRANSACTION_PROCESSING_TIME: HistogramVec = {
189        let histogram_opts = HistogramOpts::new("transaction_processing_seconds", "Transaction processing time in seconds")
190            .buckets(vec![0.1, 0.5, 1.0, 2.0, 5.0, 10.0, 30.0, 60.0, 120.0, 300.0]);
191        let histogram_vec = HistogramVec::new(histogram_opts, &["relayer_id", "network_type", "stage"]).unwrap();
192        REGISTRY.register(Box::new(histogram_vec.clone())).unwrap();
193        histogram_vec
194    };
195
196    // Histogram for RPC call latency.
197    pub static ref RPC_CALL_LATENCY: HistogramVec = {
198        let histogram_opts = HistogramOpts::new("rpc_call_latency_seconds", "RPC call latency in seconds")
199            .buckets(vec![0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 30.0]);
200        let histogram_vec = HistogramVec::new(histogram_opts, &["relayer_id", "network_type", "operation_name"]).unwrap();
201        REGISTRY.register(Box::new(histogram_vec.clone())).unwrap();
202        histogram_vec
203    };
204
205    // Counter for Stellar transaction submission failures with decoded result codes.
206    pub static ref STELLAR_SUBMISSION_FAILURES: CounterVec = {
207        let opts = Opts::new("stellar_submission_failures_total",
208            "Stellar transaction submission failures by status and result code");
209        let counter_vec = CounterVec::new(opts, &["submit_status", "result_code"]).unwrap();
210        REGISTRY.register(Box::new(counter_vec.clone())).unwrap();
211        counter_vec
212    };
213
214    // Counter for plugin calls (tracks requests to /api/v1/plugins/{plugin_id}/call endpoints).
215    pub static ref PLUGIN_CALLS: CounterVec = {
216        let opts = Opts::new("plugin_calls_total", "Total number of plugin calls");
217        let counter_vec = CounterVec::new(opts, &["plugin_id", "method", "status"]).unwrap();
218        REGISTRY.register(Box::new(counter_vec.clone())).unwrap();
219        counter_vec
220    };
221
222    // Counter for Stellar submit responses with TRY_AGAIN_LATER status.
223    pub static ref STELLAR_TRY_AGAIN_LATER: CounterVec = {
224        let opts = Opts::new(
225            "stellar_try_again_later_total",
226            "Total number of Stellar transaction submit responses with TRY_AGAIN_LATER"
227        );
228        let counter_vec = CounterVec::new(opts, &["relayer_id", "tx_status"]).unwrap();
229        REGISTRY.register(Box::new(counter_vec.clone())).unwrap();
230        counter_vec
231    };
232
233    // Counter for transactions confirmed after experiencing TRY_AGAIN_LATER.
234    pub static ref TRANSACTIONS_TRY_AGAIN_LATER_SUCCESS: CounterVec = {
235        let opts = Opts::new(
236            "transactions_try_again_later_success_total",
237            "Total number of transactions confirmed after experiencing TRY_AGAIN_LATER"
238        );
239        let counter_vec = CounterVec::new(opts, &["relayer_id", "network_type"]).unwrap();
240        REGISTRY.register(Box::new(counter_vec.clone())).unwrap();
241        counter_vec
242    };
243
244    // Counter for transactions that failed after experiencing TRY_AGAIN_LATER.
245    pub static ref TRANSACTIONS_TRY_AGAIN_LATER_FAILED: CounterVec = {
246        let opts = Opts::new(
247            "transactions_try_again_later_failed_total",
248            "Total number of transactions that failed after experiencing TRY_AGAIN_LATER"
249        );
250        let counter_vec = CounterVec::new(opts, &["relayer_id", "network_type"]).unwrap();
251        REGISTRY.register(Box::new(counter_vec.clone())).unwrap();
252        counter_vec
253    };
254
255    // Counter for transactions that encountered an insufficient fee error.
256    pub static ref TRANSACTIONS_INSUFFICIENT_FEE: CounterVec = {
257        let opts = Opts::new(
258            "transactions_insufficient_fee_total",
259            "Total number of transactions that encountered an insufficient fee error"
260        );
261        let counter_vec = CounterVec::new(opts, &["relayer_id", "network_type"]).unwrap();
262        REGISTRY.register(Box::new(counter_vec.clone())).unwrap();
263        counter_vec
264    };
265
266    // Counter for transactions confirmed after experiencing insufficient fee.
267    pub static ref TRANSACTIONS_INSUFFICIENT_FEE_SUCCESS: CounterVec = {
268        let opts = Opts::new(
269            "transactions_insufficient_fee_success_total",
270            "Total number of transactions confirmed after experiencing insufficient fee"
271        );
272        let counter_vec = CounterVec::new(opts, &["relayer_id", "network_type"]).unwrap();
273        REGISTRY.register(Box::new(counter_vec.clone())).unwrap();
274        counter_vec
275    };
276
277    // Counter for transactions that failed after experiencing insufficient fee.
278    pub static ref TRANSACTIONS_INSUFFICIENT_FEE_FAILED: CounterVec = {
279        let opts = Opts::new(
280            "transactions_insufficient_fee_failed_total",
281            "Total number of transactions that failed after experiencing insufficient fee"
282        );
283        let counter_vec = CounterVec::new(opts, &["relayer_id", "network_type"]).unwrap();
284        REGISTRY.register(Box::new(counter_vec.clone())).unwrap();
285        counter_vec
286    };
287}
288
289/// Gather all metrics and encode into the provided format.
290pub fn gather_metrics() -> Result<Vec<u8>, Box<dyn std::error::Error>> {
291    let encoder = TextEncoder::new();
292    let metric_families = REGISTRY.gather();
293    let mut buffer = Vec::new();
294    encoder.encode(&metric_families, &mut buffer)?;
295    Ok(buffer)
296}
297
298/// Get file descriptor count for current process.
299fn get_fd_count() -> Result<usize, std::io::Error> {
300    let pid = std::process::id();
301
302    #[cfg(target_os = "linux")]
303    {
304        let fd_dir = format!("/proc/{pid}/fd");
305        std::fs::read_dir(fd_dir).map(|entries| entries.count())
306    }
307
308    #[cfg(target_os = "macos")]
309    {
310        use std::process::Command;
311        let output = Command::new("lsof")
312            .args(["-p", &pid.to_string()])
313            .output()?;
314        let count = String::from_utf8_lossy(&output.stdout)
315            .lines()
316            .count()
317            .saturating_sub(1); // Subtract header line
318        Ok(count)
319    }
320
321    #[cfg(not(any(target_os = "linux", target_os = "macos")))]
322    {
323        Ok(0) // Unsupported platform
324    }
325}
326
327/// Get CLOSE_WAIT socket count.
328fn get_close_wait_count() -> Result<usize, std::io::Error> {
329    #[cfg(any(target_os = "linux", target_os = "macos"))]
330    {
331        use std::process::Command;
332        let output = Command::new("sh")
333            .args(["-c", "netstat -an | grep CLOSE_WAIT | wc -l"])
334            .output()?;
335        let count = String::from_utf8_lossy(&output.stdout)
336            .trim()
337            .parse()
338            .unwrap_or(0);
339        Ok(count)
340    }
341
342    #[cfg(not(any(target_os = "linux", target_os = "macos")))]
343    {
344        Ok(0) // Unsupported platform
345    }
346}
347
348/// Updates the system metrics for CPU and memory usage.
349pub fn update_system_metrics() {
350    let mut sys = System::new_all();
351    sys.refresh_all();
352
353    // Overall CPU usage.
354    let cpu_usage = sys.global_cpu_usage();
355    CPU_USAGE.set(cpu_usage as f64);
356
357    // Total memory (in bytes).
358    let total_memory = sys.total_memory();
359    TOTAL_MEMORY.set(total_memory as f64);
360
361    // Available memory (in bytes).
362    let available_memory = sys.available_memory();
363    AVAILABLE_MEMORY.set(available_memory as f64);
364
365    // Used memory (in bytes).
366    let memory_usage = sys.used_memory();
367    MEMORY_USAGE.set(memory_usage as f64);
368
369    // Calculate memory usage percentage
370    let memory_percentage = if total_memory > 0 {
371        (memory_usage as f64 / total_memory as f64) * 100.0
372    } else {
373        0.0
374    };
375    MEMORY_USAGE_PERCENT.set(memory_percentage);
376
377    // Calculate disk usage:
378    // Sum total space and available space across all disks.
379    let disks = Disks::new_with_refreshed_list();
380    let mut total_disk_space: u64 = 0;
381    let mut total_disk_available: u64 = 0;
382    for disk in disks.list() {
383        total_disk_space += disk.total_space();
384        total_disk_available += disk.available_space();
385    }
386    // Used disk space is total minus available ( in bytes).
387    let used_disk_space = total_disk_space.saturating_sub(total_disk_available);
388    DISK_USAGE.set(used_disk_space as f64);
389
390    // Calculate disk usage percentage.
391    let disk_percentage = if total_disk_space > 0 {
392        (used_disk_space as f64 / total_disk_space as f64) * 100.0
393    } else {
394        0.0
395    };
396    DISK_USAGE_PERCENT.set(disk_percentage);
397
398    // Update file descriptor count.
399    if let Ok(fd_count) = get_fd_count() {
400        FILE_DESCRIPTORS.set(fd_count as f64);
401    }
402
403    // Update CLOSE_WAIT socket count.
404    if let Ok(close_wait) = get_close_wait_count() {
405        CLOSE_WAIT_SOCKETS.set(close_wait as f64);
406    }
407}
408
409#[cfg(test)]
410mod actix_tests {
411    use super::*;
412    use actix_web::{
413        dev::{Service, ServiceRequest, ServiceResponse, Transform},
414        http, test, Error, HttpResponse,
415    };
416    use futures::future::{self};
417    use middleware::MetricsMiddleware;
418    use prometheus::proto::MetricFamily;
419    use std::{
420        pin::Pin,
421        task::{Context, Poll},
422    };
423
424    // Dummy service that always returns a successful response (HTTP 200 OK).
425    struct DummySuccessService;
426
427    impl Service<ServiceRequest> for DummySuccessService {
428        type Response = ServiceResponse;
429        type Error = Error;
430        type Future = Pin<Box<dyn future::Future<Output = Result<Self::Response, Self::Error>>>>;
431
432        fn poll_ready(&self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
433            Poll::Ready(Ok(()))
434        }
435
436        fn call(&self, req: ServiceRequest) -> Self::Future {
437            let resp = req.into_response(HttpResponse::Ok().finish());
438            Box::pin(async move { Ok(resp) })
439        }
440    }
441
442    // Dummy service that always returns an error.
443    struct DummyErrorService;
444
445    impl Service<ServiceRequest> for DummyErrorService {
446        type Response = ServiceResponse;
447        type Error = Error;
448        type Future = Pin<Box<dyn future::Future<Output = Result<Self::Response, Self::Error>>>>;
449
450        fn poll_ready(&self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
451            Poll::Ready(Ok(()))
452        }
453
454        fn call(&self, _req: ServiceRequest) -> Self::Future {
455            Box::pin(async move { Err(actix_web::error::ErrorInternalServerError("dummy error")) })
456        }
457    }
458
459    // Helper function to find a metric family by name.
460    fn find_metric_family<'a>(
461        name: &str,
462        families: &'a [MetricFamily],
463    ) -> Option<&'a MetricFamily> {
464        families.iter().find(|mf| mf.name() == name)
465    }
466
467    #[actix_rt::test]
468    async fn test_gather_metrics_contains_expected_names() {
469        // Update system metrics
470        update_system_metrics();
471
472        // Increment request counters to ensure they appear in output
473        REQUEST_COUNTER
474            .with_label_values(&["/test", "GET", "200"])
475            .inc();
476        RAW_REQUEST_COUNTER
477            .with_label_values(&["/test?param=value", "GET", "200"])
478            .inc();
479        REQUEST_LATENCY
480            .with_label_values(&["/test", "GET", "200"])
481            .observe(0.1);
482        ERROR_COUNTER
483            .with_label_values(&["/test", "GET", "500"])
484            .inc();
485
486        // Touch insufficient fee metrics to ensure they appear in output
487        TRANSACTIONS_INSUFFICIENT_FEE
488            .with_label_values(&["test-relayer", "stellar"])
489            .inc();
490        TRANSACTIONS_INSUFFICIENT_FEE_SUCCESS
491            .with_label_values(&["test-relayer", "stellar"])
492            .inc();
493        TRANSACTIONS_INSUFFICIENT_FEE_FAILED
494            .with_label_values(&["test-relayer", "stellar"])
495            .inc();
496
497        // Touch TRY_AGAIN_LATER metrics to ensure they appear in output
498        TRANSACTIONS_TRY_AGAIN_LATER_SUCCESS
499            .with_label_values(&["test-relayer", "stellar"])
500            .inc();
501        TRANSACTIONS_TRY_AGAIN_LATER_FAILED
502            .with_label_values(&["test-relayer", "stellar"])
503            .inc();
504
505        let metrics = gather_metrics().expect("failed to gather metrics");
506        let output = String::from_utf8(metrics).expect("metrics output is not valid UTF-8");
507
508        // System metrics
509        assert!(output.contains("cpu_usage_percentage"));
510        assert!(output.contains("memory_usage_percentage"));
511        assert!(output.contains("memory_usage_bytes"));
512        assert!(output.contains("total_memory_bytes"));
513        assert!(output.contains("available_memory_bytes"));
514        assert!(output.contains("disk_usage_bytes"));
515        assert!(output.contains("disk_usage_percentage"));
516
517        // Request metrics
518        assert!(output.contains("requests_total"));
519        assert!(output.contains("raw_requests_total"));
520        assert!(output.contains("request_latency_seconds"));
521        assert!(output.contains("error_requests_total"));
522
523        // Insufficient fee metrics
524        assert!(output.contains("transactions_insufficient_fee_total"));
525        assert!(output.contains("transactions_insufficient_fee_success_total"));
526        assert!(output.contains("transactions_insufficient_fee_failed_total"));
527
528        // TRY_AGAIN_LATER metrics
529        assert!(output.contains("transactions_try_again_later_success_total"));
530        assert!(output.contains("transactions_try_again_later_failed_total"));
531    }
532
533    #[actix_rt::test]
534    async fn test_update_system_metrics() {
535        // Reset metrics to ensure clean state
536        CPU_USAGE.set(0.0);
537        TOTAL_MEMORY.set(0.0);
538        AVAILABLE_MEMORY.set(0.0);
539        MEMORY_USAGE.set(0.0);
540        MEMORY_USAGE_PERCENT.set(0.0);
541        DISK_USAGE.set(0.0);
542        DISK_USAGE_PERCENT.set(0.0);
543
544        // Call the function we're testing
545        update_system_metrics();
546
547        // Verify that metrics have been updated with reasonable values
548        let cpu_usage = CPU_USAGE.get();
549        assert!(
550            (0.0..=100.0).contains(&cpu_usage),
551            "CPU usage should be between 0-100%, got {cpu_usage}"
552        );
553
554        let memory_usage = MEMORY_USAGE.get();
555        assert!(
556            memory_usage >= 0.0,
557            "Memory usage should be >= 0, got {memory_usage}"
558        );
559
560        let memory_percent = MEMORY_USAGE_PERCENT.get();
561        assert!(
562            (0.0..=100.0).contains(&memory_percent),
563            "Memory usage percentage should be between 0-100%, got {memory_percent}"
564        );
565
566        let total_memory = TOTAL_MEMORY.get();
567        assert!(
568            total_memory > 0.0,
569            "Total memory should be > 0, got {total_memory}"
570        );
571
572        let available_memory = AVAILABLE_MEMORY.get();
573        assert!(
574            available_memory >= 0.0,
575            "Available memory should be >= 0, got {available_memory}"
576        );
577
578        let disk_usage = DISK_USAGE.get();
579        assert!(
580            disk_usage >= 0.0,
581            "Disk usage should be >= 0, got {disk_usage}"
582        );
583
584        let disk_percent = DISK_USAGE_PERCENT.get();
585        assert!(
586            (0.0..=100.0).contains(&disk_percent),
587            "Disk usage percentage should be between 0-100%, got {disk_percent}"
588        );
589
590        // Verify that memory usage doesn't exceed total memory
591        assert!(
592            memory_usage <= total_memory,
593            "Memory usage should be <= total memory, got {memory_usage}"
594        );
595
596        // Verify that available memory plus used memory doesn't exceed total memory
597        assert!(
598            (available_memory + memory_usage) <= total_memory,
599            "Available memory plus used memory should be <= total memory {}, got {}",
600            total_memory,
601            available_memory + memory_usage
602        );
603    }
604
605    #[actix_rt::test]
606    async fn test_middleware_success() {
607        let req = test::TestRequest::with_uri("/test_success").to_srv_request();
608
609        let middleware = MetricsMiddleware;
610        let service = middleware.new_transform(DummySuccessService).await.unwrap();
611
612        let resp = service.call(req).await.unwrap();
613        assert_eq!(resp.response().status(), http::StatusCode::OK);
614
615        let families = REGISTRY.gather();
616        let counter_fam = find_metric_family("requests_total", &families)
617            .expect("requests_total metric family not found");
618
619        let mut found = false;
620        for m in counter_fam.get_metric() {
621            let labels = m.get_label();
622            if labels
623                .iter()
624                .any(|l| l.name() == "endpoint" && l.value() == "/test_success")
625            {
626                found = true;
627                assert!(m.get_counter().value() >= 1.0);
628            }
629        }
630        assert!(
631            found,
632            "Expected metric with endpoint '/test_success' not found"
633        );
634    }
635
636    #[actix_rt::test]
637    async fn test_middleware_error() {
638        let req = test::TestRequest::with_uri("/test_error").to_srv_request();
639
640        let middleware = MetricsMiddleware;
641        let service = middleware.new_transform(DummyErrorService).await.unwrap();
642
643        let result = service.call(req).await;
644        assert!(result.is_err());
645
646        let families = REGISTRY.gather();
647        let error_counter_fam = find_metric_family("error_requests_total", &families)
648            .expect("error_requests_total metric family not found");
649
650        let mut found = false;
651        for m in error_counter_fam.get_metric() {
652            let labels = m.get_label();
653            if labels
654                .iter()
655                .any(|l| l.name() == "endpoint" && l.value() == "/test_error")
656            {
657                found = true;
658                assert!(m.get_counter().value() >= 1.0);
659            }
660        }
661        assert!(
662            found,
663            "Expected error metric with endpoint '/test_error' not found"
664        );
665    }
666}
667
668#[cfg(test)]
669mod property_tests {
670    use proptest::{prelude::*, test_runner::Config};
671
672    // A helper function to compute percentage used from total.
673    fn compute_percentage(used: u64, total: u64) -> f64 {
674        if total > 0 {
675            (used as f64 / total as f64) * 100.0
676        } else {
677            0.0
678        }
679    }
680
681    proptest! {
682        // Set the number of cases to 1000
683        #![proptest_config(Config {
684          cases: 1000, ..Config::default()
685        })]
686
687        #[test]
688        fn prop_compute_percentage((total, used) in {
689            (1u64..1_000_000u64).prop_flat_map(|total| {
690                (Just(total), 0u64..=total)
691            })
692        }) {
693            let percentage = compute_percentage(used, total);
694            prop_assert!(percentage >= 0.0);
695            prop_assert!(percentage <= 100.0);
696        }
697
698        #[test]
699        fn prop_labels_are_reasonable(
700              endpoint in ".*",
701              method in prop::sample::select(vec![
702                "GET".to_string(),
703                "POST".to_string(),
704                "PUT".to_string(),
705                "DELETE".to_string()
706                ])
707            ) {
708            let endpoint_label = if endpoint.is_empty() { "/".to_string() } else { endpoint.clone() };
709            let method_label = method;
710
711            prop_assert!(endpoint_label.chars().count() <= 1024, "Endpoint label too long");
712            prop_assert!(method_label.chars().count() <= 16, "Method label too long");
713
714            let status = "200".to_string();
715            let labels = vec![endpoint_label, method_label, status];
716
717            for label in labels {
718                prop_assert!(!label.is_empty());
719                prop_assert!(label.len() < 1024);
720            }
721        }
722    }
723}
openzeppelin_relayer/metrics/mod.rs

openzeppelin_relayer/metrics/
mod.rs
