fix: add metrics collections in billing metrics (#1500)

Author: nikhilsinhaparseable

src/handlers/http/cluster/mod.rs

@@ -99,6 +99,7 @@ struct BillingMetricsCollector {
     pub total_bytes_scanned_in_object_store_calls_by_date: HashMap<String, HashMap<String, u64>>,
     pub total_input_llm_tokens_by_date: HashMap<String, HashMap<String, HashMap<String, u64>>>, // provider -> model -> date -> count
     pub total_output_llm_tokens_by_date: HashMap<String, HashMap<String, HashMap<String, u64>>>,
+    pub total_metrics_collected_by_date: HashMap<String, u64>,
     pub event_time: chrono::NaiveDateTime,
 }
 
@@ -193,6 +194,14 @@ impl BillingMetricsCollector {
                 &self.total_bytes_scanned_in_query_by_date,
             );
         }
+
+        if !self.total_metrics_collected_by_date.is_empty() {
+            add_simple_metric(
+                events,
+                "total_metrics_collected",
+                &self.total_metrics_collected_by_date,
+            );
+        }
     }
 
     /// Add object store metrics (method-based) to the events vector
@@ -1263,6 +1272,7 @@ fn is_simple_metric(metric: &str) -> bool {
             | "parseable_total_query_calls_by_date"
             | "parseable_total_files_scanned_in_query_by_date"
             | "parseable_total_bytes_scanned_in_query_by_date"
+            | "parseable_total_metrics_collected_by_date"
     )
 }
 
@@ -1329,6 +1339,11 @@ fn process_simple_metric(
                     .total_bytes_scanned_in_query_by_date
                     .insert(date.to_string(), value);
             }
+            "parseable_total_metrics_collected_by_date" => {
+                collector
+                    .total_metrics_collected_by_date
+                    .insert(date.to_string(), value);
+            }
             _ => {}
         }
     }

src/metrics/mod.rs

@@ -373,6 +373,18 @@ pub static STORAGE_REQUEST_RESPONSE_TIME: Lazy<HistogramVec> = Lazy::new(|| {
     .expect("metric can be created")
 });
 
+pub static TOTAL_METRICS_COLLECTED_BY_DATE: Lazy<IntCounterVec> = Lazy::new(|| {
+    IntCounterVec::new(
+        Opts::new(
+            "total_metrics_collected_by_date",
+            "Total metrics collected by date",
+        )
+        .namespace(METRICS_NAMESPACE),
+        &["date"],
+    )
+    .expect("metric can be created")
+});
+
 fn custom_metrics(registry: &Registry) {
     registry
         .register(Box::new(EVENTS_INGESTED.clone()))
@@ -472,6 +484,9 @@ fn custom_metrics(registry: &Registry) {
     registry
         .register(Box::new(STORAGE_REQUEST_RESPONSE_TIME.clone()))
         .expect("metric can be registered");
+    registry
+        .register(Box::new(TOTAL_METRICS_COLLECTED_BY_DATE.clone()))
+        .expect("metric can be registered");
 }
 
 pub fn build_metrics_handler() -> PrometheusMetrics {
@@ -619,6 +634,12 @@ pub fn increment_reasoning_llm_tokens_by_date(
         .inc_by(tokens);
 }
 
+pub fn increment_metrics_collected_by_date(date: &str) {
+    TOTAL_METRICS_COLLECTED_BY_DATE
+        .with_label_values(&[date])
+        .inc();
+}
+
 use actix_web::HttpResponse;
 use prometheus::Encoder;
 

src/otel/metrics.rs

@@ -24,6 +24,7 @@ use opentelemetry_proto::tonic::metrics::v1::{
 };
 use serde_json::{Map, Value};
 
+use crate::metrics::increment_metrics_collected_by_date;
 use crate::otel::otel_utils::flatten_attributes;
 
 use super::otel_utils::{
@@ -500,6 +501,9 @@ pub fn flatten_metrics_record(metrics_record: &Metric) -> Vec<Map<String, Value>
     if data_points_json.is_empty() {
         data_points_json.push(metric_json);
     }
+
+    let current_date = chrono::Utc::now().date_naive().to_string();
+    increment_metrics_collected_by_date(&current_date);
     data_points_json
 }