diff --git a/src/utils/qdf/blacklist.rs b/src/utils/qdf/blacklist.rs index 61ac029..2ab1fad 100644 --- a/src/utils/qdf/blacklist.rs +++ b/src/utils/qdf/blacklist.rs @@ -7,7 +7,7 @@ use polars::prelude::*; use std::collections::{BTreeMap, HashMap}; use std::error::Error; -use super::get_unique_metrics; +use crate::utils::qdf::get_unique_metrics; pub fn create_blacklist_from_qdf( df: &DataFrame, @@ -30,19 +30,26 @@ pub fn create_blacklist_from_qdf( BDateFreq::Daily, )?; - let null_mask = get_nan_mask(df, metrics)?; + // if none of the metrics are null or NaN, return an empty blacklist + if !metrics.iter().any(|metric| { + df.column(metric) + .map(|col| col.is_null().any()) + .unwrap_or(false) + }) { + return Ok(BTreeMap::new()); + } - let df = df.filter(&null_mask)?.clone(); + // let null_mask = get_nan_mask(df, metrics)?; + // let df = df.filter(&null_mask)?.clone(); let df = df .clone() .lazy() - // .filter(&null_mask) - // .filter( - // col(metric.as_str()) - // .is_null() - // .or(col(metric.as_str()).is_nan()), - // ) + .with_columns([ + (cols(metrics.clone()).is_null().or(cols(metrics).is_nan())).alias("null_mask") + ]) + .filter(col("null_mask")) + // if is now empty, return an empty blacklist .sort( ["cid", "xcat"], SortMultipleOptions::default().with_maintain_order(true),