magnus167/msyrs
1
0
Fork 0
mirror of https://github.com/Magnus167/msyrs.git synced 2025-08-20 04:20:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

testing...

Browse Source
This commit is contained in:
Palash Tyagi 2025-04-16 22:36:03 +01:00
parent ee531deb7a
commit 178de83d1a
1 changed files with 16 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
src/utils/qdf/blacklist.rs
View File

@ -7,7 +7,7 @@ use polars::prelude::*;
use std::collections::{BTreeMap, HashMap}; use std::collections::{BTreeMap, HashMap};
use std::error::Error; use std::error::Error;
use super::get_unique_metrics; use crate::utils::qdf::get_unique_metrics;
pub fn create_blacklist_from_qdf( pub fn create_blacklist_from_qdf(
df: &DataFrame, df: &DataFrame,
@ -30,19 +30,26 @@ pub fn create_blacklist_from_qdf(
BDateFreq::Daily, BDateFreq::Daily,
)?; )?;
let null_mask = get_nan_mask(df, metrics)?; // if none of the metrics are null or NaN, return an empty blacklist
if !metrics.iter().any(|metric| {
df.column(metric)
.map(|col| col.is_null().any())
.unwrap_or(false)
}) {
return Ok(BTreeMap::new());
}
let df = df.filter(&null_mask)?.clone(); // let null_mask = get_nan_mask(df, metrics)?;
// let df = df.filter(&null_mask)?.clone();
let df = df let df = df
.clone() .clone()
.lazy() .lazy()
// .filter(&null_mask) .with_columns([
// .filter( (cols(metrics.clone()).is_null().or(cols(metrics).is_nan())).alias("null_mask")
// col(metric.as_str()) ])
// .is_null() .filter(col("null_mask"))
// .or(col(metric.as_str()).is_nan()), // if is now empty, return an empty blacklist
// )
.sort( .sort(
["cid", "xcat"], ["cid", "xcat"],
SortMultipleOptions::default().with_maintain_order(true), SortMultipleOptions::default().with_maintain_order(true),