magnus167/msyrs
1
0
Fork 0
mirror of https://github.com/Magnus167/msyrs.git synced 2025-08-20 04:20:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

testing...

Browse Source
This commit is contained in:
Palash Tyagi 2025-04-16 22:36:03 +01:00
parent ee531deb7a
commit 178de83d1a
1 changed files with 16 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
src/utils/qdf/blacklist.rs
View File

@ -7,7 +7,7 @@ use polars::prelude::*;
use std::collections::{BTreeMap, HashMap};
use std::error::Error;
use super::get_unique_metrics;
use crate::utils::qdf::get_unique_metrics;
pub fn create_blacklist_from_qdf(
df: &DataFrame,
@ -30,19 +30,26 @@ pub fn create_blacklist_from_qdf(
BDateFreq::Daily,
)?;
let null_mask = get_nan_mask(df, metrics)?;
// if none of the metrics are null or NaN, return an empty blacklist
if !metrics.iter().any(|metric| {
df.column(metric)
.map(|col| col.is_null().any())
.unwrap_or(false)
}) {
return Ok(BTreeMap::new());
}
let df = df.filter(&null_mask)?.clone();
// let null_mask = get_nan_mask(df, metrics)?;
// let df = df.filter(&null_mask)?.clone();
let df = df
.clone()
.lazy()
// .filter(&null_mask)
// .filter(
// col(metric.as_str())
// .is_null()
// .or(col(metric.as_str()).is_nan()),
// )
.with_columns([
(cols(metrics.clone()).is_null().or(cols(metrics).is_nan())).alias("null_mask")
])
.filter(col("null_mask"))
// if is now empty, return an empty blacklist
.sort(
["cid", "xcat"],
SortMultipleOptions::default().with_maintain_order(true),