adding filter layer to jpmaqsdownload

src/download/jpmaqsdownload.rs

@@ -24,6 +24,22 @@ fn construct_expressions(tickers: Vec<String>, metrics: Vec<String>) -> Vec<Stri
         .collect::<Vec<String>>()
 }
 
+fn deconstruct_expression(expression: &str) -> (String, String, String) {
+    if !is_jpmaq_expression(expression) {
+        return (
+            expression.to_string(),
+            expression.to_string(),
+            expression.to_string(),
+        );
+    }
+    let parts = expression.split(',').collect::<Vec<&str>>();
+    let ticker = parts[1].to_string();
+    let metric = parts[2].to_string();
+    let ticker_parts = ticker.split_once('_').unwrap();
+    let cid = ticker_parts.0.to_string();
+    let xcat = ticker_parts.1.to_string();
+    (cid, xcat, metric)
+}
 fn is_jpmaq_expression(expression: &str) -> bool {
     expression.starts_with("DB(JPMAQS,")
         && expression.ends_with(")")
@@ -152,10 +168,30 @@ impl JPMaQSDownload {
 
     /// Get the catalogue of tickers available in the JPMaQS data.
     pub fn get_catalogue(&mut self) -> Result<Vec<String>, Box<dyn Error>> {
+        println!("Getting JPMaQS catalogue ...");
         let dq_catalogue = self.requester.get_catalogue("JPMAQS", 1000)?;
         Ok(dq_catalogue.all_instruments)
     }
 
+    fn filter_expressions(
+        &mut self,
+        expressions: Vec<String>,
+    ) -> Result<Vec<String>, Box<dyn Error>> {
+        // filter out expressions that are not in the catalogue
+        let dq_catalogue = self.get_catalogue()?;
+        println!("Filtering expressions based on the JPMaQS catalogue ...");
+        let filtered_expressions = expressions
+            .iter()
+            .filter(|expression| {
+                let (cid, xcat, _) = deconstruct_expression(expression);
+                dq_catalogue.contains(&format!("{}_{}", cid, xcat))
+            })
+            .map(|s| s.to_string())
+            .collect::<Vec<String>>();
+
+        Ok(filtered_expressions)
+    }
+
     /// Get the time series data for the provided expressions.
     pub fn get_expressions(
         &mut self,
@@ -184,6 +220,8 @@ impl JPMaQSDownload {
         let expressions = construct_expressions(download_args.tickers, download_args.metrics);
         assert!(all_jpmaq_expressions(expressions.clone()));
 
+        let expressions = self.filter_expressions(expressions)?;
+
         let dq_download_args = DQTimeseriesRequestArgs {
             expressions: expressions,
             start_date: download_args.start_date,

src/download/mod.rs

@@ -1,5 +1,8 @@
-pub mod helpers;
-pub mod jpmaqsdownload;
+mod helpers;
 mod oauth_client;
 mod parreq;
 mod requester;
+
+pub mod jpmaqsdownload;
+
+pub use jpmaqsdownload::*;

src/download/parreq.rs

@@ -77,6 +77,11 @@ impl ParallelRequester {
             })
             .collect();
 
+        let prog_batches = match expression_batches.len() {
+            0..=250 => 25,
+            _ => 100,
+        };
+
         let okay_response_texts = Arc::new(Mutex::new(Vec::new()));
         let failed_batches = Arc::new(Mutex::new(Vec::new()));
         let self_arc = Arc::new(self.clone());
@@ -97,7 +102,7 @@ impl ParallelRequester {
                 let ep = format!("{}?{}", TIMESERIES_ENDPOINT, args_clone.as_query_string());
                 let permit = semaphore.clone().acquire_owned().await.unwrap();
 
-                if curr_batch % 100 == 0 {
+                if curr_batch % prog_batches == 0 {
                     println!("Requesting batch {} of {}", curr_batch, total_batches);
                 }