working

src/download/jpmaqsdownload.rs

@@ -1,8 +1,8 @@
-use crate::download::oauth_client::OAuthClient;
-use crate::download::requester::DQRequester;
 use crate::download::helpers::DQTimeSeriesResponse;
 use crate::download::helpers::DQTimeseriesRequestArgs;
 use crate::download::helpers::JPMaQSIndicator;
+use crate::download::oauth_client::OAuthClient;
+use crate::download::requester::DQRequester;
 // use polars::prelude::*;
 use std::error::Error;
 
@@ -165,4 +165,103 @@ impl JPMaQSDownload {
         );
         Ok(df_main)
     }
+
+    // pub fn save_indicators_as_csv(
+    //     &mut self,
+    //     download_args: JPMaQSDownloadGetIndicatorArgs,
+    //     batch_size: usize,
+    //     folder_path: &str,
+    // ) -> Result<(), Box<dyn Error>> {
+    //     // if the folder path does not exist, create it
+    //     std::fs::create_dir_all(folder_path)?;
+
+    //     // get ticker count, and split into chunks of batch_size
+    //     let ticker_batches = download_args.tickers.chunks(batch_size);
+    //     // let batch_download_args -- create new download args for each batch with new tickers
+    //     let mut batch_download_args = Vec::new();
+    //     for ticker_batch in ticker_batches {
+    //         let mut new_args = download_args.clone();
+    //         new_args.tickers = ticker_batch.to_vec();
+    //         batch_download_args.push(new_args);
+    //     }
+    //     // print total number of download sessions
+    //     println!(
+    //         "Total number of download sessions: {}",
+    //         batch_download_args.len()
+    //     );
+    //     // download each batch and save as csv
+    //     for (_, batch_args) in batch_download_args.iter().enumerate() {
+    //         let df_list = self.get_indicators_list(batch_args.clone())?;
+    //         let total_indicators = df_list.len();
+    //         save_indicators_list_as_csvs(df_list, &folder_path)?;
+    //         println!("Saved {} indicators to {}", total_indicators, folder_path);
+    //     }
+
+    //     Ok(())
+    // }
+    pub fn save_indicators_as_csv(
+        &mut self,
+        download_args: JPMaQSDownloadGetIndicatorArgs,
+        batch_size: usize,
+        folder_path: &str,
+    ) -> Result<(), Box<dyn Error>> {
+        // if the folder path does not exist, create it
+        std::fs::create_dir_all(folder_path)?;
+    
+        // get ticker count, and split into chunks of batch_size
+        let ticker_batches = download_args.tickers.chunks(batch_size);
+        // let batch_download_args -- create new download args for each batch with new tickers
+        let mut batch_download_args = Vec::new();
+        for ticker_batch in ticker_batches {
+            let mut new_args = download_args.clone();
+            new_args.tickers = ticker_batch.to_vec();
+            batch_download_args.push(new_args);
+        }
+        // print total number of download sessions
+        println!(
+            "Total number of download sessions: {}",
+            batch_download_args.len()
+        );
+    
+        // Start timer
+        let start_time = std::time::Instant::now();
+        let total_batches = batch_download_args.len();
+    
+        // download each batch and save as csv
+        for (index, batch_args) in batch_download_args.iter().enumerate() {
+            let df_list = self.get_indicators_list(batch_args.clone())?;
+            let total_indicators = df_list.len();
+            save_indicators_list_as_csvs(df_list, &folder_path)?;
+            println!("Saved {} indicators to {}", total_indicators, folder_path);
+    
+            // Calculate and print estimated remaining time
+            let elapsed = start_time.elapsed();
+            let average_time_per_batch = elapsed / (index as u32 + 1);
+            let remaining_batches = total_batches - index - 1;
+            let estimated_remaining = average_time_per_batch * remaining_batches as u32;
+    
+            println!(
+                "Progress: {}/{}. Estimated time remaining: {:.2?}",
+                index + 1,
+                total_batches,
+                estimated_remaining
+            );
+        }
+    
+        Ok(())
+    }
+    
+}
+
+fn save_indicators_list_as_csvs(
+    indicators: Vec<JPMaQSIndicator>,
+    folder_path: &str,
+) -> Result<(), Box<dyn Error>> {
+    for indicator in indicators {
+        let ticker_folder = format!("{}/{}", folder_path, indicator.get_xcat());
+        std::fs::create_dir_all(&ticker_folder)?;
+        let file_path = format!("{}/{}.csv", ticker_folder, indicator.ticker);
+        indicator.save_as_csv(&file_path)?;
+    }
+    Ok(())
 }