magnus167/msyrs
1
0
Fork 0
mirror of https://github.com/Magnus167/msyrs.git synced 2025-08-20 08:10:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: update folder path handling in JPMaQSDownload

Browse Source
This commit is contained in:
Palash Tyagi 2024-11-13 00:57:11 +00:00
parent 6cf7a2f174
commit a091ec2b88
1 changed files with 19 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
src/download/jpmaqsdownload.rs
View File

@ -206,40 +206,43 @@ impl JPMaQSDownload {
folder_path: &str, folder_path: &str,
) -> Result<(), Box<dyn Error>> { ) -> Result<(), Box<dyn Error>> {
// if the folder path does not exist, create it // if the folder path does not exist, create it
std::fs::create_dir_all(folder_path)?; // let save_path = folder_path.to_string() + "/JPMaQSData/";
let save_path = format!("{}/JPMaQSData/", folder_path);
std::fs::create_dir_all(save_path.clone())?;
// get ticker count, and split into chunks of batch_size // get ticker count, and split into chunks of batch_size
let ticker_batches = download_args.tickers.chunks(batch_size); let ticker_batches = download_args.tickers.chunks(batch_size);
// let batch_download_args -- create new download args for each batch with new tickers let batch_download_args = ticker_batches
let mut batch_download_args = Vec::new(); .map(|ticker_batch| {
for ticker_batch in ticker_batches { let mut new_args = download_args.clone();
let mut new_args = download_args.clone(); new_args.tickers = ticker_batch.to_vec();
new_args.tickers = ticker_batch.to_vec(); new_args
batch_download_args.push(new_args); })
} .collect::<Vec<JPMaQSDownloadGetIndicatorArgs>>();
// print total number of download sessions // print total number of download sessions
println!( println!(
"Total number of download sessions: {}", "Total number of download sessions: {}",
batch_download_args.len() batch_download_args.len()
); );
// Start timer // Start timer
let start_time = std::time::Instant::now(); let start_time = std::time::Instant::now();
let total_batches = batch_download_args.len(); let total_batches = batch_download_args.len();
// download each batch and save as csv // download each batch and save as csv
for (index, batch_args) in batch_download_args.iter().enumerate() { for (index, batch_args) in batch_download_args.iter().enumerate() {
let df_list = self.get_indicators_list(batch_args.clone())?; let df_list = self.get_indicators_list(batch_args.clone())?;
let total_indicators = df_list.len(); let total_indicators = df_list.len();
save_indicators_list_as_csvs(df_list, &folder_path)?; save_indicators_list_as_csvs(df_list, save_path.clone())?;
println!("Saved {} indicators to {}", total_indicators, folder_path); println!("Saved {} indicators to {}", total_indicators, folder_path);
// Calculate and print estimated remaining time // Calculate and print estimated remaining time
let elapsed = start_time.elapsed(); let elapsed = start_time.elapsed();
let average_time_per_batch = elapsed / (index as u32 + 1); let average_time_per_batch = elapsed / (index as u32 + 1);
let remaining_batches = total_batches - index - 1; let remaining_batches = total_batches - index - 1;
let estimated_remaining = average_time_per_batch * remaining_batches as u32; let estimated_remaining = average_time_per_batch * remaining_batches as u32;
println!( println!(
"Progress: {}/{}. Estimated time remaining: {:.2?}", "Progress: {}/{}. Estimated time remaining: {:.2?}",
index + 1, index + 1,
@ -247,15 +250,14 @@ impl JPMaQSDownload {
estimated_remaining estimated_remaining
); );
} }
Ok(()) Ok(())
} }
} }
fn save_indicators_list_as_csvs( fn save_indicators_list_as_csvs(
indicators: Vec<JPMaQSIndicator>, indicators: Vec<JPMaQSIndicator>,
folder_path: &str, folder_path: String,
) -> Result<(), Box<dyn Error>> { ) -> Result<(), Box<dyn Error>> {
for indicator in indicators { for indicator in indicators {
let ticker_folder = format!("{}/{}", folder_path, indicator.get_xcat()); let ticker_folder = format!("{}/{}", folder_path, indicator.get_xcat());