magnus167/msyrs
1
0
Fork 0
mirror of https://github.com/Magnus167/msyrs.git synced 2025-08-20 04:20:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: update folder path handling in JPMaQSDownload

Browse Source
This commit is contained in:
Palash Tyagi 2024-11-13 00:57:11 +00:00
parent 6cf7a2f174
commit a091ec2b88
1 changed files with 19 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
src/download/jpmaqsdownload.rs
View File

@ -206,40 +206,43 @@ impl JPMaQSDownload {
folder_path: &str,
) -> Result<(), Box<dyn Error>> {
// if the folder path does not exist, create it
std::fs::create_dir_all(folder_path)?;
// let save_path = folder_path.to_string() + "/JPMaQSData/";
let save_path = format!("{}/JPMaQSData/", folder_path);
std::fs::create_dir_all(save_path.clone())?;
// get ticker count, and split into chunks of batch_size
let ticker_batches = download_args.tickers.chunks(batch_size);
// let batch_download_args -- create new download args for each batch with new tickers
let mut batch_download_args = Vec::new();
for ticker_batch in ticker_batches {
let mut new_args = download_args.clone();
new_args.tickers = ticker_batch.to_vec();
batch_download_args.push(new_args);
}
let batch_download_args = ticker_batches
.map(|ticker_batch| {
let mut new_args = download_args.clone();
new_args.tickers = ticker_batch.to_vec();
new_args
})
.collect::<Vec<JPMaQSDownloadGetIndicatorArgs>>();
// print total number of download sessions
println!(
"Total number of download sessions: {}",
batch_download_args.len()
);
// Start timer
let start_time = std::time::Instant::now();
let total_batches = batch_download_args.len();
// download each batch and save as csv
for (index, batch_args) in batch_download_args.iter().enumerate() {
let df_list = self.get_indicators_list(batch_args.clone())?;
let total_indicators = df_list.len();
save_indicators_list_as_csvs(df_list, &folder_path)?;
save_indicators_list_as_csvs(df_list, save_path.clone())?;
println!("Saved {} indicators to {}", total_indicators, folder_path);
// Calculate and print estimated remaining time
let elapsed = start_time.elapsed();
let average_time_per_batch = elapsed / (index as u32 + 1);
let remaining_batches = total_batches - index - 1;
let estimated_remaining = average_time_per_batch * remaining_batches as u32;
println!(
"Progress: {}/{}. Estimated time remaining: {:.2?}",
index + 1,
@ -247,15 +250,14 @@ impl JPMaQSDownload {
estimated_remaining
);
}
Ok(())
}
}
fn save_indicators_list_as_csvs(
indicators: Vec<JPMaQSIndicator>,
folder_path: &str,
folder_path: String,
) -> Result<(), Box<dyn Error>> {
for indicator in indicators {
let ticker_folder = format!("{}/{}", folder_path, indicator.get_xcat());