magnus167/msyrs
1
0
Fork 0
mirror of https://github.com/Magnus167/msyrs.git synced 2025-08-20 13:00:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

refactor: adjust batch size for ticker downloads and simplify main function logic

Browse Source
This commit is contained in:
Palash Tyagi 2024-11-13 01:08:57 +00:00
parent a091ec2b88
commit d05618f106
2 changed files with 3 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
src/download/jpmaqsdownload.rs
View File

@ -202,7 +202,6 @@ impl JPMaQSDownload {
pub fn save_indicators_as_csv( pub fn save_indicators_as_csv(
&mut self, &mut self,
download_args: JPMaQSDownloadGetIndicatorArgs, download_args: JPMaQSDownloadGetIndicatorArgs,
batch_size: usize,
folder_path: &str, folder_path: &str,
) -> Result<(), Box<dyn Error>> { ) -> Result<(), Box<dyn Error>> {
// if the folder path does not exist, create it // if the folder path does not exist, create it
@ -211,7 +210,7 @@ impl JPMaQSDownload {
std::fs::create_dir_all(save_path.clone())?; std::fs::create_dir_all(save_path.clone())?;
// get ticker count, and split into chunks of batch_size // get ticker count, and split into chunks of batch_size
let ticker_batches = download_args.tickers.chunks(batch_size); let ticker_batches = download_args.tickers.chunks(500);
let batch_download_args = ticker_batches let batch_download_args = ticker_batches
.map(|ticker_batch| { .map(|ticker_batch| {
let mut new_args = download_args.clone(); let mut new_args = download_args.clone();

6
src/main.rs
View File

@ -1,5 +1,4 @@
use msyrs::download::jpmaqsdownload::{JPMaQSDownload, JPMaQSDownloadGetIndicatorArgs}; use msyrs::download::jpmaqsdownload::{JPMaQSDownload, JPMaQSDownloadGetIndicatorArgs};
use polars::prelude::*;
fn main() { fn main() {
println!("Authentication to DataQuery API"); println!("Authentication to DataQuery API");
@ -17,7 +16,7 @@ fn main() {
start.elapsed() start.elapsed()
); );
let num_ticks = tickers.len(); let num_ticks = 100;
let sel_tickers: Vec<String> = tickers let sel_tickers: Vec<String> = tickers
.iter() .iter()
.take(num_ticks) .take(num_ticks)
@ -27,13 +26,12 @@ fn main() {
println!("Retrieving indicators for {} tickers", sel_tickers.len()); println!("Retrieving indicators for {} tickers", sel_tickers.len());
start = std::time::Instant::now(); start = std::time::Instant::now();
let res = jpamqs_download.save_indicators_as_csv( let res = jpamqs_download.save_indicators_as_csv(
JPMaQSDownloadGetIndicatorArgs { JPMaQSDownloadGetIndicatorArgs {
tickers: sel_tickers.clone(), tickers: sel_tickers.clone(),
..Default::default() ..Default::default()
}, },
5000,
"./data/", "./data/",
); );