working

src/download/jpmaqsdownload.rs

@@ -1,7 +1,8 @@
 use crate::download::oauth_client::OAuthClient;
 use crate::download::requester::DQRequester;
-use crate::download::timeseries::DQTimeseriesRequestArgs;
+use crate::download::timeseries::DQTimeSeries;
 use crate::download::timeseries::DQTimeSeriesResponse;
+use crate::download::timeseries::DQTimeseriesRequestArgs;
 use crate::download::timeseries::JPMaQSIndicator;
 use rayon::prelude::*;
 use std::error::Error;
@@ -92,10 +93,13 @@ impl JPMaQSDownload {
         &mut self,
         expressions: Vec<String>,
     ) -> Result<Vec<DQTimeSeriesResponse>, Box<dyn Error>> {
-        let dqts_vec = self.requester.get_timeseries(DQTimeseriesRequestArgs {
-            expressions: expressions,
-            ..Default::default()
-        }).unwrap();
+        let dqts_vec = self
+            .requester
+            .get_timeseries(DQTimeseriesRequestArgs {
+                expressions: expressions,
+                ..Default::default()
+            })
+            .unwrap();
 
         Ok(dqts_vec)
     }
@@ -113,35 +117,50 @@ impl JPMaQSDownload {
         let dqts_vec = self.get_expressions(expressions)?;
 
         // println!("Retrieved {} time series", -- sum[dqts_vec.iter().map(|dqts| dqts.len())]);
-        println!(
-            "Retrieved {} time series",
-            dqts_vec
-                .iter()
-                .map(|dqts| dqts.list_expressions().len())
-                .sum::<usize>()
-        );
-
+        println!("Retrieved all time series",);
+        // say pausing for 30 seconds
+        println!("Pausing for 10 seconds");
+        std::thread::sleep(std::time::Duration::from_secs(10));
+        println!("Resuming");
         let start = std::time::Instant::now();
 
-        // let indicators = dqts_vec
-        //     .iter()
-        //     .flat_map(|dqts| dqts.get_timeseries_by_ticker())
-        //     .map(|tsv| JPMaQSIndicator::new(tsv))
-        //     .collect::<Result<Vec<JPMaQSIndicator>, Box<dyn Error>>>()?;
-        let indicators: Vec<_> = dqts_vec
-            .into_par_iter() // Use into_par_iter() to parallelize and consume dqts_vec
-            .flat_map(|dqts| {
-                dqts.get_timeseries_by_ticker()
-                    .into_par_iter()
-                    .filter_map(|tsv| JPMaQSIndicator::new(tsv).ok())
-            })
-            .collect();
+        let indicators = dq_response_vec_to_jpmaqs_indicators(dqts_vec);
 
         println!(
             "Converted time series to indicators in {:?}",
             start.elapsed()
         );
-
+        println!("Pausing for 10 seconds");
+        std::thread::sleep(std::time::Duration::from_secs(10));
+        println!("Resuming");
         Ok(indicators)
     }
 }
+
+// fn dq_response_vec_to_jpmaqs_indicators(
+//     dqts_vec: Vec<DQTimeSeriesResponse>,
+// ) -> Vec<JPMaQSIndicator> {
+//     let mut indicators: Vec<JPMaQSIndicator> = Vec::new();
+//     for dqts in dqts_vec {
+//         indicators.extend(
+//             dqts.consume_to_grouped_by_ticker() // moves the values to free up memory
+//                 .into_iter()
+//                 .filter_map(|tsv| JPMaQSIndicator::new(tsv).ok()),
+//         );
+//     }
+
+//     indicators
+// }
+fn dq_response_vec_to_jpmaqs_indicators(
+    dqts_vec: Vec<DQTimeSeriesResponse>,
+) -> Vec<JPMaQSIndicator> {
+    dqts_vec
+        .into_par_iter()
+        .flat_map(|dqts| {
+            dqts.consume_to_grouped_by_ticker()
+                .into_iter()
+                .filter_map(|tsv| JPMaQSIndicator::new(tsv).ok())
+                .collect::<Vec<JPMaQSIndicator>>()
+        })
+        .collect()
+}