working with recursive

This commit is contained in:
Palash Tyagi 2024-11-07 20:26:50 +00:00
parent b14a59b259
commit 63c1cf64b2
5 changed files with 196 additions and 317 deletions

360
Cargo.lock generated
View File

@ -77,9 +77,9 @@ dependencies = [
[[package]]
name = "anyhow"
version = "1.0.92"
version = "1.0.93"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "74f37166d7d48a0284b99dd824694c26119c700b53bf0d1540cdb147dbdaaf13"
checksum = "4c95c10ba0b00a02636238b814946408b1322d5ac4760326e6fb8ec956d85775"
[[package]]
name = "argminmax"
@ -180,27 +180,15 @@ dependencies = [
"miniz_oxide",
"object",
"rustc-demangle",
"windows-targets 0.52.6",
"windows-targets",
]
[[package]]
name = "base64"
version = "0.21.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
[[package]]
name = "base64"
version = "0.22.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
[[package]]
name = "bitflags"
version = "1.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
[[package]]
name = "bitflags"
version = "2.6.0"
@ -333,7 +321,7 @@ dependencies = [
"iana-time-zone",
"num-traits",
"serde",
"windows-targets 0.52.6",
"windows-targets",
]
[[package]]
@ -492,7 +480,7 @@ version = "0.27.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f476fe445d41c9e991fd07515a6f463074b782242ccf4a5b7b1d1012e70824df"
dependencies = [
"bitflags 2.6.0",
"bitflags",
"crossterm_winapi",
"libc",
"parking_lot",
@ -805,25 +793,6 @@ version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
[[package]]
name = "h2"
version = "0.3.26"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8"
dependencies = [
"bytes",
"fnv",
"futures-core",
"futures-sink",
"futures-util",
"http 0.2.12",
"indexmap",
"slab",
"tokio",
"tokio-util",
"tracing",
]
[[package]]
name = "h2"
version = "0.4.6"
@ -835,7 +804,7 @@ dependencies = [
"fnv",
"futures-core",
"futures-sink",
"http 1.1.0",
"http",
"indexmap",
"slab",
"tokio",
@ -921,17 +890,6 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "http"
version = "0.2.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1"
dependencies = [
"bytes",
"fnv",
"itoa",
]
[[package]]
name = "http"
version = "1.1.0"
@ -943,17 +901,6 @@ dependencies = [
"itoa",
]
[[package]]
name = "http-body"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2"
dependencies = [
"bytes",
"http 0.2.12",
"pin-project-lite",
]
[[package]]
name = "http-body"
version = "1.0.1"
@ -961,7 +908,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
dependencies = [
"bytes",
"http 1.1.0",
"http",
]
[[package]]
@ -972,8 +919,8 @@ checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f"
dependencies = [
"bytes",
"futures-util",
"http 1.1.0",
"http-body 1.0.1",
"http",
"http-body",
"pin-project-lite",
]
@ -983,42 +930,12 @@ version = "1.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946"
[[package]]
name = "httpdate"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
[[package]]
name = "humantime"
version = "2.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
[[package]]
name = "hyper"
version = "0.14.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c08302e8fa335b151b788c775ff56e7a03ae64ff85c548ee820fecb70356e85"
dependencies = [
"bytes",
"futures-channel",
"futures-core",
"futures-util",
"h2 0.3.26",
"http 0.2.12",
"http-body 0.4.6",
"httparse",
"httpdate",
"itoa",
"pin-project-lite",
"socket2",
"tokio",
"tower-service",
"tracing",
"want",
]
[[package]]
name = "hyper"
version = "1.5.0"
@ -1028,9 +945,9 @@ dependencies = [
"bytes",
"futures-channel",
"futures-util",
"h2 0.4.6",
"http 1.1.0",
"http-body 1.0.1",
"h2",
"http",
"http-body",
"httparse",
"itoa",
"pin-project-lite",
@ -1046,8 +963,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333"
dependencies = [
"futures-util",
"http 1.1.0",
"hyper 1.5.0",
"http",
"hyper",
"hyper-util",
"rustls",
"rustls-native-certs",
@ -1059,15 +976,18 @@ dependencies = [
[[package]]
name = "hyper-tls"
version = "0.5.0"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0"
dependencies = [
"bytes",
"hyper 0.14.31",
"http-body-util",
"hyper",
"hyper-util",
"native-tls",
"tokio",
"tokio-native-tls",
"tower-service",
]
[[package]]
@ -1079,9 +999,9 @@ dependencies = [
"bytes",
"futures-channel",
"futures-util",
"http 1.1.0",
"http-body 1.0.1",
"hyper 1.5.0",
"http",
"http-body",
"hyper",
"pin-project-lite",
"socket2",
"tokio",
@ -1242,9 +1162,10 @@ name = "macrosynergy_dataquery"
version = "0.0.1"
dependencies = [
"anyhow",
"log",
"polars",
"rand",
"reqwest 0.11.27",
"reqwest",
"serde",
"serde_json",
"serde_urlencoded",
@ -1405,19 +1326,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6da452820c715ce78221e8202ccc599b4a52f3e1eb3eedb487b680c81a8e3f3"
dependencies = [
"async-trait",
"base64 0.22.1",
"base64",
"bytes",
"chrono",
"futures",
"humantime",
"hyper 1.5.0",
"hyper",
"itertools",
"md-5",
"parking_lot",
"percent-encoding",
"quick-xml",
"rand",
"reqwest 0.12.9",
"reqwest",
"ring",
"serde",
"serde_json",
@ -1440,7 +1361,7 @@ version = "0.10.68"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5"
dependencies = [
"bitflags 2.6.0",
"bitflags",
"cfg-if",
"foreign-types",
"libc",
@ -1498,7 +1419,7 @@ dependencies = [
"libc",
"redox_syscall",
"smallvec",
"windows-targets 0.52.6",
"windows-targets",
]
[[package]]
@ -1672,7 +1593,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd5df9b55e614088a3270b06f8649dce76537c268d6b1ca4d9c37008b2be5949"
dependencies = [
"ahash",
"bitflags 2.6.0",
"bitflags",
"bytemuck",
"chrono",
"chrono-tz",
@ -1721,7 +1642,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea1b431ed816cba1120cff200f06b962748001bbb2e615ce53cfbbdf701cc136"
dependencies = [
"ahash",
"bitflags 2.6.0",
"bitflags",
"hashbrown 0.15.0",
"num-traits",
"once_cell",
@ -1774,7 +1695,7 @@ dependencies = [
"pyo3",
"rayon",
"regex",
"reqwest 0.12.9",
"reqwest",
"ryu",
"serde",
"serde_json",
@ -1813,7 +1734,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4a8ca74f42e7b47cad241b36b98d991cc7fbb51b8d0695a055eb937588d1f310"
dependencies = [
"ahash",
"bitflags 2.6.0",
"bitflags",
"memchr",
"once_cell",
"polars-arrow",
@ -1862,7 +1783,7 @@ checksum = "035c800fbe5bbd820afeb8313713ed345853bb014e0f821a4025d40cf0d60e1a"
dependencies = [
"ahash",
"argminmax",
"base64 0.22.1",
"base64",
"bytemuck",
"chrono",
"chrono-tz",
@ -1895,7 +1816,7 @@ checksum = "91dcf1d9f048079376949eaf2e24e240b313ff4a102fb83b57c9a5f807cdca52"
dependencies = [
"ahash",
"async-stream",
"base64 0.22.1",
"base64",
"brotli",
"bytemuck",
"ethnum",
@ -1958,7 +1879,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23de436f33f4d1134c58f24e7059a221b957ec20730807e0ef0c80c8e4b3d06a"
dependencies = [
"ahash",
"bitflags 2.6.0",
"bitflags",
"bytemuck",
"bytes",
"chrono",
@ -2322,7 +2243,7 @@ version = "11.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ab240315c661615f2ee9f0f2cd32d5a7343a84d5ebcccb99d46e6637565e7b0"
dependencies = [
"bitflags 2.6.0",
"bitflags",
]
[[package]]
@ -2371,7 +2292,7 @@ version = "0.5.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f"
dependencies = [
"bitflags 2.6.0",
"bitflags",
]
[[package]]
@ -2425,20 +2346,24 @@ checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
[[package]]
name = "reqwest"
version = "0.11.27"
version = "0.12.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62"
checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f"
dependencies = [
"base64 0.21.7",
"base64",
"bytes",
"encoding_rs",
"futures-channel",
"futures-core",
"futures-util",
"h2 0.3.26",
"http 0.2.12",
"http-body 0.4.6",
"hyper 0.14.31",
"h2",
"http",
"http-body",
"http-body-util",
"hyper",
"hyper-rustls",
"hyper-tls",
"hyper-util",
"ipnet",
"js-sys",
"log",
@ -2447,56 +2372,18 @@ dependencies = [
"once_cell",
"percent-encoding",
"pin-project-lite",
"rustls-pemfile 1.0.4",
"serde",
"serde_json",
"serde_urlencoded",
"sync_wrapper 0.1.2",
"system-configuration",
"tokio",
"tokio-native-tls",
"tower-service",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
"winreg",
]
[[package]]
name = "reqwest"
version = "0.12.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a77c62af46e79de0a562e1a9849205ffcb7fc1238876e9bd743357570e04046f"
dependencies = [
"base64 0.22.1",
"bytes",
"futures-core",
"futures-util",
"h2 0.4.6",
"http 1.1.0",
"http-body 1.0.1",
"http-body-util",
"hyper 1.5.0",
"hyper-rustls",
"hyper-util",
"ipnet",
"js-sys",
"log",
"mime",
"once_cell",
"percent-encoding",
"pin-project-lite",
"quinn",
"rustls",
"rustls-native-certs",
"rustls-pemfile 2.2.0",
"rustls-pemfile",
"rustls-pki-types",
"serde",
"serde_json",
"serde_urlencoded",
"sync_wrapper 1.0.1",
"sync_wrapper",
"system-configuration",
"tokio",
"tokio-native-tls",
"tokio-rustls",
"tokio-util",
"tower-service",
@ -2541,7 +2428,7 @@ version = "0.38.38"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa260229e6538e52293eeb577aabd09945a09d6d9cc0fc550ed7529056c2e32a"
dependencies = [
"bitflags 2.6.0",
"bitflags",
"errno",
"libc",
"linux-raw-sys",
@ -2569,21 +2456,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fcaf18a4f2be7326cd874a5fa579fae794320a0f388d365dca7e480e55f83f8a"
dependencies = [
"openssl-probe",
"rustls-pemfile 2.2.0",
"rustls-pemfile",
"rustls-pki-types",
"schannel",
"security-framework",
]
[[package]]
name = "rustls-pemfile"
version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c"
dependencies = [
"base64 0.21.7",
]
[[package]]
name = "rustls-pemfile"
version = "2.2.0"
@ -2652,7 +2530,7 @@ version = "2.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02"
dependencies = [
"bitflags 2.6.0",
"bitflags",
"core-foundation",
"core-foundation-sys",
"libc",
@ -2912,12 +2790,6 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "sync_wrapper"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
[[package]]
name = "sync_wrapper"
version = "1.0.1"
@ -2942,20 +2814,20 @@ dependencies = [
[[package]]
name = "system-configuration"
version = "0.5.1"
version = "0.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7"
checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b"
dependencies = [
"bitflags 1.3.2",
"bitflags",
"core-foundation",
"system-configuration-sys",
]
[[package]]
name = "system-configuration-sys"
version = "0.5.0"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9"
checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4"
dependencies = [
"core-foundation-sys",
"libc",
@ -3391,7 +3263,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12342cb4d8e3b046f3d80effd474a7a02447231330ef77d71daa6fbc40681143"
dependencies = [
"windows-core 0.57.0",
"windows-targets 0.52.6",
"windows-targets",
]
[[package]]
@ -3400,7 +3272,7 @@ version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
dependencies = [
"windows-targets 0.52.6",
"windows-targets",
]
[[package]]
@ -3412,7 +3284,7 @@ dependencies = [
"windows-implement",
"windows-interface",
"windows-result 0.1.2",
"windows-targets 0.52.6",
"windows-targets",
]
[[package]]
@ -3445,7 +3317,7 @@ checksum = "e400001bb720a623c1c69032f8e3e4cf09984deec740f007dd2b03ec864804b0"
dependencies = [
"windows-result 0.2.0",
"windows-strings",
"windows-targets 0.52.6",
"windows-targets",
]
[[package]]
@ -3454,7 +3326,7 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8"
dependencies = [
"windows-targets 0.52.6",
"windows-targets",
]
[[package]]
@ -3463,7 +3335,7 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d1043d8214f791817bab27572aaa8af63732e11bf84aa21a45a78d6c317ae0e"
dependencies = [
"windows-targets 0.52.6",
"windows-targets",
]
[[package]]
@ -3473,16 +3345,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4cd9b125c486025df0eabcb585e62173c6c9eddcec5d117d3b6e8c30e2ee4d10"
dependencies = [
"windows-result 0.2.0",
"windows-targets 0.52.6",
]
[[package]]
name = "windows-sys"
version = "0.48.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
dependencies = [
"windows-targets 0.48.5",
"windows-targets",
]
[[package]]
@ -3491,7 +3354,7 @@ version = "0.52.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
dependencies = [
"windows-targets 0.52.6",
"windows-targets",
]
[[package]]
@ -3500,22 +3363,7 @@ version = "0.59.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
dependencies = [
"windows-targets 0.52.6",
]
[[package]]
name = "windows-targets"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
dependencies = [
"windows_aarch64_gnullvm 0.48.5",
"windows_aarch64_msvc 0.48.5",
"windows_i686_gnu 0.48.5",
"windows_i686_msvc 0.48.5",
"windows_x86_64_gnu 0.48.5",
"windows_x86_64_gnullvm 0.48.5",
"windows_x86_64_msvc 0.48.5",
"windows-targets",
]
[[package]]
@ -3524,46 +3372,28 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
dependencies = [
"windows_aarch64_gnullvm 0.52.6",
"windows_aarch64_msvc 0.52.6",
"windows_i686_gnu 0.52.6",
"windows_aarch64_gnullvm",
"windows_aarch64_msvc",
"windows_i686_gnu",
"windows_i686_gnullvm",
"windows_i686_msvc 0.52.6",
"windows_x86_64_gnu 0.52.6",
"windows_x86_64_gnullvm 0.52.6",
"windows_x86_64_msvc 0.52.6",
"windows_i686_msvc",
"windows_x86_64_gnu",
"windows_x86_64_gnullvm",
"windows_x86_64_msvc",
]
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
[[package]]
name = "windows_aarch64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
[[package]]
name = "windows_aarch64_msvc"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
[[package]]
name = "windows_aarch64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
[[package]]
name = "windows_i686_gnu"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
[[package]]
name = "windows_i686_gnu"
version = "0.52.6"
@ -3576,64 +3406,30 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
[[package]]
name = "windows_i686_msvc"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
[[package]]
name = "windows_i686_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
[[package]]
name = "windows_x86_64_gnu"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
[[package]]
name = "windows_x86_64_gnu"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
[[package]]
name = "windows_x86_64_gnullvm"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
[[package]]
name = "windows_x86_64_msvc"
version = "0.48.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
[[package]]
name = "windows_x86_64_msvc"
version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
[[package]]
name = "winreg"
version = "0.50.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1"
dependencies = [
"cfg-if",
"windows-sys 0.48.0",
]
[[package]]
name = "xxhash-rust"
version = "0.8.12"

View File

@ -4,11 +4,12 @@ version = "0.0.1"
edition = "2021"
[dependencies]
reqwest = { version = "0.11", features = ["blocking", "json"] }
reqwest = { version = "0.12.9", features = ["blocking", "json"] }
serde_json = "1.0"
serde_urlencoded = "0.7"
serde = { version = "1.0", features = ["derive"] }
polars = { version = "0.44.2", features = ["temporal", "lazy"] }
anyhow = "1.0"
anyhow = "1.0.92"
rand = "0.8"
threadpool = "1.8.1"
log = "0.4.22"

View File

@ -1,8 +1,11 @@
use log;
use macrosynergy_dataquery::oauth_client::OAuthClient;
use macrosynergy_dataquery::requester;
use macrosynergy_dataquery::requester::*;
use macrosynergy_dataquery::timeseries::*;
use rand::seq::SliceRandom;
use std::collections::HashSet;
use std::time::Instant;
fn main() {
let mut requester = requester::DQRequester::new(OAuthClient::default());
@ -10,7 +13,6 @@ fn main() {
let responses = requester.get_catalogue("JPMAQS", 1000).unwrap();
let mut tickers_vec = vec![];
for rsp in responses {
let response: DQCatalogueResponse = serde_json::from_str(&rsp.to_string()).unwrap();
@ -41,12 +43,31 @@ fn main() {
}
}
let rsp = requester.get_timeseries(DQTimeseriesRequestArgs {
expressions: expressions_vec[0..40].to_vec(),
..Default::default()
});
// let rsp = requester.get_timeseries(DQTimeseriesRequestArgs {
// expressions: expressions_vec[0..40].to_vec(),
// ..Default::default()
// });
for response in rsp {
// for response in rsp {
// let dq_response: DQTimeSeriesResponse =
// serde_json::from_str(&response.text().unwrap()).unwrap();
// for ts_group in dq_response.get_timeseries_by_ticker() {
// let jpmi = JPMaQSIndicator::new(ts_group).unwrap();
// println!("{:?}", jpmi.as_qdf().unwrap());
// }
// }
// fetch first 50 expressions
let start = Instant::now();
let responses = requester
.get_timeseries(DQTimeseriesRequestArgs {
expressions: expressions_vec[0..50].to_vec(),
..Default::default()
})
.unwrap();
for response in responses {
let dq_response: DQTimeSeriesResponse =
serde_json::from_str(&response.text().unwrap()).unwrap();
for ts_group in dq_response.get_timeseries_by_ticker() {

View File

@ -167,10 +167,14 @@ impl DQRequester {
Ok(responses)
}
pub fn get_single_timeseries_batch(
pub fn _fetch_single_timeseries_batch(
&mut self,
args: DQTimeseriesRequestArgs,
) -> Result<reqwest::blocking::Response, Box<dyn Error>> {
log::info!(
"Fetching timeseries batch with {} expressions",
args.expressions.len()
);
if args.expressions.len() < 1 || args.expressions.len() > 20 {
return Err("Number of expressions must be between 1 and 20".into());
}
@ -179,49 +183,94 @@ impl DQRequester {
let endpoint = format!("{}?{}", TIMESERIES_ENDPOINT, query_string);
let response = self._request(reqwest::Method::GET, &endpoint)?;
log::info!("Got response: {:?}", response.status());
Ok(response)
}
pub fn get_single_timeseries_with_defaults(
&mut self,
expressions: Vec<&str>,
) -> Result<reqwest::blocking::Response, Box<dyn Error>> {
let args = DQTimeseriesRequestArgs {
expressions: expressions.into_iter().map(|s| s.to_string()).collect(),
..Default::default()
};
// pub fn get_timeseries(
// &mut self,
// args: DQTimeseriesRequestArgs,
// ) -> Vec<reqwest::blocking::Response> {
// let expressions_chunks: Vec<Vec<String>> = args
// .expressions
// .chunks(20)
// .map(|chunk| chunk.to_vec())
// .collect();
self.get_single_timeseries_batch(args)
}
// let mut responses: Vec<Result<reqwest::blocking::Response, Box<dyn Error>>> = Vec::new();
pub fn get_timeseries(
// for expressions in expressions_chunks {
// let mut args = args.clone();
// args.update_expressions(expressions);
// let response = self.get_single_timeseries_batch(args);
// responses.push(response);
// // just sleep for a bit not threadsleep
// // println!("Sleeping for 500ms");
// std::thread::sleep(std::time::Duration::from_millis(500));
// }
// responses.into_iter().map(|r| r.unwrap()).collect()
// }
fn _fetch_timeseries_recursive(
&mut self,
args: DQTimeseriesRequestArgs,
) -> Vec<reqwest::blocking::Response> {
let expressions_chunks: Vec<Vec<String>> = args
max_retries: u32,
) -> Result<Vec<reqwest::blocking::Response>, Box<dyn Error>> {
let expression_batches: Vec<Vec<String>> = args
.expressions
.chunks(20)
.map(|chunk| chunk.to_vec())
.collect();
let mut responses: Vec<Result<reqwest::blocking::Response, Box<dyn Error>>> = Vec::new();
let mut okay_responses: Vec<reqwest::blocking::Response> = Vec::new();
let mut failed_batches: Vec<Vec<String>> = Vec::new();
for expressions in expressions_chunks {
for expr_batch in expression_batches {
let mut args = args.clone();
args.update_expressions(expressions);
println!(
"Requesting timeseries for expressions: {:?}",
args.expressions
);
let response = self.get_single_timeseries_batch(args);
responses.push(response);
// just sleep for a bit not threadsleep
println!("Sleeping for 500ms");
std::thread::sleep(std::time::Duration::from_millis(500));
args.update_expressions(expr_batch.clone());
let response = self._fetch_single_timeseries_batch(args);
std::thread::sleep(std::time::Duration::from_millis(200));
match response {
Ok(r) => {
okay_responses.push(r);
log::info!("Got batch: {:?}", expr_batch);
}
Err(e) => {
failed_batches.push(expr_batch.clone());
log::error!("Failed to get batch: {:?} : {:?}", expr_batch, e);
}
}
}
responses.into_iter().map(|r| r.unwrap()).collect()
if !failed_batches.is_empty() && max_retries == 0 {
return Err("Max retries reached".into());
}
if !failed_batches.is_empty() && max_retries > 0 {
log::info!("Retrying failed batches");
let mut new_args = args.clone();
new_args.update_expressions(failed_batches.concat());
log::info!("Retrying with {} failed expressions", failed_batches.len());
let mut retry_responses =
self._fetch_timeseries_recursive(new_args, max_retries - 1)?;
okay_responses.append(&mut retry_responses);
}
log::info!("Returning {} responses", okay_responses.len());
Ok(okay_responses)
}
pub fn get_timeseries(
&mut self,
args: DQTimeseriesRequestArgs,
) -> Result<Vec<reqwest::blocking::Response>, Box<dyn Error>> {
let max_retries = 5;
println!(
"Invoking recursive function for {:?} expressions",
args.expressions.len()
);
self._fetch_timeseries_recursive(args, max_retries)
}
}

View File

@ -108,6 +108,18 @@ impl DQTimeSeries {
}
impl DQTimeSeriesResponse {
pub fn list_expressions(&self) -> Vec<String> {
self.instruments
.iter()
.flat_map(|instrument| {
instrument
.attributes
.iter()
.map(|attribute| attribute.expression.clone())
})
.collect()
}
/// Return a list of all DQTimeSeries in the response
pub fn get_all_timeseries(&self) -> Vec<DQTimeSeries> {
self.instruments