{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Build and install the package\n",
"\n",
"First patch `pyo3-polars`:\n",
"\n",
"- Use [this diff](https://github.com/pola-rs/pyo3-polars/compare/main...Magnus167:pyo3-polars:main) to make changes to the `pyo3-polars` package.\n",
"\n",
"Install the package:\n",
"\n",
"```bash\n",
"python -m venv .venv\n",
"\n",
"# source .venv/bin/activate\n",
"./.venv/Scripts/activate\n",
"\n",
"pip install maturin ipywidgets\n",
"\n",
"maturin develop --release\n",
"```\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Import Python packages\n"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import macrosynergy\n",
"import pandas as pd\n",
"import numpy as np\n",
"import polars as pl\n",
"import os"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Import Python bindings - `msyrs`\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"import msyrs"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"DATA_FOLDER_PATH = \"E:/Work/jpmaqs-data\"\n",
"# DATA_FOLDER_PATH = \"C:/Users/PalashTyagi/Code/go-dataquery/jpmaqs-data\"\n",
"DQ_CLIENT_ID = os.getenv(\"DQ_CLIENT_ID\")\n",
"DQ_CLIENT_SECRET = os.getenv(\"DQ_CLIENT_SECRET\")"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"import time\n",
"\n",
"nb_start_time = time.time()"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Time taken to load qdf: 0.0040130615234375\n"
]
},
{
"data": {
"text/html": [
"
\n",
"
shape: (5, 7)real_date | cid | xcat | value | grading | eop_lag | mop_lag |
---|
date | str | str | f64 | f64 | i64 | i64 |
2010-03-03 | "USD" | "ADPEMPL_SA_P1M1ML1" | -0.173806 | 3.0 | 3 | 33 |
2010-03-04 | "USD" | "ADPEMPL_SA_P1M1ML1" | -0.173806 | 3.0 | 4 | 34 |
2010-03-05 | "USD" | "ADPEMPL_SA_P1M1ML1" | -0.173806 | 3.0 | 5 | 35 |
2010-03-08 | "USD" | "ADPEMPL_SA_P1M1ML1" | -0.173806 | 3.0 | 8 | 38 |
2010-03-09 | "USD" | "ADPEMPL_SA_P1M1ML1" | -0.173806 | 3.0 | 9 | 39 |
"
],
"text/plain": [
"shape: (5, 7)\n",
"┌────────────┬─────┬────────────────────┬───────────┬─────────┬─────────┬─────────┐\n",
"│ real_date ┆ cid ┆ xcat ┆ value ┆ grading ┆ eop_lag ┆ mop_lag │\n",
"│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n",
"│ date ┆ str ┆ str ┆ f64 ┆ f64 ┆ i64 ┆ i64 │\n",
"╞════════════╪═════╪════════════════════╪═══════════╪═════════╪═════════╪═════════╡\n",
"│ 2010-03-03 ┆ USD ┆ ADPEMPL_SA_P1M1ML1 ┆ -0.173806 ┆ 3.0 ┆ 3 ┆ 33 │\n",
"│ 2010-03-04 ┆ USD ┆ ADPEMPL_SA_P1M1ML1 ┆ -0.173806 ┆ 3.0 ┆ 4 ┆ 34 │\n",
"│ 2010-03-05 ┆ USD ┆ ADPEMPL_SA_P1M1ML1 ┆ -0.173806 ┆ 3.0 ┆ 5 ┆ 35 │\n",
"│ 2010-03-08 ┆ USD ┆ ADPEMPL_SA_P1M1ML1 ┆ -0.173806 ┆ 3.0 ┆ 8 ┆ 38 │\n",
"│ 2010-03-09 ┆ USD ┆ ADPEMPL_SA_P1M1ML1 ┆ -0.173806 ┆ 3.0 ┆ 9 ┆ 39 │\n",
"└────────────┴─────┴────────────────────┴───────────┴─────────┴─────────┴─────────┘"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dfpath = f\"{DATA_FOLDER_PATH}/data/ADPEMPL_SA_P1M1ML1/USD_ADPEMPL_SA_P1M1ML1.csv\"\n",
"\n",
"starttime = time.time()\n",
"ldf: pl.DataFrame = msyrs.qdf.load_qdf(dfpath)\n",
"print(f\"Time taken to load qdf: {time.time() - starttime}\")\n",
"ldf.head(5)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"cids_dm = \"AUD.CAD.CHF.EUR.GBP.JPY.NOK.NZD.SEK.USD\".split(\".\")\n",
"cids_em = \"CLP.COP.CZK.HUF.IDR.ILS.INR.KRW.MXN.PLN.THB.TRY.TWD.ZAR\".split(\".\")\n",
"cids = cids_dm + cids_em\n",
"cids_dux = list(set(cids) - set([\"IDR\", \"NZD\"]))\n",
"ecos = \"CPIC_SA_P1M1ML12.CPIC_SJA_P3M3ML3AR.CPIC_SJA_P6M6ML6AR.CPIH_SA_P1M1ML12.CPIH_SJA_P3M3ML3AR.CPIH_SJA_P6M6ML6AR.INFTEFF_NSA.INTRGDP_NSA_P1M1ML12_3MMA.INTRGDPv5Y_NSA_P1M1ML12_3MMA.PCREDITGDP_SJA_D1M1ML12.RGDP_SA_P1Q1QL4_20QMA.RYLDIRS02Y_NSA.RYLDIRS05Y_NSA.PCREDITBN_SJA_P1M1ML12\".split(\n",
" \".\"\n",
")\n",
"\n",
"\n",
"mkts = \"DU02YXR_NSA.DU05YXR_NSA.DU02YXR_VT10.DU05YXR_VT10.EQXR_NSA.EQXR_VT10.FXXR_NSA.FXXR_VT10.FXCRR_NSA.FXTARGETED_NSA.FXUNTRADABLE_NSA\".split(\n",
" \".\"\n",
")\n",
"xcats = ecos + mkts\n",
"\n",
"tickers = [f\"{c}_{x}\" for c in cids for x in xcats]"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"
shape: (5, 7)real_date | cid | xcat | value | grading | eop_lag | mop_lag |
---|
date | str | str | f64 | f64 | f64 | f64 |
1990-04-26 | "AUD" | "CPIC_SA_P1M1ML12" | 6.434599 | 2.0 | 26.0 | 223.0 |
1990-04-27 | "AUD" | "CPIC_SA_P1M1ML12" | 6.434599 | 2.0 | 27.0 | 224.0 |
1990-04-30 | "AUD" | "CPIC_SA_P1M1ML12" | 6.434599 | 2.0 | 30.0 | 227.0 |
1990-05-01 | "AUD" | "CPIC_SA_P1M1ML12" | 6.434599 | 2.0 | 31.0 | 228.0 |
1990-05-02 | "AUD" | "CPIC_SA_P1M1ML12" | 6.434599 | 2.0 | 32.0 | 229.0 |
"
],
"text/plain": [
"shape: (5, 7)\n",
"┌────────────┬─────┬──────────────────┬──────────┬─────────┬─────────┬─────────┐\n",
"│ real_date ┆ cid ┆ xcat ┆ value ┆ grading ┆ eop_lag ┆ mop_lag │\n",
"│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n",
"│ date ┆ str ┆ str ┆ f64 ┆ f64 ┆ f64 ┆ f64 │\n",
"╞════════════╪═════╪══════════════════╪══════════╪═════════╪═════════╪═════════╡\n",
"│ 1990-04-26 ┆ AUD ┆ CPIC_SA_P1M1ML12 ┆ 6.434599 ┆ 2.0 ┆ 26.0 ┆ 223.0 │\n",
"│ 1990-04-27 ┆ AUD ┆ CPIC_SA_P1M1ML12 ┆ 6.434599 ┆ 2.0 ┆ 27.0 ┆ 224.0 │\n",
"│ 1990-04-30 ┆ AUD ┆ CPIC_SA_P1M1ML12 ┆ 6.434599 ┆ 2.0 ┆ 30.0 ┆ 227.0 │\n",
"│ 1990-05-01 ┆ AUD ┆ CPIC_SA_P1M1ML12 ┆ 6.434599 ┆ 2.0 ┆ 31.0 ┆ 228.0 │\n",
"│ 1990-05-02 ┆ AUD ┆ CPIC_SA_P1M1ML12 ┆ 6.434599 ┆ 2.0 ┆ 32.0 ┆ 229.0 │\n",
"└────────────┴─────┴──────────────────┴──────────┴─────────┴─────────┴─────────┘"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"downloaded_df: pl.DataFrame = msyrs.download.download_jpmaqs_indicators_as_df(\n",
" client_id=DQ_CLIENT_ID,\n",
" client_secret=DQ_CLIENT_SECRET,\n",
" tickers=tickers,\n",
")\n",
"downloaded_df.head(5)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"downloaded_df = None"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Downloading the JPMAQS catalogue from DataQuery...\n",
"Downloaded JPMAQS catalogue with 18711 tickers.\n",
"Removed 21/600 expressions that are not in the JPMaQS catalogue.\n",
"Downloading data from JPMaQS.\n",
"Timestamp UTC: 2024-11-22 20:35:07\n",
"Connection successful!\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Requesting data: 100%|██████████| 29/29 [00:05<00:00, 4.93it/s]\n",
"Downloading data: 100%|██████████| 29/29 [00:16<00:00, 1.80it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Some dates are missing from the downloaded data. \n",
"2 out of 9107 dates are missing.\n"
]
}
],
"source": [
"pddf = macrosynergy.download.JPMaQSDownload().download(\n",
" tickers=tickers,\n",
" get_catalogue=True,\n",
" show_progress=True,\n",
" start_date=\"1990-01-01\",\n",
")\n",
"pddf = macrosynergy.management.types.QuantamentalDataFrame(pddf)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Time taken to load qdf batch: 1.1382660865783691\n"
]
},
{
"data": {
"text/html": [
"\n",
"
shape: (5, 7)real_date | cid | xcat | value | grading | eop_lag | mop_lag |
---|
date | str | str | f64 | f64 | i64 | i64 |
1990-04-26 | "AUD" | "CPIC_SA_P1M1ML12" | 6.434599 | 2.0 | 26 | 223 |
1990-04-27 | "AUD" | "CPIC_SA_P1M1ML12" | 6.434599 | 2.0 | 27 | 224 |
1990-04-30 | "AUD" | "CPIC_SA_P1M1ML12" | 6.434599 | 2.0 | 30 | 227 |
1990-05-01 | "AUD" | "CPIC_SA_P1M1ML12" | 6.434599 | 2.0 | 31 | 228 |
1990-05-02 | "AUD" | "CPIC_SA_P1M1ML12" | 6.434599 | 2.0 | 32 | 229 |
"
],
"text/plain": [
"shape: (5, 7)\n",
"┌────────────┬─────┬──────────────────┬──────────┬─────────┬─────────┬─────────┐\n",
"│ real_date ┆ cid ┆ xcat ┆ value ┆ grading ┆ eop_lag ┆ mop_lag │\n",
"│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n",
"│ date ┆ str ┆ str ┆ f64 ┆ f64 ┆ i64 ┆ i64 │\n",
"╞════════════╪═════╪══════════════════╪══════════╪═════════╪═════════╪═════════╡\n",
"│ 1990-04-26 ┆ AUD ┆ CPIC_SA_P1M1ML12 ┆ 6.434599 ┆ 2.0 ┆ 26 ┆ 223 │\n",
"│ 1990-04-27 ┆ AUD ┆ CPIC_SA_P1M1ML12 ┆ 6.434599 ┆ 2.0 ┆ 27 ┆ 224 │\n",
"│ 1990-04-30 ┆ AUD ┆ CPIC_SA_P1M1ML12 ┆ 6.434599 ┆ 2.0 ┆ 30 ┆ 227 │\n",
"│ 1990-05-01 ┆ AUD ┆ CPIC_SA_P1M1ML12 ┆ 6.434599 ┆ 2.0 ┆ 31 ┆ 228 │\n",
"│ 1990-05-02 ┆ AUD ┆ CPIC_SA_P1M1ML12 ┆ 6.434599 ┆ 2.0 ┆ 32 ┆ 229 │\n",
"└────────────┴─────┴──────────────────┴──────────┴─────────┴─────────┴─────────┘"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"starttime = time.time()\n",
"\n",
"big_df: pl.DataFrame = msyrs.qdf.load_qdf_from_download_bank(\n",
"\n",
" folder_path=DATA_FOLDER_PATH,\n",
" xcats=xcats,\n",
"\n",
" # folder_path=DATA_FOLDER_PATH, cids=cids\n",
"\n",
")\n",
"print(f\"Time taken to load qdf batch: {time.time() - starttime}\")\n",
"\n",
"\n",
"big_df.head(5)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"275.7299613952637"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"big_df.estimated_size(\"mb\")"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" real_date | \n",
" cid | \n",
" xcat | \n",
" value | \n",
" grading | \n",
" eop_lag | \n",
" mop_lag | \n",
"
\n",
" \n",
" \n",
" \n",
" 0 | \n",
" 1990-04-26 | \n",
" AUD | \n",
" CPIC_SA_P1M1ML12 | \n",
" 6.434599 | \n",
" 2.0 | \n",
" 26 | \n",
" 223 | \n",
"
\n",
" \n",
" 1 | \n",
" 1990-04-27 | \n",
" AUD | \n",
" CPIC_SA_P1M1ML12 | \n",
" 6.434599 | \n",
" 2.0 | \n",
" 27 | \n",
" 224 | \n",
"
\n",
" \n",
" 2 | \n",
" 1990-04-30 | \n",
" AUD | \n",
" CPIC_SA_P1M1ML12 | \n",
" 6.434599 | \n",
" 2.0 | \n",
" 30 | \n",
" 227 | \n",
"
\n",
" \n",
" 3 | \n",
" 1990-05-01 | \n",
" AUD | \n",
" CPIC_SA_P1M1ML12 | \n",
" 6.434599 | \n",
" 2.0 | \n",
" 31 | \n",
" 228 | \n",
"
\n",
" \n",
" 4 | \n",
" 1990-05-02 | \n",
" AUD | \n",
" CPIC_SA_P1M1ML12 | \n",
" 6.434599 | \n",
" 2.0 | \n",
" 32 | \n",
" 229 | \n",
"
\n",
" \n",
" ... | \n",
" ... | \n",
" ... | \n",
" ... | \n",
" ... | \n",
" ... | \n",
" ... | \n",
" ... | \n",
"
\n",
" \n",
" 5285577 | \n",
" 2024-11-08 | \n",
" ZAR | \n",
" RYLDIRS05Y_NSA | \n",
" 3.097534 | \n",
" 1.0 | \n",
" 0 | \n",
" 0 | \n",
"
\n",
" \n",
" 5285578 | \n",
" 2024-11-12 | \n",
" ZAR | \n",
" RYLDIRS05Y_NSA | \n",
" 3.189884 | \n",
" 1.0 | \n",
" 0 | \n",
" 0 | \n",
"
\n",
" \n",
" 5285579 | \n",
" 2024-11-13 | \n",
" ZAR | \n",
" RYLDIRS05Y_NSA | \n",
" 3.108095 | \n",
" 1.0 | \n",
" 0 | \n",
" 0 | \n",
"
\n",
" \n",
" 5285580 | \n",
" 2024-11-14 | \n",
" ZAR | \n",
" RYLDIRS05Y_NSA | \n",
" 3.060968 | \n",
" 1.0 | \n",
" 0 | \n",
" 0 | \n",
"
\n",
" \n",
" 5285581 | \n",
" 2024-11-15 | \n",
" ZAR | \n",
" RYLDIRS05Y_NSA | \n",
" 3.102066 | \n",
" 1.0 | \n",
" 0 | \n",
" 0 | \n",
"
\n",
" \n",
"
\n",
"
5285582 rows × 7 columns
\n",
"
"
],
"text/plain": [
" real_date cid xcat value grading eop_lag mop_lag\n",
"0 1990-04-26 AUD CPIC_SA_P1M1ML12 6.434599 2.0 26 223\n",
"1 1990-04-27 AUD CPIC_SA_P1M1ML12 6.434599 2.0 27 224\n",
"2 1990-04-30 AUD CPIC_SA_P1M1ML12 6.434599 2.0 30 227\n",
"3 1990-05-01 AUD CPIC_SA_P1M1ML12 6.434599 2.0 31 228\n",
"4 1990-05-02 AUD CPIC_SA_P1M1ML12 6.434599 2.0 32 229\n",
"... ... ... ... ... ... ... ...\n",
"5285577 2024-11-08 ZAR RYLDIRS05Y_NSA 3.097534 1.0 0 0\n",
"5285578 2024-11-12 ZAR RYLDIRS05Y_NSA 3.189884 1.0 0 0\n",
"5285579 2024-11-13 ZAR RYLDIRS05Y_NSA 3.108095 1.0 0 0\n",
"5285580 2024-11-14 ZAR RYLDIRS05Y_NSA 3.060968 1.0 0 0\n",
"5285581 2024-11-15 ZAR RYLDIRS05Y_NSA 3.102066 1.0 0 0\n",
"\n",
"[5285582 rows x 7 columns]"
]
},
"execution_count": 28,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"big_df.to_pandas()"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"789.8839149475098"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"big_df.to_pandas().memory_usage(deep=True).sum() / 1024**2"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"\n",
"
\n",
" \n",
" \n",
" | \n",
" real_date | \n",
" cid | \n",
" xcat | \n",
" value | \n",
" grading | \n",
" eop_lag | \n",
" mop_lag | \n",
"
\n",
" \n",
" \n",
" \n",
" 0 | \n",
" 1990-04-26 | \n",
" AUD | \n",
" CPIC_SA_P1M1ML12 | \n",
" 6.434599 | \n",
" 2.0 | \n",
" 26 | \n",
" 223 | \n",
"
\n",
" \n",
" 1 | \n",
" 1990-04-27 | \n",
" AUD | \n",
" CPIC_SA_P1M1ML12 | \n",
" 6.434599 | \n",
" 2.0 | \n",
" 27 | \n",
" 224 | \n",
"
\n",
" \n",
" 2 | \n",
" 1990-04-30 | \n",
" AUD | \n",
" CPIC_SA_P1M1ML12 | \n",
" 6.434599 | \n",
" 2.0 | \n",
" 30 | \n",
" 227 | \n",
"
\n",
" \n",
" 3 | \n",
" 1990-05-01 | \n",
" AUD | \n",
" CPIC_SA_P1M1ML12 | \n",
" 6.434599 | \n",
" 2.0 | \n",
" 31 | \n",
" 228 | \n",
"
\n",
" \n",
" 4 | \n",
" 1990-05-02 | \n",
" AUD | \n",
" CPIC_SA_P1M1ML12 | \n",
" 6.434599 | \n",
" 2.0 | \n",
" 32 | \n",
" 229 | \n",
"
\n",
" \n",
" ... | \n",
" ... | \n",
" ... | \n",
" ... | \n",
" ... | \n",
" ... | \n",
" ... | \n",
" ... | \n",
"
\n",
" \n",
" 5285577 | \n",
" 2024-11-08 | \n",
" ZAR | \n",
" RYLDIRS05Y_NSA | \n",
" 3.097534 | \n",
" 1.0 | \n",
" 0 | \n",
" 0 | \n",
"
\n",
" \n",
" 5285578 | \n",
" 2024-11-12 | \n",
" ZAR | \n",
" RYLDIRS05Y_NSA | \n",
" 3.189884 | \n",
" 1.0 | \n",
" 0 | \n",
" 0 | \n",
"
\n",
" \n",
" 5285579 | \n",
" 2024-11-13 | \n",
" ZAR | \n",
" RYLDIRS05Y_NSA | \n",
" 3.108095 | \n",
" 1.0 | \n",
" 0 | \n",
" 0 | \n",
"
\n",
" \n",
" 5285580 | \n",
" 2024-11-14 | \n",
" ZAR | \n",
" RYLDIRS05Y_NSA | \n",
" 3.060968 | \n",
" 1.0 | \n",
" 0 | \n",
" 0 | \n",
"
\n",
" \n",
" 5285581 | \n",
" 2024-11-15 | \n",
" ZAR | \n",
" RYLDIRS05Y_NSA | \n",
" 3.102066 | \n",
" 1.0 | \n",
" 0 | \n",
" 0 | \n",
"
\n",
" \n",
"
\n",
"
5285582 rows × 7 columns
\n",
"
"
],
"text/plain": [
" real_date cid xcat value grading eop_lag mop_lag\n",
"0 1990-04-26 AUD CPIC_SA_P1M1ML12 6.434599 2.0 26 223\n",
"1 1990-04-27 AUD CPIC_SA_P1M1ML12 6.434599 2.0 27 224\n",
"2 1990-04-30 AUD CPIC_SA_P1M1ML12 6.434599 2.0 30 227\n",
"3 1990-05-01 AUD CPIC_SA_P1M1ML12 6.434599 2.0 31 228\n",
"4 1990-05-02 AUD CPIC_SA_P1M1ML12 6.434599 2.0 32 229\n",
"... ... ... ... ... ... ... ...\n",
"5285577 2024-11-08 ZAR RYLDIRS05Y_NSA 3.097534 1.0 0 0\n",
"5285578 2024-11-12 ZAR RYLDIRS05Y_NSA 3.189884 1.0 0 0\n",
"5285579 2024-11-13 ZAR RYLDIRS05Y_NSA 3.108095 1.0 0 0\n",
"5285580 2024-11-14 ZAR RYLDIRS05Y_NSA 3.060968 1.0 0 0\n",
"5285581 2024-11-15 ZAR RYLDIRS05Y_NSA 3.102066 1.0 0 0\n",
"\n",
"[5285582 rows x 7 columns]"
]
},
"execution_count": 31,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"macrosynergy.management.types.QuantamentalDataFrame(big_df.to_pandas())"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"211.71554374694824"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"macrosynergy.management.types.QuantamentalDataFrame(big_df.to_pandas()).memory_usage(\n",
" deep=True\n",
").sum() / 1024**2"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"sel_cids = [\"USD\", \"EUR\", \"GBP\", \"AUD\", \"CAD\"]\n",
"start = \"1990-01-01\""
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Time taken to reduce qdf: 0.2668800354003906\n"
]
},
{
"data": {
"text/html": [
"\n",
"
shape: (73_069, 7)real_date | cid | xcat | value | grading | eop_lag | mop_lag |
---|
date | str | str | f64 | f64 | i64 | i64 |
2000-05-04 | "AUD" | "EQXR_NSA" | -1.251605 | 1.0 | 0 | 0 |
2000-05-05 | "AUD" | "EQXR_NSA" | 1.787455 | 1.0 | 0 | 0 |
2000-05-08 | "AUD" | "EQXR_NSA" | -0.574713 | 1.0 | 0 | 0 |
2000-05-09 | "AUD" | "EQXR_NSA" | -0.931278 | 1.0 | 0 | 0 |
2000-05-10 | "AUD" | "EQXR_NSA" | -1.523501 | 1.0 | 0 | 0 |
… | … | … | … | … | … | … |
2024-11-08 | "USD" | "EQXR_VT10" | 0.313875 | 1.0 | 0 | 0 |
2024-11-12 | "USD" | "EQXR_VT10" | -0.272457 | 1.0 | 0 | 0 |
2024-11-13 | "USD" | "EQXR_VT10" | 0.043729 | 1.0 | 0 | 0 |
2024-11-14 | "USD" | "EQXR_VT10" | -0.549983 | 1.0 | 0 | 0 |
2024-11-15 | "USD" | "EQXR_VT10" | -1.198544 | 1.0 | 0 | 0 |
"
],
"text/plain": [
"shape: (73_069, 7)\n",
"┌────────────┬─────┬───────────┬───────────┬─────────┬─────────┬─────────┐\n",
"│ real_date ┆ cid ┆ xcat ┆ value ┆ grading ┆ eop_lag ┆ mop_lag │\n",
"│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n",
"│ date ┆ str ┆ str ┆ f64 ┆ f64 ┆ i64 ┆ i64 │\n",
"╞════════════╪═════╪═══════════╪═══════════╪═════════╪═════════╪═════════╡\n",
"│ 2000-05-04 ┆ AUD ┆ EQXR_NSA ┆ -1.251605 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2000-05-05 ┆ AUD ┆ EQXR_NSA ┆ 1.787455 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2000-05-08 ┆ AUD ┆ EQXR_NSA ┆ -0.574713 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2000-05-09 ┆ AUD ┆ EQXR_NSA ┆ -0.931278 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2000-05-10 ┆ AUD ┆ EQXR_NSA ┆ -1.523501 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n",
"│ 2024-11-08 ┆ USD ┆ EQXR_VT10 ┆ 0.313875 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2024-11-12 ┆ USD ┆ EQXR_VT10 ┆ -0.272457 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2024-11-13 ┆ USD ┆ EQXR_VT10 ┆ 0.043729 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2024-11-14 ┆ USD ┆ EQXR_VT10 ┆ -0.549983 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2024-11-15 ┆ USD ┆ EQXR_VT10 ┆ -1.198544 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"└────────────┴─────┴───────────┴───────────┴─────────┴─────────┴─────────┘"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"starttime = time.time()\n",
"eq_df = msyrs.qdf.reduce_dataframe(\n",
" df=big_df,\n",
" cids=sel_cids,\n",
" xcats=[\"EQXR_NSA\", \"EQXR_VT10\"],\n",
"\n",
" start=start,\n",
")\n",
"print(f\"Time taken to reduce qdf: {time.time() - starttime}\")\n",
"eq_df"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Time taken to reduce qdf: 0.057086944580078125\n"
]
}
],
"source": [
"starttime = time.time()\n",
"eq_pd_df = pddf.reduce_df(cids=sel_cids, xcats=[\"EQXR_NSA\", \"EQXR_VT10\"], start=start)\n",
"print(f\"Time taken to reduce qdf: {time.time() - starttime}\")"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Time taken to reduce qdf: 0.298140287399292\n"
]
}
],
"source": [
"fx_xcats = [xc for xc in xcats if xc.startswith(\"FX\")]\n",
"starttime = time.time()\n",
"\n",
"fx_df = msyrs.qdf.reduce_dataframe(\n",
" df=big_df, cids=sel_cids, start=start, xcats=fx_xcats, intersect=True\n",
")\n",
"print(f\"Time taken to reduce qdf: {time.time() - starttime}\")"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Time taken to reduce qdf: 0.08732080459594727\n"
]
}
],
"source": [
"starttime = time.time()\n",
"fx_pd_df = pddf.reduce_df(cids=sel_cids, xcats=fx_xcats, start=start, intersect=True)\n",
"print(f\"Time taken to reduce qdf: {time.time() - starttime}\")"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Time taken: 0.01651310920715332\n"
]
},
{
"data": {
"text/html": [
"\n",
"
shape: (10, 7)real_date | cid | xcat | value | grading | eop_lag | mop_lag |
---|
date | str | str | f64 | f64 | i64 | i64 |
2000-05-04 | "AUD" | "EQXR_NSA" | -1.251605 | 1.0 | 0 | 0 |
2000-05-05 | "AUD" | "EQXR_NSA" | 1.787455 | 1.0 | 0 | 0 |
2000-05-08 | "AUD" | "EQXR_NSA" | -0.574713 | 1.0 | 0 | 0 |
2000-05-09 | "AUD" | "EQXR_NSA" | -0.931278 | 1.0 | 0 | 0 |
2000-05-10 | "AUD" | "EQXR_NSA" | -1.523501 | 1.0 | 0 | 0 |
2000-05-11 | "AUD" | "EQXR_NSA" | -1.579987 | 1.0 | 0 | 0 |
2000-05-12 | "AUD" | "EQXR_NSA" | 1.80602 | 1.0 | 0 | 0 |
2000-05-15 | "AUD" | "EQXR_NSA" | 0.295664 | 1.0 | 0 | 0 |
2000-05-16 | "AUD" | "EQXR_NSA" | 1.310187 | 1.0 | 0 | 0 |
2000-05-17 | "AUD" | "EQXR_NSA" | -0.711284 | 1.0 | 0 | 0 |
"
],
"text/plain": [
"shape: (10, 7)\n",
"┌────────────┬─────┬──────────┬───────────┬─────────┬─────────┬─────────┐\n",
"│ real_date ┆ cid ┆ xcat ┆ value ┆ grading ┆ eop_lag ┆ mop_lag │\n",
"│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n",
"│ date ┆ str ┆ str ┆ f64 ┆ f64 ┆ i64 ┆ i64 │\n",
"╞════════════╪═════╪══════════╪═══════════╪═════════╪═════════╪═════════╡\n",
"│ 2000-05-04 ┆ AUD ┆ EQXR_NSA ┆ -1.251605 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2000-05-05 ┆ AUD ┆ EQXR_NSA ┆ 1.787455 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2000-05-08 ┆ AUD ┆ EQXR_NSA ┆ -0.574713 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2000-05-09 ┆ AUD ┆ EQXR_NSA ┆ -0.931278 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2000-05-10 ┆ AUD ┆ EQXR_NSA ┆ -1.523501 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2000-05-11 ┆ AUD ┆ EQXR_NSA ┆ -1.579987 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2000-05-12 ┆ AUD ┆ EQXR_NSA ┆ 1.80602 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2000-05-15 ┆ AUD ┆ EQXR_NSA ┆ 0.295664 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2000-05-16 ┆ AUD ┆ EQXR_NSA ┆ 1.310187 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2000-05-17 ┆ AUD ┆ EQXR_NSA ┆ -0.711284 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"└────────────┴─────┴──────────┴───────────┴─────────┴─────────┴─────────┘"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"starttime = time.time()\n",
"new_df: pl.DataFrame = msyrs.qdf.update_dataframe(df=eq_df, df_add=fx_df)\n",
"print(\"Time taken: \", time.time() - starttime)\n",
"new_df.head(10)"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Time taken: 0.4808037281036377\n"
]
}
],
"source": [
"starttime = time.time()\n",
"new_pd_df = pddf.update_df(df_add=eq_pd_df,)\n",
"print(\"Time taken: \", time.time() - starttime)"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"\n",
"
shape: (10, 7)real_date | cid | xcat | value | grading | eop_lag | mop_lag |
---|
date | str | str | f64 | f64 | i64 | i64 |
2024-11-01 | "GBP" | "FXXR_VT10" | 1.37805 | 1.0 | 0 | 0 |
2024-11-04 | "GBP" | "FXXR_VT10" | -0.111931 | 1.0 | 0 | 0 |
2024-11-05 | "GBP" | "FXXR_VT10" | 0.25294 | 1.0 | 0 | 0 |
2024-11-06 | "GBP" | "FXXR_VT10" | -0.175161 | 1.0 | 0 | 0 |
2024-11-07 | "GBP" | "FXXR_VT10" | 0.806682 | 1.0 | 0 | 0 |
2024-11-08 | "GBP" | "FXXR_VT10" | -0.247346 | 1.0 | 0 | 0 |
2024-11-12 | "GBP" | "FXXR_VT10" | -1.083137 | 1.0 | 0 | 0 |
2024-11-13 | "GBP" | "FXXR_VT10" | -0.328958 | 1.0 | 0 | 0 |
2024-11-14 | "GBP" | "FXXR_VT10" | -0.110526 | 1.0 | 0 | 0 |
2024-11-15 | "GBP" | "FXXR_VT10" | -0.700977 | 1.0 | 0 | 0 |
"
],
"text/plain": [
"shape: (10, 7)\n",
"┌────────────┬─────┬───────────┬───────────┬─────────┬─────────┬─────────┐\n",
"│ real_date ┆ cid ┆ xcat ┆ value ┆ grading ┆ eop_lag ┆ mop_lag │\n",
"│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n",
"│ date ┆ str ┆ str ┆ f64 ┆ f64 ┆ i64 ┆ i64 │\n",
"╞════════════╪═════╪═══════════╪═══════════╪═════════╪═════════╪═════════╡\n",
"│ 2024-11-01 ┆ GBP ┆ FXXR_VT10 ┆ 1.37805 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2024-11-04 ┆ GBP ┆ FXXR_VT10 ┆ -0.111931 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2024-11-05 ┆ GBP ┆ FXXR_VT10 ┆ 0.25294 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2024-11-06 ┆ GBP ┆ FXXR_VT10 ┆ -0.175161 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2024-11-07 ┆ GBP ┆ FXXR_VT10 ┆ 0.806682 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2024-11-08 ┆ GBP ┆ FXXR_VT10 ┆ -0.247346 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2024-11-12 ┆ GBP ┆ FXXR_VT10 ┆ -1.083137 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2024-11-13 ┆ GBP ┆ FXXR_VT10 ┆ -0.328958 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2024-11-14 ┆ GBP ┆ FXXR_VT10 ┆ -0.110526 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2024-11-15 ┆ GBP ┆ FXXR_VT10 ┆ -0.700977 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"└────────────┴─────┴───────────┴───────────┴─────────┴─────────┴─────────┘"
]
},
"execution_count": 21,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"new_df.tail(10)"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Time taken: 0.0010075569152832031\n",
"Time taken: 0.0\n"
]
}
],
"source": [
"# df: polars::prelude::DataFrame,\n",
"# xcat: String,\n",
"# cids: Option>,\n",
"# lback_periods: Option,\n",
"# lback_method: Option,\n",
"# half_life: Option,\n",
"# start: Option,\n",
"# end: Option,\n",
"# est_freq: Option,\n",
"# remove_zeros: Option,\n",
"# postfix: Option,\n",
"# nan_tolerance: Option,\n",
"\n",
"starttime = time.time()\n",
"hv = msyrs.panel.historic_vol(\n",
" df=new_df,\n",
" xcat=\"EQXR_NSA\",\n",
" cids=None,\n",
" lback_periods=252,\n",
" lback_method=\"calendar\",\n",
" half_life=None,\n",
" start=None,\n",
" end=None,\n",
" est_freq=None,\n",
" remove_zeros=None,\n",
" postfix=\"_HV\",\n",
" nan_tolerance=None,\n",
")\n",
"print(f\"Time taken: {time.time() - starttime}\")\n",
"\n",
"starttime = time.time()\n",
"a = 1 + 5\n",
"print(\"Time taken: \", time.time() - starttime)"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Time taken: 0.05028486251831055\n"
]
}
],
"source": [
"starttime = time.time()\n",
"msyrs.qdf.pivot_dataframe_by_ticker(df=new_df).head(10)\n",
"print(\"Time taken: \", time.time() - starttime)"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [],
"source": [
"new_pd_df = macrosynergy.management.types.QuantamentalDataFrame(new_pd_df)"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Time taken: 2.0133273601531982\n"
]
}
],
"source": [
"starttime = time.time()\n",
"new_pd_df.to_wide()\n",
"print(\"Time taken: \", time.time() - starttime)"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Time taken: 130.01306796073914 seconds\n"
]
}
],
"source": [
"end_time = time.time()\n",
"print(f\"Time taken: {end_time - nb_start_time} seconds\")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": ".venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.7"
}
},
"nbformat": 4,
"nbformat_minor": 4
}