magnus167/msyrs
1
0
Fork 0
mirror of https://github.com/Magnus167/msyrs.git synced 2025-08-20 08:19:59 +00:00
Code Issues Packages Projects Releases Wiki Activity
msyrs/notebooks/python-notebook.ipynb
Palash Tyagi 26a220cc4c moving stuff around
2024-11-22 23:39:56 +00:00

1131 lines
46 KiB
Plaintext
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Build and install the package\n",
"\n",
"First patch `pyo3-polars`:\n",
"\n",
"- Use [this diff](https://github.com/pola-rs/pyo3-polars/compare/main...Magnus167:pyo3-polars:main) to make changes to the `pyo3-polars` package.\n",
"\n",
"Install the package:\n",
"\n",
"```bash\n",
"python -m venv .venv\n",
"\n",
"# source .venv/bin/activate\n",
"./.venv/Scripts/activate\n",
"\n",
"pip install maturin ipywidgets\n",
"\n",
"maturin develop --release\n",
"```\n"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Import Python packages\n"
]
},
{
"cell_type": "code",
"execution_count": 1,
"metadata": {},
"outputs": [],
"source": [
"import macrosynergy\n",
"import pandas as pd\n",
"import numpy as np\n",
"import polars as pl\n",
"import os"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Import Python bindings - `msyrs`\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
"import msyrs"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [],
"source": [
"DATA_FOLDER_PATH = \"E:/Work/jpmaqs-data\"\n",
"# DATA_FOLDER_PATH = \"C:/Users/PalashTyagi/Code/go-dataquery/jpmaqs-data\"\n",
"DQ_CLIENT_ID = os.getenv(\"DQ_CLIENT_ID\")\n",
"DQ_CLIENT_SECRET = os.getenv(\"DQ_CLIENT_SECRET\")"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {},
"outputs": [],
"source": [
"import time\n",
"\n",
"nb_start_time = time.time()"
]
},
{
"cell_type": "code",
"execution_count": 5,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Time taken to load qdf: 0.0040130615234375\n"
]
},
{
"data": {
"text/html": [
"<div><style>\n",
".dataframe > thead > tr,\n",
".dataframe > tbody > tr {\n",
" text-align: right;\n",
" white-space: pre-wrap;\n",
"}\n",
"</style>\n",
"<small>shape: (5, 7)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>real_date</th><th>cid</th><th>xcat</th><th>value</th><th>grading</th><th>eop_lag</th><th>mop_lag</th></tr><tr><td>date</td><td>str</td><td>str</td><td>f64</td><td>f64</td><td>i64</td><td>i64</td></tr></thead><tbody><tr><td>2010-03-03</td><td>&quot;USD&quot;</td><td>&quot;ADPEMPL_SA_P1M1ML1&quot;</td><td>-0.173806</td><td>3.0</td><td>3</td><td>33</td></tr><tr><td>2010-03-04</td><td>&quot;USD&quot;</td><td>&quot;ADPEMPL_SA_P1M1ML1&quot;</td><td>-0.173806</td><td>3.0</td><td>4</td><td>34</td></tr><tr><td>2010-03-05</td><td>&quot;USD&quot;</td><td>&quot;ADPEMPL_SA_P1M1ML1&quot;</td><td>-0.173806</td><td>3.0</td><td>5</td><td>35</td></tr><tr><td>2010-03-08</td><td>&quot;USD&quot;</td><td>&quot;ADPEMPL_SA_P1M1ML1&quot;</td><td>-0.173806</td><td>3.0</td><td>8</td><td>38</td></tr><tr><td>2010-03-09</td><td>&quot;USD&quot;</td><td>&quot;ADPEMPL_SA_P1M1ML1&quot;</td><td>-0.173806</td><td>3.0</td><td>9</td><td>39</td></tr></tbody></table></div>"
],
"text/plain": [
"shape: (5, 7)\n",
"┌────────────┬─────┬────────────────────┬───────────┬─────────┬─────────┬─────────┐\n",
"│ real_date ┆ cid ┆ xcat ┆ value ┆ grading ┆ eop_lag ┆ mop_lag │\n",
"│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n",
"│ date ┆ str ┆ str ┆ f64 ┆ f64 ┆ i64 ┆ i64 │\n",
"╞════════════╪═════╪════════════════════╪═══════════╪═════════╪═════════╪═════════╡\n",
"│ 2010-03-03 ┆ USD ┆ ADPEMPL_SA_P1M1ML1 ┆ -0.173806 ┆ 3.0 ┆ 3 ┆ 33 │\n",
"│ 2010-03-04 ┆ USD ┆ ADPEMPL_SA_P1M1ML1 ┆ -0.173806 ┆ 3.0 ┆ 4 ┆ 34 │\n",
"│ 2010-03-05 ┆ USD ┆ ADPEMPL_SA_P1M1ML1 ┆ -0.173806 ┆ 3.0 ┆ 5 ┆ 35 │\n",
"│ 2010-03-08 ┆ USD ┆ ADPEMPL_SA_P1M1ML1 ┆ -0.173806 ┆ 3.0 ┆ 8 ┆ 38 │\n",
"│ 2010-03-09 ┆ USD ┆ ADPEMPL_SA_P1M1ML1 ┆ -0.173806 ┆ 3.0 ┆ 9 ┆ 39 │\n",
"└────────────┴─────┴────────────────────┴───────────┴─────────┴─────────┴─────────┘"
]
},
"execution_count": 5,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"dfpath = f\"{DATA_FOLDER_PATH}/data/ADPEMPL_SA_P1M1ML1/USD_ADPEMPL_SA_P1M1ML1.csv\"\n",
"\n",
"starttime = time.time()\n",
"ldf: pl.DataFrame = msyrs.qdf.load_qdf(dfpath)\n",
"print(f\"Time taken to load qdf: {time.time() - starttime}\")\n",
"ldf.head(5)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [],
"source": [
"cids_dm = \"AUD.CAD.CHF.EUR.GBP.JPY.NOK.NZD.SEK.USD\".split(\".\")\n",
"cids_em = \"CLP.COP.CZK.HUF.IDR.ILS.INR.KRW.MXN.PLN.THB.TRY.TWD.ZAR\".split(\".\")\n",
"cids = cids_dm + cids_em\n",
"cids_dux = list(set(cids) - set([\"IDR\", \"NZD\"]))\n",
"ecos = \"CPIC_SA_P1M1ML12.CPIC_SJA_P3M3ML3AR.CPIC_SJA_P6M6ML6AR.CPIH_SA_P1M1ML12.CPIH_SJA_P3M3ML3AR.CPIH_SJA_P6M6ML6AR.INFTEFF_NSA.INTRGDP_NSA_P1M1ML12_3MMA.INTRGDPv5Y_NSA_P1M1ML12_3MMA.PCREDITGDP_SJA_D1M1ML12.RGDP_SA_P1Q1QL4_20QMA.RYLDIRS02Y_NSA.RYLDIRS05Y_NSA.PCREDITBN_SJA_P1M1ML12\".split(\n",
" \".\"\n",
")\n",
"\n",
"\n",
"mkts = \"DU02YXR_NSA.DU05YXR_NSA.DU02YXR_VT10.DU05YXR_VT10.EQXR_NSA.EQXR_VT10.FXXR_NSA.FXXR_VT10.FXCRR_NSA.FXTARGETED_NSA.FXUNTRADABLE_NSA\".split(\n",
" \".\"\n",
")\n",
"xcats = ecos + mkts\n",
"\n",
"tickers = [f\"{c}_{x}\" for c in cids for x in xcats]"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div><style>\n",
".dataframe > thead > tr,\n",
".dataframe > tbody > tr {\n",
" text-align: right;\n",
" white-space: pre-wrap;\n",
"}\n",
"</style>\n",
"<small>shape: (5, 7)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>real_date</th><th>cid</th><th>xcat</th><th>value</th><th>grading</th><th>eop_lag</th><th>mop_lag</th></tr><tr><td>date</td><td>str</td><td>str</td><td>f64</td><td>f64</td><td>f64</td><td>f64</td></tr></thead><tbody><tr><td>1990-04-26</td><td>&quot;AUD&quot;</td><td>&quot;CPIC_SA_P1M1ML12&quot;</td><td>6.434599</td><td>2.0</td><td>26.0</td><td>223.0</td></tr><tr><td>1990-04-27</td><td>&quot;AUD&quot;</td><td>&quot;CPIC_SA_P1M1ML12&quot;</td><td>6.434599</td><td>2.0</td><td>27.0</td><td>224.0</td></tr><tr><td>1990-04-30</td><td>&quot;AUD&quot;</td><td>&quot;CPIC_SA_P1M1ML12&quot;</td><td>6.434599</td><td>2.0</td><td>30.0</td><td>227.0</td></tr><tr><td>1990-05-01</td><td>&quot;AUD&quot;</td><td>&quot;CPIC_SA_P1M1ML12&quot;</td><td>6.434599</td><td>2.0</td><td>31.0</td><td>228.0</td></tr><tr><td>1990-05-02</td><td>&quot;AUD&quot;</td><td>&quot;CPIC_SA_P1M1ML12&quot;</td><td>6.434599</td><td>2.0</td><td>32.0</td><td>229.0</td></tr></tbody></table></div>"
],
"text/plain": [
"shape: (5, 7)\n",
"┌────────────┬─────┬──────────────────┬──────────┬─────────┬─────────┬─────────┐\n",
"│ real_date ┆ cid ┆ xcat ┆ value ┆ grading ┆ eop_lag ┆ mop_lag │\n",
"│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n",
"│ date ┆ str ┆ str ┆ f64 ┆ f64 ┆ f64 ┆ f64 │\n",
"╞════════════╪═════╪══════════════════╪══════════╪═════════╪═════════╪═════════╡\n",
"│ 1990-04-26 ┆ AUD ┆ CPIC_SA_P1M1ML12 ┆ 6.434599 ┆ 2.0 ┆ 26.0 ┆ 223.0 │\n",
"│ 1990-04-27 ┆ AUD ┆ CPIC_SA_P1M1ML12 ┆ 6.434599 ┆ 2.0 ┆ 27.0 ┆ 224.0 │\n",
"│ 1990-04-30 ┆ AUD ┆ CPIC_SA_P1M1ML12 ┆ 6.434599 ┆ 2.0 ┆ 30.0 ┆ 227.0 │\n",
"│ 1990-05-01 ┆ AUD ┆ CPIC_SA_P1M1ML12 ┆ 6.434599 ┆ 2.0 ┆ 31.0 ┆ 228.0 │\n",
"│ 1990-05-02 ┆ AUD ┆ CPIC_SA_P1M1ML12 ┆ 6.434599 ┆ 2.0 ┆ 32.0 ┆ 229.0 │\n",
"└────────────┴─────┴──────────────────┴──────────┴─────────┴─────────┴─────────┘"
]
},
"execution_count": 7,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"downloaded_df: pl.DataFrame = msyrs.download.download_jpmaqs_indicators_as_df(\n",
" client_id=DQ_CLIENT_ID,\n",
" client_secret=DQ_CLIENT_SECRET,\n",
" tickers=tickers,\n",
")\n",
"downloaded_df.head(5)"
]
},
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"downloaded_df = None"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Downloading the JPMAQS catalogue from DataQuery...\n",
"Downloaded JPMAQS catalogue with 18711 tickers.\n",
"Removed 21/600 expressions that are not in the JPMaQS catalogue.\n",
"Downloading data from JPMaQS.\n",
"Timestamp UTC: 2024-11-22 20:35:07\n",
"Connection successful!\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"Requesting data: 100%|██████████| 29/29 [00:05<00:00, 4.93it/s]\n",
"Downloading data: 100%|██████████| 29/29 [00:16<00:00, 1.80it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Some dates are missing from the downloaded data. \n",
"2 out of 9107 dates are missing.\n"
]
}
],
"source": [
"pddf = macrosynergy.download.JPMaQSDownload().download(\n",
" tickers=tickers,\n",
" get_catalogue=True,\n",
" show_progress=True,\n",
" start_date=\"1990-01-01\",\n",
")\n",
"pddf = macrosynergy.management.types.QuantamentalDataFrame(pddf)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Time taken to load qdf batch: 1.1382660865783691\n"
]
},
{
"data": {
"text/html": [
"<div><style>\n",
".dataframe > thead > tr,\n",
".dataframe > tbody > tr {\n",
" text-align: right;\n",
" white-space: pre-wrap;\n",
"}\n",
"</style>\n",
"<small>shape: (5, 7)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>real_date</th><th>cid</th><th>xcat</th><th>value</th><th>grading</th><th>eop_lag</th><th>mop_lag</th></tr><tr><td>date</td><td>str</td><td>str</td><td>f64</td><td>f64</td><td>i64</td><td>i64</td></tr></thead><tbody><tr><td>1990-04-26</td><td>&quot;AUD&quot;</td><td>&quot;CPIC_SA_P1M1ML12&quot;</td><td>6.434599</td><td>2.0</td><td>26</td><td>223</td></tr><tr><td>1990-04-27</td><td>&quot;AUD&quot;</td><td>&quot;CPIC_SA_P1M1ML12&quot;</td><td>6.434599</td><td>2.0</td><td>27</td><td>224</td></tr><tr><td>1990-04-30</td><td>&quot;AUD&quot;</td><td>&quot;CPIC_SA_P1M1ML12&quot;</td><td>6.434599</td><td>2.0</td><td>30</td><td>227</td></tr><tr><td>1990-05-01</td><td>&quot;AUD&quot;</td><td>&quot;CPIC_SA_P1M1ML12&quot;</td><td>6.434599</td><td>2.0</td><td>31</td><td>228</td></tr><tr><td>1990-05-02</td><td>&quot;AUD&quot;</td><td>&quot;CPIC_SA_P1M1ML12&quot;</td><td>6.434599</td><td>2.0</td><td>32</td><td>229</td></tr></tbody></table></div>"
],
"text/plain": [
"shape: (5, 7)\n",
"┌────────────┬─────┬──────────────────┬──────────┬─────────┬─────────┬─────────┐\n",
"│ real_date ┆ cid ┆ xcat ┆ value ┆ grading ┆ eop_lag ┆ mop_lag │\n",
"│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n",
"│ date ┆ str ┆ str ┆ f64 ┆ f64 ┆ i64 ┆ i64 │\n",
"╞════════════╪═════╪══════════════════╪══════════╪═════════╪═════════╪═════════╡\n",
"│ 1990-04-26 ┆ AUD ┆ CPIC_SA_P1M1ML12 ┆ 6.434599 ┆ 2.0 ┆ 26 ┆ 223 │\n",
"│ 1990-04-27 ┆ AUD ┆ CPIC_SA_P1M1ML12 ┆ 6.434599 ┆ 2.0 ┆ 27 ┆ 224 │\n",
"│ 1990-04-30 ┆ AUD ┆ CPIC_SA_P1M1ML12 ┆ 6.434599 ┆ 2.0 ┆ 30 ┆ 227 │\n",
"│ 1990-05-01 ┆ AUD ┆ CPIC_SA_P1M1ML12 ┆ 6.434599 ┆ 2.0 ┆ 31 ┆ 228 │\n",
"│ 1990-05-02 ┆ AUD ┆ CPIC_SA_P1M1ML12 ┆ 6.434599 ┆ 2.0 ┆ 32 ┆ 229 │\n",
"└────────────┴─────┴──────────────────┴──────────┴─────────┴─────────┴─────────┘"
]
},
"execution_count": 10,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"starttime = time.time()\n",
"\n",
"big_df: pl.DataFrame = msyrs.qdf.load_qdf_from_download_bank(\n",
"\n",
" folder_path=DATA_FOLDER_PATH,\n",
" xcats=xcats,\n",
"\n",
" # folder_path=DATA_FOLDER_PATH, cids=cids\n",
"\n",
")\n",
"print(f\"Time taken to load qdf batch: {time.time() - starttime}\")\n",
"\n",
"\n",
"big_df.head(5)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"275.7299613952637"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"big_df.estimated_size(\"mb\")"
]
},
{
"cell_type": "code",
"execution_count": 28,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>real_date</th>\n",
" <th>cid</th>\n",
" <th>xcat</th>\n",
" <th>value</th>\n",
" <th>grading</th>\n",
" <th>eop_lag</th>\n",
" <th>mop_lag</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1990-04-26</td>\n",
" <td>AUD</td>\n",
" <td>CPIC_SA_P1M1ML12</td>\n",
" <td>6.434599</td>\n",
" <td>2.0</td>\n",
" <td>26</td>\n",
" <td>223</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>1990-04-27</td>\n",
" <td>AUD</td>\n",
" <td>CPIC_SA_P1M1ML12</td>\n",
" <td>6.434599</td>\n",
" <td>2.0</td>\n",
" <td>27</td>\n",
" <td>224</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>1990-04-30</td>\n",
" <td>AUD</td>\n",
" <td>CPIC_SA_P1M1ML12</td>\n",
" <td>6.434599</td>\n",
" <td>2.0</td>\n",
" <td>30</td>\n",
" <td>227</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>1990-05-01</td>\n",
" <td>AUD</td>\n",
" <td>CPIC_SA_P1M1ML12</td>\n",
" <td>6.434599</td>\n",
" <td>2.0</td>\n",
" <td>31</td>\n",
" <td>228</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>1990-05-02</td>\n",
" <td>AUD</td>\n",
" <td>CPIC_SA_P1M1ML12</td>\n",
" <td>6.434599</td>\n",
" <td>2.0</td>\n",
" <td>32</td>\n",
" <td>229</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5285577</th>\n",
" <td>2024-11-08</td>\n",
" <td>ZAR</td>\n",
" <td>RYLDIRS05Y_NSA</td>\n",
" <td>3.097534</td>\n",
" <td>1.0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5285578</th>\n",
" <td>2024-11-12</td>\n",
" <td>ZAR</td>\n",
" <td>RYLDIRS05Y_NSA</td>\n",
" <td>3.189884</td>\n",
" <td>1.0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5285579</th>\n",
" <td>2024-11-13</td>\n",
" <td>ZAR</td>\n",
" <td>RYLDIRS05Y_NSA</td>\n",
" <td>3.108095</td>\n",
" <td>1.0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5285580</th>\n",
" <td>2024-11-14</td>\n",
" <td>ZAR</td>\n",
" <td>RYLDIRS05Y_NSA</td>\n",
" <td>3.060968</td>\n",
" <td>1.0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5285581</th>\n",
" <td>2024-11-15</td>\n",
" <td>ZAR</td>\n",
" <td>RYLDIRS05Y_NSA</td>\n",
" <td>3.102066</td>\n",
" <td>1.0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>5285582 rows × 7 columns</p>\n",
"</div>"
],
"text/plain": [
" real_date cid xcat value grading eop_lag mop_lag\n",
"0 1990-04-26 AUD CPIC_SA_P1M1ML12 6.434599 2.0 26 223\n",
"1 1990-04-27 AUD CPIC_SA_P1M1ML12 6.434599 2.0 27 224\n",
"2 1990-04-30 AUD CPIC_SA_P1M1ML12 6.434599 2.0 30 227\n",
"3 1990-05-01 AUD CPIC_SA_P1M1ML12 6.434599 2.0 31 228\n",
"4 1990-05-02 AUD CPIC_SA_P1M1ML12 6.434599 2.0 32 229\n",
"... ... ... ... ... ... ... ...\n",
"5285577 2024-11-08 ZAR RYLDIRS05Y_NSA 3.097534 1.0 0 0\n",
"5285578 2024-11-12 ZAR RYLDIRS05Y_NSA 3.189884 1.0 0 0\n",
"5285579 2024-11-13 ZAR RYLDIRS05Y_NSA 3.108095 1.0 0 0\n",
"5285580 2024-11-14 ZAR RYLDIRS05Y_NSA 3.060968 1.0 0 0\n",
"5285581 2024-11-15 ZAR RYLDIRS05Y_NSA 3.102066 1.0 0 0\n",
"\n",
"[5285582 rows x 7 columns]"
]
},
"execution_count": 28,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"big_df.to_pandas()"
]
},
{
"cell_type": "code",
"execution_count": 12,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"789.8839149475098"
]
},
"execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"big_df.to_pandas().memory_usage(deep=True).sum() / 1024**2"
]
},
{
"cell_type": "code",
"execution_count": 31,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>real_date</th>\n",
" <th>cid</th>\n",
" <th>xcat</th>\n",
" <th>value</th>\n",
" <th>grading</th>\n",
" <th>eop_lag</th>\n",
" <th>mop_lag</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>1990-04-26</td>\n",
" <td>AUD</td>\n",
" <td>CPIC_SA_P1M1ML12</td>\n",
" <td>6.434599</td>\n",
" <td>2.0</td>\n",
" <td>26</td>\n",
" <td>223</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>1990-04-27</td>\n",
" <td>AUD</td>\n",
" <td>CPIC_SA_P1M1ML12</td>\n",
" <td>6.434599</td>\n",
" <td>2.0</td>\n",
" <td>27</td>\n",
" <td>224</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>1990-04-30</td>\n",
" <td>AUD</td>\n",
" <td>CPIC_SA_P1M1ML12</td>\n",
" <td>6.434599</td>\n",
" <td>2.0</td>\n",
" <td>30</td>\n",
" <td>227</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>1990-05-01</td>\n",
" <td>AUD</td>\n",
" <td>CPIC_SA_P1M1ML12</td>\n",
" <td>6.434599</td>\n",
" <td>2.0</td>\n",
" <td>31</td>\n",
" <td>228</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>1990-05-02</td>\n",
" <td>AUD</td>\n",
" <td>CPIC_SA_P1M1ML12</td>\n",
" <td>6.434599</td>\n",
" <td>2.0</td>\n",
" <td>32</td>\n",
" <td>229</td>\n",
" </tr>\n",
" <tr>\n",
" <th>...</th>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" <td>...</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5285577</th>\n",
" <td>2024-11-08</td>\n",
" <td>ZAR</td>\n",
" <td>RYLDIRS05Y_NSA</td>\n",
" <td>3.097534</td>\n",
" <td>1.0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5285578</th>\n",
" <td>2024-11-12</td>\n",
" <td>ZAR</td>\n",
" <td>RYLDIRS05Y_NSA</td>\n",
" <td>3.189884</td>\n",
" <td>1.0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5285579</th>\n",
" <td>2024-11-13</td>\n",
" <td>ZAR</td>\n",
" <td>RYLDIRS05Y_NSA</td>\n",
" <td>3.108095</td>\n",
" <td>1.0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5285580</th>\n",
" <td>2024-11-14</td>\n",
" <td>ZAR</td>\n",
" <td>RYLDIRS05Y_NSA</td>\n",
" <td>3.060968</td>\n",
" <td>1.0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5285581</th>\n",
" <td>2024-11-15</td>\n",
" <td>ZAR</td>\n",
" <td>RYLDIRS05Y_NSA</td>\n",
" <td>3.102066</td>\n",
" <td>1.0</td>\n",
" <td>0</td>\n",
" <td>0</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>5285582 rows × 7 columns</p>\n",
"</div>"
],
"text/plain": [
" real_date cid xcat value grading eop_lag mop_lag\n",
"0 1990-04-26 AUD CPIC_SA_P1M1ML12 6.434599 2.0 26 223\n",
"1 1990-04-27 AUD CPIC_SA_P1M1ML12 6.434599 2.0 27 224\n",
"2 1990-04-30 AUD CPIC_SA_P1M1ML12 6.434599 2.0 30 227\n",
"3 1990-05-01 AUD CPIC_SA_P1M1ML12 6.434599 2.0 31 228\n",
"4 1990-05-02 AUD CPIC_SA_P1M1ML12 6.434599 2.0 32 229\n",
"... ... ... ... ... ... ... ...\n",
"5285577 2024-11-08 ZAR RYLDIRS05Y_NSA 3.097534 1.0 0 0\n",
"5285578 2024-11-12 ZAR RYLDIRS05Y_NSA 3.189884 1.0 0 0\n",
"5285579 2024-11-13 ZAR RYLDIRS05Y_NSA 3.108095 1.0 0 0\n",
"5285580 2024-11-14 ZAR RYLDIRS05Y_NSA 3.060968 1.0 0 0\n",
"5285581 2024-11-15 ZAR RYLDIRS05Y_NSA 3.102066 1.0 0 0\n",
"\n",
"[5285582 rows x 7 columns]"
]
},
"execution_count": 31,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"macrosynergy.management.types.QuantamentalDataFrame(big_df.to_pandas())"
]
},
{
"cell_type": "code",
"execution_count": 13,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"211.71554374694824"
]
},
"execution_count": 13,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"macrosynergy.management.types.QuantamentalDataFrame(big_df.to_pandas()).memory_usage(\n",
" deep=True\n",
").sum() / 1024**2"
]
},
{
"cell_type": "code",
"execution_count": 14,
"metadata": {},
"outputs": [],
"source": [
"sel_cids = [\"USD\", \"EUR\", \"GBP\", \"AUD\", \"CAD\"]\n",
"start = \"1990-01-01\""
]
},
{
"cell_type": "code",
"execution_count": 15,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Time taken to reduce qdf: 0.2668800354003906\n"
]
},
{
"data": {
"text/html": [
"<div><style>\n",
".dataframe > thead > tr,\n",
".dataframe > tbody > tr {\n",
" text-align: right;\n",
" white-space: pre-wrap;\n",
"}\n",
"</style>\n",
"<small>shape: (73_069, 7)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>real_date</th><th>cid</th><th>xcat</th><th>value</th><th>grading</th><th>eop_lag</th><th>mop_lag</th></tr><tr><td>date</td><td>str</td><td>str</td><td>f64</td><td>f64</td><td>i64</td><td>i64</td></tr></thead><tbody><tr><td>2000-05-04</td><td>&quot;AUD&quot;</td><td>&quot;EQXR_NSA&quot;</td><td>-1.251605</td><td>1.0</td><td>0</td><td>0</td></tr><tr><td>2000-05-05</td><td>&quot;AUD&quot;</td><td>&quot;EQXR_NSA&quot;</td><td>1.787455</td><td>1.0</td><td>0</td><td>0</td></tr><tr><td>2000-05-08</td><td>&quot;AUD&quot;</td><td>&quot;EQXR_NSA&quot;</td><td>-0.574713</td><td>1.0</td><td>0</td><td>0</td></tr><tr><td>2000-05-09</td><td>&quot;AUD&quot;</td><td>&quot;EQXR_NSA&quot;</td><td>-0.931278</td><td>1.0</td><td>0</td><td>0</td></tr><tr><td>2000-05-10</td><td>&quot;AUD&quot;</td><td>&quot;EQXR_NSA&quot;</td><td>-1.523501</td><td>1.0</td><td>0</td><td>0</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>2024-11-08</td><td>&quot;USD&quot;</td><td>&quot;EQXR_VT10&quot;</td><td>0.313875</td><td>1.0</td><td>0</td><td>0</td></tr><tr><td>2024-11-12</td><td>&quot;USD&quot;</td><td>&quot;EQXR_VT10&quot;</td><td>-0.272457</td><td>1.0</td><td>0</td><td>0</td></tr><tr><td>2024-11-13</td><td>&quot;USD&quot;</td><td>&quot;EQXR_VT10&quot;</td><td>0.043729</td><td>1.0</td><td>0</td><td>0</td></tr><tr><td>2024-11-14</td><td>&quot;USD&quot;</td><td>&quot;EQXR_VT10&quot;</td><td>-0.549983</td><td>1.0</td><td>0</td><td>0</td></tr><tr><td>2024-11-15</td><td>&quot;USD&quot;</td><td>&quot;EQXR_VT10&quot;</td><td>-1.198544</td><td>1.0</td><td>0</td><td>0</td></tr></tbody></table></div>"
],
"text/plain": [
"shape: (73_069, 7)\n",
"┌────────────┬─────┬───────────┬───────────┬─────────┬─────────┬─────────┐\n",
"│ real_date ┆ cid ┆ xcat ┆ value ┆ grading ┆ eop_lag ┆ mop_lag │\n",
"│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n",
"│ date ┆ str ┆ str ┆ f64 ┆ f64 ┆ i64 ┆ i64 │\n",
"╞════════════╪═════╪═══════════╪═══════════╪═════════╪═════════╪═════════╡\n",
"│ 2000-05-04 ┆ AUD ┆ EQXR_NSA ┆ -1.251605 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2000-05-05 ┆ AUD ┆ EQXR_NSA ┆ 1.787455 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2000-05-08 ┆ AUD ┆ EQXR_NSA ┆ -0.574713 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2000-05-09 ┆ AUD ┆ EQXR_NSA ┆ -0.931278 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2000-05-10 ┆ AUD ┆ EQXR_NSA ┆ -1.523501 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ … ┆ … ┆ … ┆ … ┆ … ┆ … ┆ … │\n",
"│ 2024-11-08 ┆ USD ┆ EQXR_VT10 ┆ 0.313875 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2024-11-12 ┆ USD ┆ EQXR_VT10 ┆ -0.272457 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2024-11-13 ┆ USD ┆ EQXR_VT10 ┆ 0.043729 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2024-11-14 ┆ USD ┆ EQXR_VT10 ┆ -0.549983 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2024-11-15 ┆ USD ┆ EQXR_VT10 ┆ -1.198544 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"└────────────┴─────┴───────────┴───────────┴─────────┴─────────┴─────────┘"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"starttime = time.time()\n",
"eq_df = msyrs.qdf.reduce_dataframe(\n",
" df=big_df,\n",
" cids=sel_cids,\n",
" xcats=[\"EQXR_NSA\", \"EQXR_VT10\"],\n",
"\n",
" start=start,\n",
")\n",
"print(f\"Time taken to reduce qdf: {time.time() - starttime}\")\n",
"eq_df"
]
},
{
"cell_type": "code",
"execution_count": 16,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Time taken to reduce qdf: 0.057086944580078125\n"
]
}
],
"source": [
"starttime = time.time()\n",
"eq_pd_df = pddf.reduce_df(cids=sel_cids, xcats=[\"EQXR_NSA\", \"EQXR_VT10\"], start=start)\n",
"print(f\"Time taken to reduce qdf: {time.time() - starttime}\")"
]
},
{
"cell_type": "code",
"execution_count": 17,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Time taken to reduce qdf: 0.298140287399292\n"
]
}
],
"source": [
"fx_xcats = [xc for xc in xcats if xc.startswith(\"FX\")]\n",
"starttime = time.time()\n",
"\n",
"fx_df = msyrs.qdf.reduce_dataframe(\n",
" df=big_df, cids=sel_cids, start=start, xcats=fx_xcats, intersect=True\n",
")\n",
"print(f\"Time taken to reduce qdf: {time.time() - starttime}\")"
]
},
{
"cell_type": "code",
"execution_count": 18,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Time taken to reduce qdf: 0.08732080459594727\n"
]
}
],
"source": [
"starttime = time.time()\n",
"fx_pd_df = pddf.reduce_df(cids=sel_cids, xcats=fx_xcats, start=start, intersect=True)\n",
"print(f\"Time taken to reduce qdf: {time.time() - starttime}\")"
]
},
{
"cell_type": "code",
"execution_count": 19,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Time taken: 0.01651310920715332\n"
]
},
{
"data": {
"text/html": [
"<div><style>\n",
".dataframe > thead > tr,\n",
".dataframe > tbody > tr {\n",
" text-align: right;\n",
" white-space: pre-wrap;\n",
"}\n",
"</style>\n",
"<small>shape: (10, 7)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>real_date</th><th>cid</th><th>xcat</th><th>value</th><th>grading</th><th>eop_lag</th><th>mop_lag</th></tr><tr><td>date</td><td>str</td><td>str</td><td>f64</td><td>f64</td><td>i64</td><td>i64</td></tr></thead><tbody><tr><td>2000-05-04</td><td>&quot;AUD&quot;</td><td>&quot;EQXR_NSA&quot;</td><td>-1.251605</td><td>1.0</td><td>0</td><td>0</td></tr><tr><td>2000-05-05</td><td>&quot;AUD&quot;</td><td>&quot;EQXR_NSA&quot;</td><td>1.787455</td><td>1.0</td><td>0</td><td>0</td></tr><tr><td>2000-05-08</td><td>&quot;AUD&quot;</td><td>&quot;EQXR_NSA&quot;</td><td>-0.574713</td><td>1.0</td><td>0</td><td>0</td></tr><tr><td>2000-05-09</td><td>&quot;AUD&quot;</td><td>&quot;EQXR_NSA&quot;</td><td>-0.931278</td><td>1.0</td><td>0</td><td>0</td></tr><tr><td>2000-05-10</td><td>&quot;AUD&quot;</td><td>&quot;EQXR_NSA&quot;</td><td>-1.523501</td><td>1.0</td><td>0</td><td>0</td></tr><tr><td>2000-05-11</td><td>&quot;AUD&quot;</td><td>&quot;EQXR_NSA&quot;</td><td>-1.579987</td><td>1.0</td><td>0</td><td>0</td></tr><tr><td>2000-05-12</td><td>&quot;AUD&quot;</td><td>&quot;EQXR_NSA&quot;</td><td>1.80602</td><td>1.0</td><td>0</td><td>0</td></tr><tr><td>2000-05-15</td><td>&quot;AUD&quot;</td><td>&quot;EQXR_NSA&quot;</td><td>0.295664</td><td>1.0</td><td>0</td><td>0</td></tr><tr><td>2000-05-16</td><td>&quot;AUD&quot;</td><td>&quot;EQXR_NSA&quot;</td><td>1.310187</td><td>1.0</td><td>0</td><td>0</td></tr><tr><td>2000-05-17</td><td>&quot;AUD&quot;</td><td>&quot;EQXR_NSA&quot;</td><td>-0.711284</td><td>1.0</td><td>0</td><td>0</td></tr></tbody></table></div>"
],
"text/plain": [
"shape: (10, 7)\n",
"┌────────────┬─────┬──────────┬───────────┬─────────┬─────────┬─────────┐\n",
"│ real_date ┆ cid ┆ xcat ┆ value ┆ grading ┆ eop_lag ┆ mop_lag │\n",
"│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n",
"│ date ┆ str ┆ str ┆ f64 ┆ f64 ┆ i64 ┆ i64 │\n",
"╞════════════╪═════╪══════════╪═══════════╪═════════╪═════════╪═════════╡\n",
"│ 2000-05-04 ┆ AUD ┆ EQXR_NSA ┆ -1.251605 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2000-05-05 ┆ AUD ┆ EQXR_NSA ┆ 1.787455 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2000-05-08 ┆ AUD ┆ EQXR_NSA ┆ -0.574713 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2000-05-09 ┆ AUD ┆ EQXR_NSA ┆ -0.931278 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2000-05-10 ┆ AUD ┆ EQXR_NSA ┆ -1.523501 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2000-05-11 ┆ AUD ┆ EQXR_NSA ┆ -1.579987 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2000-05-12 ┆ AUD ┆ EQXR_NSA ┆ 1.80602 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2000-05-15 ┆ AUD ┆ EQXR_NSA ┆ 0.295664 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2000-05-16 ┆ AUD ┆ EQXR_NSA ┆ 1.310187 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2000-05-17 ┆ AUD ┆ EQXR_NSA ┆ -0.711284 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"└────────────┴─────┴──────────┴───────────┴─────────┴─────────┴─────────┘"
]
},
"execution_count": 19,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"starttime = time.time()\n",
"new_df: pl.DataFrame = msyrs.qdf.update_dataframe(df=eq_df, df_add=fx_df)\n",
"print(\"Time taken: \", time.time() - starttime)\n",
"new_df.head(10)"
]
},
{
"cell_type": "code",
"execution_count": 20,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Time taken: 0.4808037281036377\n"
]
}
],
"source": [
"starttime = time.time()\n",
"new_pd_df = pddf.update_df(df_add=eq_pd_df,)\n",
"print(\"Time taken: \", time.time() - starttime)"
]
},
{
"cell_type": "code",
"execution_count": 21,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div><style>\n",
".dataframe > thead > tr,\n",
".dataframe > tbody > tr {\n",
" text-align: right;\n",
" white-space: pre-wrap;\n",
"}\n",
"</style>\n",
"<small>shape: (10, 7)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>real_date</th><th>cid</th><th>xcat</th><th>value</th><th>grading</th><th>eop_lag</th><th>mop_lag</th></tr><tr><td>date</td><td>str</td><td>str</td><td>f64</td><td>f64</td><td>i64</td><td>i64</td></tr></thead><tbody><tr><td>2024-11-01</td><td>&quot;GBP&quot;</td><td>&quot;FXXR_VT10&quot;</td><td>1.37805</td><td>1.0</td><td>0</td><td>0</td></tr><tr><td>2024-11-04</td><td>&quot;GBP&quot;</td><td>&quot;FXXR_VT10&quot;</td><td>-0.111931</td><td>1.0</td><td>0</td><td>0</td></tr><tr><td>2024-11-05</td><td>&quot;GBP&quot;</td><td>&quot;FXXR_VT10&quot;</td><td>0.25294</td><td>1.0</td><td>0</td><td>0</td></tr><tr><td>2024-11-06</td><td>&quot;GBP&quot;</td><td>&quot;FXXR_VT10&quot;</td><td>-0.175161</td><td>1.0</td><td>0</td><td>0</td></tr><tr><td>2024-11-07</td><td>&quot;GBP&quot;</td><td>&quot;FXXR_VT10&quot;</td><td>0.806682</td><td>1.0</td><td>0</td><td>0</td></tr><tr><td>2024-11-08</td><td>&quot;GBP&quot;</td><td>&quot;FXXR_VT10&quot;</td><td>-0.247346</td><td>1.0</td><td>0</td><td>0</td></tr><tr><td>2024-11-12</td><td>&quot;GBP&quot;</td><td>&quot;FXXR_VT10&quot;</td><td>-1.083137</td><td>1.0</td><td>0</td><td>0</td></tr><tr><td>2024-11-13</td><td>&quot;GBP&quot;</td><td>&quot;FXXR_VT10&quot;</td><td>-0.328958</td><td>1.0</td><td>0</td><td>0</td></tr><tr><td>2024-11-14</td><td>&quot;GBP&quot;</td><td>&quot;FXXR_VT10&quot;</td><td>-0.110526</td><td>1.0</td><td>0</td><td>0</td></tr><tr><td>2024-11-15</td><td>&quot;GBP&quot;</td><td>&quot;FXXR_VT10&quot;</td><td>-0.700977</td><td>1.0</td><td>0</td><td>0</td></tr></tbody></table></div>"
],
"text/plain": [
"shape: (10, 7)\n",
"┌────────────┬─────┬───────────┬───────────┬─────────┬─────────┬─────────┐\n",
"│ real_date ┆ cid ┆ xcat ┆ value ┆ grading ┆ eop_lag ┆ mop_lag │\n",
"│ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- ┆ --- │\n",
"│ date ┆ str ┆ str ┆ f64 ┆ f64 ┆ i64 ┆ i64 │\n",
"╞════════════╪═════╪═══════════╪═══════════╪═════════╪═════════╪═════════╡\n",
"│ 2024-11-01 ┆ GBP ┆ FXXR_VT10 ┆ 1.37805 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2024-11-04 ┆ GBP ┆ FXXR_VT10 ┆ -0.111931 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2024-11-05 ┆ GBP ┆ FXXR_VT10 ┆ 0.25294 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2024-11-06 ┆ GBP ┆ FXXR_VT10 ┆ -0.175161 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2024-11-07 ┆ GBP ┆ FXXR_VT10 ┆ 0.806682 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2024-11-08 ┆ GBP ┆ FXXR_VT10 ┆ -0.247346 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2024-11-12 ┆ GBP ┆ FXXR_VT10 ┆ -1.083137 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2024-11-13 ┆ GBP ┆ FXXR_VT10 ┆ -0.328958 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2024-11-14 ┆ GBP ┆ FXXR_VT10 ┆ -0.110526 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"│ 2024-11-15 ┆ GBP ┆ FXXR_VT10 ┆ -0.700977 ┆ 1.0 ┆ 0 ┆ 0 │\n",
"└────────────┴─────┴───────────┴───────────┴─────────┴─────────┴─────────┘"
]
},
"execution_count": 21,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"new_df.tail(10)"
]
},
{
"cell_type": "code",
"execution_count": 22,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Time taken: 0.0010075569152832031\n",
"Time taken: 0.0\n"
]
}
],
"source": [
"# df: polars::prelude::DataFrame,\n",
"# xcat: String,\n",
"# cids: Option<Vec<String>>,\n",
"# lback_periods: Option<usize>,\n",
"# lback_method: Option<String>,\n",
"# half_life: Option<f64>,\n",
"# start: Option<String>,\n",
"# end: Option<String>,\n",
"# est_freq: Option<String>,\n",
"# remove_zeros: Option<bool>,\n",
"# postfix: Option<String>,\n",
"# nan_tolerance: Option<f64>,\n",
"\n",
"starttime = time.time()\n",
"hv = msyrs.panel.historic_vol(\n",
" df=new_df,\n",
" xcat=\"EQXR_NSA\",\n",
" cids=None,\n",
" lback_periods=252,\n",
" lback_method=\"calendar\",\n",
" half_life=None,\n",
" start=None,\n",
" end=None,\n",
" est_freq=None,\n",
" remove_zeros=None,\n",
" postfix=\"_HV\",\n",
" nan_tolerance=None,\n",
")\n",
"print(f\"Time taken: {time.time() - starttime}\")\n",
"\n",
"starttime = time.time()\n",
"a = 1 + 5\n",
"print(\"Time taken: \", time.time() - starttime)"
]
},
{
"cell_type": "code",
"execution_count": 23,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Time taken: 0.05028486251831055\n"
]
}
],
"source": [
"starttime = time.time()\n",
"msyrs.qdf.pivot_dataframe_by_ticker(df=new_df).head(10)\n",
"print(\"Time taken: \", time.time() - starttime)"
]
},
{
"cell_type": "code",
"execution_count": 24,
"metadata": {},
"outputs": [],
"source": [
"new_pd_df = macrosynergy.management.types.QuantamentalDataFrame(new_pd_df)"
]
},
{
"cell_type": "code",
"execution_count": 25,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Time taken: 2.0133273601531982\n"
]
}
],
"source": [
"starttime = time.time()\n",
"new_pd_df.to_wide()\n",
"print(\"Time taken: \", time.time() - starttime)"
]
},
{
"cell_type": "code",
"execution_count": 26,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Time taken: 130.01306796073914 seconds\n"
]
}
],
"source": [
"end_time = time.time()\n",
"print(f\"Time taken: {end_time - nb_start_time} seconds\")"
]
}
],
"metadata": {
"kernelspec": {
"display_name": ".venv",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.7"
}
},
"nbformat": 4,
"nbformat_minor": 4
}
Reference in New Issue View Git Blame Copy Permalink