diff --git a/Cargo.toml b/Cargo.toml index a547cfe..16671cf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ futures = "0.3" # pyo3 = { version = "0.23.1", features = ["extension-module"] } # pyo3 = { version = "0.21.2", features = ["extension-module"] } # pyo3 = { version = "*", features = ["abi3-py38"] } +# polars-core = { version = "*" } pyo3 = { version = "*", features = ["extension-module", "abi3-py37"] } pyo3-polars = { version = "*" } polars = { version = "*", features = [ diff --git a/notebooks/python-notebook.ipynb b/notebooks/python-notebook.ipynb index fb445b8..d6e7334 100644 --- a/notebooks/python-notebook.ipynb +++ b/notebooks/python-notebook.ipynb @@ -6,6 +6,14 @@ "source": [ "# Build and install the package\n", "\n", + "First patch `pyo3-polars`:\n", + "\n", + "\n", + "- Use [this diff](https://github.com/pola-rs/pyo3-polars/compare/main...Magnus167:pyo3-polars:main) to make changes to the `pyo3-polars` package.\n", + "\n", + "\n", + "Install the package:\n", + "\n", "```bash\n", "python -m venv .venv\n", "\n", @@ -18,6 +26,13 @@ "```\n" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "markdown", "metadata": {}, @@ -27,7 +42,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -47,7 +62,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -56,19 +71,19 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": {}, "outputs": [], "source": [ - "DATA_FOLDER_PATH = \"E:/Work/jpmaqs-data\"\n", - "\n", + "# DATA_FOLDER_PATH = \"E:/Work/jpmaqs-data\"\n", + "DATA_FOLDER_PATH = \"C:/Users/PalashTyagi/Code/go-dataquery/jpmaqs-data\"\n", "DQ_CLIENT_ID = os.getenv(\"DQ_CLIENT_ID\")\n", "DQ_CLIENT_SECRET = os.getenv(\"DQ_CLIENT_SECRET\")" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -98,7 +113,7 @@ "└────────────┴─────┴────────────────────┴───────────┴─────────┴─────────┴─────────┘" ] }, - "execution_count": 5, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -113,7 +128,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": {}, "outputs": [], "source": [ @@ -136,7 +151,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -166,7 +181,7 @@ "└────────────┴─────┴──────────────────┴──────────┴─────────┴─────────┴─────────┘" ] }, - "execution_count": 7, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -182,7 +197,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -212,7 +227,7 @@ "└────────────┴─────┴──────────────────┴──────────┴─────────┴─────────┴─────────┘" ] }, - "execution_count": 8, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -226,7 +241,69 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "241.2352533340454" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "big_df.estimated_size(\"mb\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "625.9191436767578" + ] + }, + "execution_count": 20, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "big_df.to_pandas().memory_usage(deep=True).sum() / 1024**2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "152.3249340057373" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "macrosynergy.management.types.QuantamentalDataFrame(big_df.to_pandas()).memory_usage(\n", + " deep=True\n", + ").sum() / 1024**2" + ] + }, + { + "cell_type": "code", + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -236,7 +313,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 9, "metadata": {}, "outputs": [ { @@ -272,7 +349,7 @@ "└────────────┴─────┴───────────┴───────────┴─────────┴─────────┴─────────┘" ] }, - "execution_count": 10, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -289,7 +366,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -301,7 +378,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -336,7 +413,7 @@ "└────────────┴─────┴──────────┴───────────┴─────────┴─────────┴─────────┘" ] }, - "execution_count": 12, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -349,7 +426,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -384,7 +461,7 @@ "└────────────┴─────┴───────────┴───────────┴─────────┴─────────┴─────────┘" ] }, - "execution_count": 13, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -410,7 +487,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.7" + "version": "3.11.0" } }, "nbformat": 4,