diff --git a/Screenshot 2026-03-29 at 12.35.51 AM.png b/Screenshot 2026-03-29 at 12.35.51 AM.png new file mode 100644 index 0000000..aa978d4 Binary files /dev/null and b/Screenshot 2026-03-29 at 12.35.51 AM.png differ diff --git a/database.ipynb b/database.ipynb new file mode 100644 index 0000000..9da64d1 --- /dev/null +++ b/database.ipynb @@ -0,0 +1,891 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "4cae6bf1", + "metadata": {}, + "outputs": [], + "source": [ + "from sqlalchemy import create_engine, text\n", + "import pandas as pd\n", + "from datetime import datetime" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "f5040527", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Connection successful\n" + ] + } + ], + "source": [ + "### MYSQL ###\n", + "engine = create_engine('mysql+pymysql://root:pwd@localhost/polymarket')\n", + "try:\n", + " with engine.connect() as conn:\n", + " print(\"Connection successful\")\n", + "except Exception as e:\n", + " print(f\"Connection failed: {e}\") " + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "72059b3f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Connection successful\n" + ] + } + ], + "source": [ + "### MYSQL ###\n", + "engine_inter_storage = create_engine('mysql+pymysql://root:pwd@100.84.226.40/polymarket')\n", + "try:\n", + " with engine.connect() as conn:\n", + " print(\"Connection successful\")\n", + "except Exception as e:\n", + " print(f\"Connection failed: {e}\") " + ] + }, + { + "cell_type": "code", + "execution_count": 48, + "id": "b723a51f", + "metadata": {}, + "outputs": [], + "source": [ + "# with engine.connect() as conn:\n", + "# print(\"Connection successful\")\n", + "# sql = text(\"TRUNCATE TABLE coinbase_btcusd_trades;\")\n", + "# conn.execute(sql)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "5c23110d", + "metadata": {}, + "outputs": [], + "source": [ + "q_binance = '''\n", + "SELECT * FROM binance_btcusd_trades;\n", + "'''\n", + "q_coinbase = '''\n", + "SELECT * FROM coinbase_btcusd_trades;\n", + "'''\n", + "q_rtds = '''\n", + "SELECT * FROM poly_rtds_cl_btcusd;\n", + "'''\n", + "q_clob = '''\n", + "SELECT * FROM poly_btcusd_trades;\n", + "'''" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "a866e9ca", + "metadata": {}, + "outputs": [], + "source": [ + "# df_binance = pd.read_sql(q_binance, con=engine)\n", + "# df_coinbase = pd.read_sql(q_coinbase, con=engine)\n", + "# df_rtds = pd.read_sql(q_rtds, con=engine)\n", + "df_clob = pd.read_sql(q_clob, con=engine)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "954a3c3c", + "metadata": {}, + "outputs": [], + "source": [ + "# df_binance['timestamp_arrival'] = pd.to_datetime(df_binance['timestamp_arrival'], unit='ms')\n", + "# df_coinbase['timestamp_arrival'] = pd.to_datetime(df_coinbase['timestamp_arrival'], unit='ms')\n", + "# df_rtds['timestamp_arrival'] = pd.to_datetime(df_rtds['timestamp_arrival'], unit='ms')\n", + "df_clob['timestamp_arrival_dt'] = pd.to_datetime(df_clob['timestamp_arrival'], unit='ms')" + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "id": "50c6339f", + "metadata": {}, + "outputs": [], + "source": [ + "def copy_table_data_btw_servers(df, table_name, engine_destination) -> None:\n", + " rows_imported = df.to_sql(name=table_name, con=engine_destination, if_exists='append')\n", + " if rows_imported == len(df):\n", + " print(f'SUCCESS: COPIED {rows_imported} to table \"{table_name}\" on INTERSERVER_STORAGE')\n", + " else:\n", + " raise ValueError(f'FAILED: COPIED {rows_imported} rows to table {table_name} on INTERSERVER_STORAGE; EXPECTED {len(df)}')\n", + " \n", + "def truncate_table(engine, table):\n", + " with engine.connect() as conn:\n", + " sql = text(f\"TRUNCATE TABLE {table};\")\n", + " conn.execute(sql)\n", + " conn.commit()" + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "id": "d0399a96", + "metadata": {}, + "outputs": [], + "source": [ + "def backup_all_tables(engine_origin, engine_destination, tables_to_copy):\n", + " for t in tables_to_copy:\n", + " q = f'''\n", + " SELECT * FROM {t};\n", + " '''\n", + " df = pd.read_sql(q, con=engine_origin)\n", + " print('-------------------------------------------------------------------------')\n", + " print(f'Loaded Data for Table: {t}...Attempting to Transfer to Destination Server')\n", + " copy_table_data_btw_servers(\n", + " df=df,\n", + " table_name=t,\n", + " engine_destination=engine_destination,\n", + " )\n", + " print(f'Attempting to Truncate Table: {t}...')\n", + " \n", + " ### FOR REALTIME - instead of truncate, need to delete rows using a conditon (e.g. delete all rows <= max timestamp arrival in the DF)\n", + " \n", + " truncate_table(\n", + " engine=engine_origin,\n", + " table=t,\n", + " )\n", + " print(f'...Successfully Truncated Table: {t}')\n", + " print(f'Done Transferring Data for Table: {t}')\n", + " \n" + ] + }, + { + "cell_type": "code", + "execution_count": 59, + "id": "0de1629a", + "metadata": {}, + "outputs": [], + "source": [ + "tables_to_copy = [\n", + " # 'binance_btcusd_trades',\n", + " # 'coinbase_btcusd_trades',\n", + " 'poly_btcusd_trades',\n", + " 'poly_rtds_cl_btcusd',\n", + " # 'user_stream_orders',\n", + " # 'user_stream_trades',\n", + "]" + ] + }, + { + "cell_type": "code", + "execution_count": 60, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "-------------------------------------------------------------------------\n", + "Loaded Data for Table: poly_btcusd_trades...Attempting to Transfer to Destination Server\n", + "SUCCESS: COPIED 720568 to table \"poly_btcusd_trades\" on INTERSERVER_STORAGE\n", + "Attempting to Truncate Table: poly_btcusd_trades...\n", + "...Successfully Truncated Table: poly_btcusd_trades\n", + "Done Transferring Data for Table: poly_btcusd_trades\n", + "-------------------------------------------------------------------------\n", + "Loaded Data for Table: poly_rtds_cl_btcusd...Attempting to Transfer to Destination Server\n", + "SUCCESS: COPIED 73771 to table \"poly_rtds_cl_btcusd\" on INTERSERVER_STORAGE\n", + "Attempting to Truncate Table: poly_rtds_cl_btcusd...\n", + "...Successfully Truncated Table: poly_rtds_cl_btcusd\n", + "Done Transferring Data for Table: poly_rtds_cl_btcusd\n" + ] + } + ], + "source": [ + "backup_all_tables(\n", + " engine_origin=engine,\n", + " engine_destination=engine_inter_storage,\n", + " tables_to_copy=tables_to_copy\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 75, + "id": "85555ab4", + "metadata": {}, + "outputs": [], + "source": [ + "sql = text('''\n", + "OPTIMIZE TABLE binance_btcusd_trades;\n", + "''')\n", + "sql = text('''\n", + "SELECT \n", + " table_name, \n", + " data_length, \n", + " index_length, \n", + " data_free \n", + "FROM information_schema.tables;\n", + "''')" + ] + }, + { + "cell_type": "code", + "execution_count": 70, + "id": "a665c36f", + "metadata": {}, + "outputs": [], + "source": [ + "with engine.connect() as conn:\n", + " conn.execute(sql)\n", + " conn.commit()" + ] + }, + { + "cell_type": "code", + "execution_count": 76, + "id": "db71f3b0", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
TABLE_NAMEDATA_LENGTHINDEX_LENGTHDATA_FREE
0innodb_table_stats16384.00.04194304.0
1innodb_index_stats16384.00.04194304.0
2CHARACTER_SETS0.00.00.0
3CHECK_CONSTRAINTS0.00.00.0
4COLLATIONS0.00.00.0
...............
342user_stream_trades81920.00.00.0
343user_stream_orders16384.00.00.0
344executions_orders16384.00.00.0
345poly_btcusd_trades37289984.00.04194304.0
346binance_btcusd_trades58294272.00.04194304.0
\n", + "

347 rows × 4 columns

\n", + "
" + ], + "text/plain": [ + " TABLE_NAME DATA_LENGTH INDEX_LENGTH DATA_FREE\n", + "0 innodb_table_stats 16384.0 0.0 4194304.0\n", + "1 innodb_index_stats 16384.0 0.0 4194304.0\n", + "2 CHARACTER_SETS 0.0 0.0 0.0\n", + "3 CHECK_CONSTRAINTS 0.0 0.0 0.0\n", + "4 COLLATIONS 0.0 0.0 0.0\n", + ".. ... ... ... ...\n", + "342 user_stream_trades 81920.0 0.0 0.0\n", + "343 user_stream_orders 16384.0 0.0 0.0\n", + "344 executions_orders 16384.0 0.0 0.0\n", + "345 poly_btcusd_trades 37289984.0 0.0 4194304.0\n", + "346 binance_btcusd_trades 58294272.0 0.0 4194304.0\n", + "\n", + "[347 rows x 4 columns]" + ] + }, + "execution_count": 76, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pd.read_sql(sql, con=engine)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b06c6a3e", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
timestamp_arrivaltimestamp_msgtimestamp_valuepriceqtyside_takerup_or_downtimestamp_arrival_dt
01775064793645177506479363017750647936300.59477.003500BUYUP2026-04-01 17:33:13.645
11775064793763177506479375317750647937530.4323.255812BUYDOWN2026-04-01 17:33:13.763
21775064793843177506479383017750647938300.432.325580BUYDOWN2026-04-01 17:33:13.843
31775064793915177506479390517750647939050.5810.020000BUYUP2026-04-01 17:33:13.915
41775064794077177506479406417750647940640.435.000000BUYDOWN2026-04-01 17:33:14.077
...........................
6726291775158573032177515857302217751585730220.373.243242BUYDOWN2026-04-02 19:36:13.032
6726301775158573316177515857330417751585733040.6415.625000BUYUP2026-04-02 19:36:13.316
6726311775158573365177515857335217751585733520.648.200000BUYUP2026-04-02 19:36:13.365
6726321775158573672177515857366117751585736610.37200.000000BUYDOWN2026-04-02 19:36:13.672
6726331775158573933177515857392117751585739210.647.812500BUYUP2026-04-02 19:36:13.933
\n", + "

672634 rows × 8 columns

\n", + "
" + ], + "text/plain": [ + " timestamp_arrival timestamp_msg timestamp_value price qty \\\n", + "0 1775064793645 1775064793630 1775064793630 0.59 477.003500 \n", + "1 1775064793763 1775064793753 1775064793753 0.43 23.255812 \n", + "2 1775064793843 1775064793830 1775064793830 0.43 2.325580 \n", + "3 1775064793915 1775064793905 1775064793905 0.58 10.020000 \n", + "4 1775064794077 1775064794064 1775064794064 0.43 5.000000 \n", + "... ... ... ... ... ... \n", + "672629 1775158573032 1775158573022 1775158573022 0.37 3.243242 \n", + "672630 1775158573316 1775158573304 1775158573304 0.64 15.625000 \n", + "672631 1775158573365 1775158573352 1775158573352 0.64 8.200000 \n", + "672632 1775158573672 1775158573661 1775158573661 0.37 200.000000 \n", + "672633 1775158573933 1775158573921 1775158573921 0.64 7.812500 \n", + "\n", + " side_taker up_or_down timestamp_arrival_dt \n", + "0 BUY UP 2026-04-01 17:33:13.645 \n", + "1 BUY DOWN 2026-04-01 17:33:13.763 \n", + "2 BUY DOWN 2026-04-01 17:33:13.843 \n", + "3 BUY UP 2026-04-01 17:33:13.915 \n", + "4 BUY DOWN 2026-04-01 17:33:14.077 \n", + "... ... ... ... \n", + "672629 BUY DOWN 2026-04-02 19:36:13.032 \n", + "672630 BUY UP 2026-04-02 19:36:13.316 \n", + "672631 BUY UP 2026-04-02 19:36:13.365 \n", + "672632 BUY DOWN 2026-04-02 19:36:13.672 \n", + "672633 BUY UP 2026-04-02 19:36:13.933 \n", + "\n", + "[672634 rows x 8 columns]" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_clob" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "48b47799", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
timestamp_arrivaltimestamp_msgtimestamp_valuepriceqtyside_takerup_or_downtimestamp_arrival_dt
6571181775157300177177515730016617751573001660.482.083332BUYUP2026-04-02 19:15:00.177
6571191775157300554177515730054017751573005400.476.000000SELLUP2026-04-02 19:15:00.554
6571201775157300575177515730056117751573005610.533.000000BUYDOWN2026-04-02 19:15:00.575
6571211775157300645177515730063417751573006340.4829.570000BUYUP2026-04-02 19:15:00.645
6571221775157300689177515730067717751573006770.5020.000000BUYUP2026-04-02 19:15:00.689
...........................
6571931775157304972177515730494017751573049400.5040.000000BUYDOWN2026-04-02 19:15:04.972
6571941775157304979177515730495517751573049550.5040.000000BUYDOWN2026-04-02 19:15:04.979
6571951775157304986177515730496517751573049650.5010.200000BUYDOWN2026-04-02 19:15:04.986
6571961775157304991177515730497317751573049730.506.000000BUYDOWN2026-04-02 19:15:04.991
6571971775157304999177515730498817751573049880.5040.000000BUYDOWN2026-04-02 19:15:04.999
\n", + "

80 rows × 8 columns

\n", + "
" + ], + "text/plain": [ + " timestamp_arrival timestamp_msg timestamp_value price qty \\\n", + "657118 1775157300177 1775157300166 1775157300166 0.48 2.083332 \n", + "657119 1775157300554 1775157300540 1775157300540 0.47 6.000000 \n", + "657120 1775157300575 1775157300561 1775157300561 0.53 3.000000 \n", + "657121 1775157300645 1775157300634 1775157300634 0.48 29.570000 \n", + "657122 1775157300689 1775157300677 1775157300677 0.50 20.000000 \n", + "... ... ... ... ... ... \n", + "657193 1775157304972 1775157304940 1775157304940 0.50 40.000000 \n", + "657194 1775157304979 1775157304955 1775157304955 0.50 40.000000 \n", + "657195 1775157304986 1775157304965 1775157304965 0.50 10.200000 \n", + "657196 1775157304991 1775157304973 1775157304973 0.50 6.000000 \n", + "657197 1775157304999 1775157304988 1775157304988 0.50 40.000000 \n", + "\n", + " side_taker up_or_down timestamp_arrival_dt \n", + "657118 BUY UP 2026-04-02 19:15:00.177 \n", + "657119 SELL UP 2026-04-02 19:15:00.554 \n", + "657120 BUY DOWN 2026-04-02 19:15:00.575 \n", + "657121 BUY UP 2026-04-02 19:15:00.645 \n", + "657122 BUY UP 2026-04-02 19:15:00.689 \n", + "... ... ... ... \n", + "657193 BUY DOWN 2026-04-02 19:15:04.972 \n", + "657194 BUY DOWN 2026-04-02 19:15:04.979 \n", + "657195 BUY DOWN 2026-04-02 19:15:04.986 \n", + "657196 BUY DOWN 2026-04-02 19:15:04.991 \n", + "657197 BUY DOWN 2026-04-02 19:15:04.999 \n", + "\n", + "[80 rows x 8 columns]" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df_clob.loc[(df_clob['timestamp_arrival']>1775157300*1000)&(df_clob['timestamp_arrival']<1775157305*1000)]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e7aa7cfd", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9bc2cecb", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "734c2302", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8a293522", + "metadata": {}, + "outputs": [], + "source": [ + "a" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5ba7be5f", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "py_313", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..a9401d3 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,51 @@ +services: + ws_binance: + container_name: ws_binance + restart: "unless-stopped" + build: + context: ./ + dockerfile: ./ws_binance/Dockerfile + volumes: + - /home/ubuntu/data:/home/ubuntu/data:rw # Read-write access to data + - /home/ubuntu/logs:/home/ubuntu/logs:rw # Read-write access to data + network_mode: "host" + ws_clob: + container_name: ws_clob + restart: "unless-stopped" + build: + context: ./ + dockerfile: ./ws_clob/Dockerfile + volumes: + - /home/ubuntu/data:/home/ubuntu/data:rw # Read-write access to data + - /home/ubuntu/logs:/home/ubuntu/logs:rw # Read-write access to data + network_mode: "host" + ws_rtds: + container_name: ws_rtds + restart: "unless-stopped" + build: + context: ./ + dockerfile: ./ws_rtds/Dockerfile + volumes: + - /home/ubuntu/data:/home/ubuntu/data:rw # Read-write access to data + - /home/ubuntu/logs:/home/ubuntu/logs:rw # Read-write access to data + network_mode: "host" + ws_user: + container_name: ws_user + restart: "unless-stopped" + build: + context: ./ + dockerfile: ./ws_user/Dockerfile + volumes: + - /home/ubuntu/data:/home/ubuntu/data:rw # Read-write access to data + - /home/ubuntu/logs:/home/ubuntu/logs:rw # Read-write access to data + network_mode: "host" + ng: + container_name: ng + restart: "unless-stopped" + build: + context: ./ + dockerfile: ./ng/Dockerfile + volumes: + - /home/ubuntu/data:/home/ubuntu/data:rw # Read-write access to data + - /home/ubuntu/logs:/home/ubuntu/logs:rw # Read-write access to data + network_mode: "host" \ No newline at end of file diff --git a/main copy.py b/main copy.py new file mode 100644 index 0000000..62eeb0e --- /dev/null +++ b/main copy.py @@ -0,0 +1,942 @@ +import asyncio +import json +from dataclasses import dataclass +import logging +import math +import os +import time +from datetime import datetime, timezone +from typing import AsyncContextManager +import traceback +import numpy as np +import pandas as pd +import requests +import talib +import valkey +from dotenv import load_dotenv +from py_clob_client.clob_types import ( + OrderArgs, + OrderType, + PartialCreateOrderOptions, + PostOrdersArgs, + BalanceAllowanceParams, + OpenOrderParams +) +from py_clob_client.order_builder.constants import BUY, SELL +from sqlalchemy import text +from sqlalchemy.ext.asyncio import create_async_engine +from functools import wraps +import modules.api as api + +### Custom Order Args ### +@dataclass +class Custom_OrderArgs(OrderArgs): + max_price: float = 0.00 + post_only: bool = False + + +### Database ### +CLIENT = None +CON: AsyncContextManager | None = None +VAL_KEY = None + +### Logging ### +load_dotenv() +LOG_FILEPATH: str = os.getenv("LOGS_PATH") + '/Polymarket_5min_Algo.log' + +### ALGO CONFIG / CONSTANTS ### +SLOPE_YES_THRESH = 0.01 # In Percent % Chg (e.g. 0.02 == 0.02%) +ENDTIME_BUFFER_SEC = 30 # Stop trading, cancel all open orders and exit positions this many seconds before mkt settles. +TGT_PX_INDEX_DIFF_THRESH = 0.05 # In Percent % Chg (e.g. 0.02 == 0.02%) +DEFAULT_ORDER_SIZE = 5 # In USDe +MIN_ORDER_SIZE = 5 # In USDe +TGT_PROFIT_CENTS = 0.03 +# CHASE_TO_BUY_CENTS = 0.05 +MAX_ALLOWED_POLY_PX = 0.90 + +### GLOBALS ### +ORDER_LOCK = 0 + +SLUG_END_TIME = 0 + +FREE_CASH: float = 0 + +POLY_BINANCE = {} +POLY_REF = {} +POLY_CLOB = {} +POLY_CLOB_DOWN = {} +USER_TRADES = {} +USER_ORDERS = {} +SLOPE_HIST = [] + +LOCAL_ACTIVE_ORDERS = [] +LOCAL_TOKEN_BALANCES = {} +# LOCAL_ACTIVE_POSITIONS = [] + +ACTIVE_BALANCES_EXIST = False ### REMOVE + + +### Decorators ### +def async_timeit(func): + @wraps(func) + async def wrapper(*args, **kwargs): + start_time = time.perf_counter() + try: + return await func(*args, **kwargs) + finally: + end_time = time.perf_counter() + total_time = (end_time - start_time)*1000 + print(f"Function '{func.__name__}' executed in {total_time:.4f} ms") + + return wrapper + +### Database Funcs ### +# @async_timeit +async def create_executions_orders_table( + CON: AsyncContextManager, + engine: str = 'mysql', # mysql | duckdb + ) -> None: + if CON is None: + logging.info("NO DB CONNECTION, SKIPPING Create Statements") + else: + if engine == 'mysql': + logging.info('Creating Table if Does Not Exist: executions_orders') + await CON.execute(text(""" + CREATE TABLE IF NOT EXISTS executions_orders ( + timestamp_sent BIGINT, + token_id VARCHAR(100), + limit_price DOUBLE, + size DOUBLE, + side VARCHAR(8), + order_type VARCHAR(8), + post_only BOOL, + resp_errorMsg VARCHAR(100), + resp_orderID VARCHAR(100), + resp_takingAmount DOUBLE, + resp_makingAmount DOUBLE, + resp_status VARCHAR(20), + resp_success BOOL + ); + """)) + await CON.commit() + else: + raise ValueError('Only MySQL engine is implemented') + +# @async_timeit +async def insert_executions_orders_table( + timestamp_sent: int, + token_id: str, + limit_price: float, + size: float, + side: str, + order_type: str, + post_only: bool, + resp_errorMsg: str, + resp_orderID: str, + resp_takingAmount: float, + resp_makingAmount: float, + resp_status: str, + resp_success: bool, + CON: AsyncContextManager, + engine: str = 'mysql', # mysql | duckdb + ) -> None: + params={ + 'timestamp_sent': timestamp_sent, + 'token_id': token_id, + 'limit_price': limit_price, + 'size': size, + 'side': side, + 'order_type': order_type, + 'post_only': post_only, + 'resp_errorMsg': resp_errorMsg, + 'resp_orderID': resp_orderID, + 'resp_takingAmount': resp_takingAmount, + 'resp_makingAmount': resp_makingAmount, + 'resp_status': resp_status, + 'resp_success': resp_success, + } + if CON is None: + logging.info("NO DB CONNECTION, SKIPPING Insert Statements") + else: + if engine == 'mysql': + await CON.execute(text(""" + INSERT INTO executions_orders + ( + timestamp_sent, + token_id, + limit_price, + size, + side, + order_type, + post_only, + resp_errorMsg, + resp_orderID, + resp_takingAmount, + resp_makingAmount, + resp_status, + resp_success + ) + VALUES + ( + :timestamp_sent, + :token_id, + :limit_price, + :size, + :side, + :order_type, + :post_only, + :resp_errorMsg, + :resp_orderID, + :resp_takingAmount, + :resp_makingAmount, + :resp_status, + :resp_success + ) + """), + parameters=params + ) + await CON.commit() + else: + raise ValueError('Only MySQL engine is implemented') + +### Functions ### +# @async_timeit +def upsert_list_of_dicts_by_id(list_of_dicts, new_dict, id='id'): + for index, item in enumerate(list_of_dicts): + if item.get(id) == new_dict.get(id): + list_of_dicts[index] = new_dict + return list_of_dicts + + list_of_dicts.append(new_dict) + return list_of_dicts + +# @async_timeit +async def slope_decision() -> list[bool, str]: + hist_trades = np.array(POLY_BINANCE.get('hist_trades', [])) + + if ( np.max(hist_trades[:, 0] )*1000 ) - ( np.min(hist_trades[:, 0])*1000 ) < 5: + logging.info('Max - Min Trade In History is < 5 Seconds Apart') + return False, '' + + last_px = POLY_BINANCE['value'] + last_px_ts = POLY_BINANCE['timestamp_value'] + + ts_min_1_sec = last_px_ts - 1000 + price_min_1_sec_index = (np.abs(hist_trades[:, 0] - ts_min_1_sec)).argmin() + price_min_1_sec = hist_trades[:, 1][price_min_1_sec_index] + + ts_min_5_sec = last_px_ts - 5000 + price_min_5_sec_index = (np.abs(hist_trades[:, 0] - ts_min_5_sec)).argmin() + price_min_5_sec = hist_trades[:, 1][price_min_5_sec_index] + + slope = (last_px - price_min_1_sec) / price_min_1_sec + slope_5 = (last_px - price_min_5_sec) / price_min_5_sec + SLOPE_HIST.append(slope) + + # print(f'Avg Binance: {np.mean(hist_trades[:, 1])}') + # print(f'Len Hist : {len(hist_trades[:, 1])}') + # print(f'First Hist : {pd.to_datetime(np.min(hist_trades[:, 0]), unit='ms')}') + # print(f'Latest Hist: {pd.to_datetime(np.max(hist_trades[:, 0]), unit='ms')}') + # print(f'Slope Hist Avg: {np.mean(SLOPE_HIST):.4%}') + # print(f'Slope Hist Max: {np.max(SLOPE_HIST):.4%}') + # print(f'Slope Hist Std: {np.std(SLOPE_HIST):.4%}') + slope_1_buy = abs(slope) >= ( SLOPE_YES_THRESH / 100) + slope_5_buy = abs(slope_5) >= ( SLOPE_YES_THRESH / 100) + + print(f'SLOPE_1: {slope:.4%} == {slope_1_buy}; SLOPE_5: {slope_5:.4%} == {slope_5_buy};') + + ### DECISION ### + if slope_1_buy and slope_5_buy: + print(f'🤑🤑🤑🤑🤑🤑🤑🤑🤑🤑 Slope: {slope_5:.4%};') + side = 'UP' if slope > 0.00 else 'DOWN' + return True, side + else: + return False, '' + +# @async_timeit +async def cancel_all_orders(CLIENT): + logging.info('Attempting to Cancel All Orders') + cxl_resp = CLIENT.cancel_all() + if bool(cxl_resp.get('not_canceled', True)): + logging.warning(f'*** Cancel Request FAILED, trying again and shutting down: {cxl_resp}') + cxl_resp = CLIENT.cancel_all() + raise Exception('*** Cancel Request FAILED') + logging.info(f'Cancel Successful: {cxl_resp}') + +# @async_timeit +async def cancel_single_order_by_id(CLIENT, order_id): + global LOCAL_ACTIVE_ORDERS + + logging.info(f'Attempting to Cancel Single Order: {order_id}') + cxl_resp = CLIENT.cancel(order_id=order_id) + + for idx, o in enumerate(LOCAL_ACTIVE_ORDERS): + if o.get('orderID') == order_id: + if bool(cxl_resp.get('not_canceled', True)): + if cxl_resp.get('not_canceled', {}).get(order_id, None) == "matched orders can't be canceled": + # LOCAL_ACTIVE_ORDERS[idx]['status'] = 'MATCHED' + local_local = LOCAL_ACTIVE_ORDERS.copy() + local_local = local_local[idx] + local_local['status'] = 'MATCHED' + LOCAL_ACTIVE_ORDERS = upsert_list_of_dicts_by_id(LOCAL_ACTIVE_ORDERS, local_local) + logging.info(f'Cancel request failed b/c already matched: {cxl_resp}') + return True + elif cxl_resp.get('not_canceled', {}).get(order_id, None) == "order can't be found - already canceled or matched": + logging.info(f'Cancel request failed b/c already matched or cancelled: {cxl_resp}') + # GET ORDER STATUS + order_status = CLIENT.get_orders( + OpenOrderParams(id=o['orderID']) + )[0]['status'].upper() + logging.info(f'Fetched status from CLOB: {order_status} for order: {o['orderID']}') + if order_status == 'MATCHED': + logging.info('Order is MATCHED') + return True + elif order_status == 'CANCELED': + logging.info('Order is CANCELED') + LOCAL_ACTIVE_ORDERS.pop(idx) + return False + else: + raise ValueError(f'ORDER CXL FAILED AND ORDER STILL SHOWS AS LIVE: {cxl_resp}; STATUS: {order_status}; ID: {o.get('orderID')}') + else: + logging.warning(f'*** Cancel Request FAILED, shutting down: {cxl_resp}') + raise Exception('*** Cancel Request FAILED - SHUTDONW') + else: + LOCAL_ACTIVE_ORDERS.pop(idx) + logging.info(f'Cancel Successful: {cxl_resp}') + return False + +# @async_timeit +async def flatten_open_positions(CLIENT, token_id_up, token_id_down): + up = await get_balance_by_token_id(CLIENT=CLIENT, token_id=token_id_up) + down = await get_balance_by_token_id(CLIENT=CLIENT, token_id=token_id_down) + + logging.info('*********FLATTENING*********') + logging.info(f'UP BALANCE = {up}') + logging.info(f'DOWN BALANCE = {down}') + + ### Submit orders to flatten outstanding balances ### + if abs(up) > MIN_ORDER_SIZE: + logging.info(f'Flattening Up Position: {up}') + await post_order( + CLIENT = CLIENT, + tick_size = POLY_CLOB['tick_size'], + neg_risk = POLY_CLOB['neg_risk'], + OrderArgs_list = [Custom_OrderArgs( + token_id=token_id_up, + price=float(POLY_CLOB['price'])-0.05, + size=up, + side=SELL, + )] + ) + if abs(down) > MIN_ORDER_SIZE: + logging.info(f'Flattening Down Position: {down}') + await post_order( + CLIENT = CLIENT, + tick_size = POLY_CLOB['tick_size'], + neg_risk = POLY_CLOB['neg_risk'], + OrderArgs_list = [Custom_OrderArgs( + token_id=token_id_down, + price=float(POLY_CLOB_DOWN['price'])-0.05, + size=down, + side=SELL, + + )] + ) + logging.info('**************************') + +# @async_timeit +async def get_balance_by_token_id(CLIENT, token_id): + collateral = CLIENT.get_balance_allowance( + BalanceAllowanceParams( + asset_type='CONDITIONAL', + token_id=token_id, + ) + ) + balance = float(collateral['balance']) / 1_000_000 + balance = balance if balance > 4.99 else 0.00 + return balance + +# @async_timeit +async def get_usde_balance(CLIENT): + collateral = CLIENT.get_balance_allowance( + BalanceAllowanceParams( + asset_type='COLLATERAL' + ) + ) + return int(collateral['balance']) / 1_000_000 + +@async_timeit +async def check_for_open_positions(CLIENT, token_id_up, token_id_down): + global LOCAL_TOKEN_BALANCES + + if token_id_up is None or token_id_down is None: + logging.critical('Token Id is None, Exiting') + raise ValueError('Token Id is None, Exiting') + # return False + up = await get_balance_by_token_id(CLIENT=CLIENT, token_id=token_id_up) + down = await get_balance_by_token_id(CLIENT=CLIENT, token_id=token_id_down) + + LOCAL_TOKEN_BALANCES = { + token_id_up: up if up else 0, + token_id_down: down if down else 0, + } + + logging.info(f'LOCAL_TOKEN_BALANCES: {LOCAL_TOKEN_BALANCES}') + + if ( abs(up) > 0 ) or ( abs(down) > 0 ): + return True + else: + return False + +@async_timeit +async def post_order(CLIENT, OrderArgs_list: list[Custom_OrderArgs], tick_size: float | str, neg_risk: bool) -> list[dict]: # Returns order response dict + global LOCAL_ACTIVE_ORDERS + global LOCAL_TOKEN_BALANCES + + orders = [] + for oa in OrderArgs_list: + orders.append( + PostOrdersArgs( + order=CLIENT.create_order( + order_args=oa, + options=PartialCreateOrderOptions( + tick_size=str(tick_size), + neg_risk=neg_risk + ), + ), + orderType=OrderType.GTC, + postOnly=oa.post_only, + ), + ) + + ### POST + response = CLIENT.post_orders(orders) + for idx, d in enumerate(response): + if d['errorMsg'] == '': + d['token_id'] = OrderArgs_list[idx].token_id + if d['token_id'] == POLY_CLOB['token_id_up']: + d['outcome'] = "UP" + elif d['token_id'] == POLY_CLOB['token_id_down']: + d['outcome'] = "DOWN" + else: + d['outcome'] = "UNKNOWN" + raise ValueError(f'UNKNOWN outcome for order: {d}') + + d['price'] = OrderArgs_list[idx].price + d['max_price'] = OrderArgs_list[idx].max_price + d['size'] = OrderArgs_list[idx].size + d['side'] = str(OrderArgs_list[idx].side).upper() + + if d['status'].upper() =='MATCHED': + ### Order Immediately Matched, Can Put in Offsetting Order Depending on State ### + print('******** ORDER APPEND TO LOCAL - MATCHED ********* ') + LOCAL_ACTIVE_ORDERS.append(d) + elif d['status'].upper() == 'CONFIRMED': + current_balance = float(LOCAL_TOKEN_BALANCES.get(d['token_id'], 0.00)) + if d['side'] == 'BUY': + size = float(d['size']) + else: + size = float(d['size']) * -1 + + LOCAL_TOKEN_BALANCES[d['token_id']] = current_balance + size + print('******** TRADE FILLED, BAL UPDATED ********* ') + else: + print('******** ORDER APPEND TO LOCAL - LIVE ********* ') + LOCAL_ACTIVE_ORDERS.append(d) + elif d['errorMsg'] == "invalid post-only order: order crosses book": + await cancel_all_orders(CLIENT=CLIENT) + logging.info(f'invalid post-only order: order crosses book. posted: {OrderArgs_list[idx].price}') + else: + await cancel_all_orders(CLIENT=CLIENT) + raise ValueError(f'Order entry failed: {d}') + + logging.info(f'Order Posted Resp: {response}') + print(f'Order Posted Resp: {response}') + return response + +### Routes ### +async def no_orders(entry_or_exit: str = 'ENTRY'): + global ORDER_LOCK + + ### Check for Price Bands ### + up_px = float(POLY_CLOB.get('price', 0)) + down_px = float(POLY_CLOB_DOWN.get('price', 0)) + + if entry_or_exit == 'ENTRY': + if (up_px > MAX_ALLOWED_POLY_PX) or (down_px > MAX_ALLOWED_POLY_PX): + logging.info(f'Outside max allowed px: {MAX_ALLOWED_POLY_PX}') + return False + + if entry_or_exit == 'ENTRY': + ### Check for Index vs. Target Px ### + tgt_px = float(POLY_CLOB.get('target_price', 0)) + ref_px = float(POLY_REF.get('value')) + tgt_px_diff_to_index = ( abs( tgt_px - ref_px ) / tgt_px) + if tgt_px_diff_to_index > (TGT_PX_INDEX_DIFF_THRESH / 100): + logging.info(f'Tgt Diff to Index Outside Limit ({TGT_PX_INDEX_DIFF_THRESH}%); Diff {tgt_px_diff_to_index:.4%}; Index: {ref_px:.2f}; Tgt: {tgt_px:.2f}') + return False + + ### Check Slope ### + slope_bool, slope_side = await slope_decision() + if not slope_bool: + logging.info('Failed Slope Check') + return False + + token_id_up = POLY_CLOB.get('token_id_up', None) + token_id_down = POLY_CLOB.get('token_id_down', None) + + ### Order Entry ### + if slope_side == 'UP': + if entry_or_exit == 'ENTRY': + side = BUY + size = DEFAULT_ORDER_SIZE + up_px = up_px + 0.01 + down_px = down_px - TGT_PROFIT_CENTS + up_post_only = False + down_post_only = False # T + else: # entry_or_exit == 'EXIT' + side = SELL + size = await get_balance_by_token_id(CLIENT=CLIENT, token_id=token_id_up) + up_px = up_px + TGT_PROFIT_CENTS + down_px = down_px - 0.01 + up_post_only = False # T + down_post_only = False + else: # slope_side == 'DOWN' + if entry_or_exit == 'ENTRY': + side = BUY + size = DEFAULT_ORDER_SIZE + up_px = up_px - TGT_PROFIT_CENTS + down_px = down_px + 0.01 + up_post_only = False # T + down_post_only = False + else: + side = SELL + size = await get_balance_by_token_id(CLIENT=CLIENT, token_id=token_id_up) + up_px = up_px - 0.01 + down_px = down_px + TGT_PROFIT_CENTS + up_post_only = False + down_post_only = False # T + + buy_up_leg = Custom_OrderArgs( + token_id=token_id_up, + price=up_px, + size=size, + side=side, + max_price = 0.99, + post_only=up_post_only + ) + buy_down_leg = Custom_OrderArgs( + token_id=token_id_down, + price=down_px, + size=size, + side=side, + max_price = 0.99, + post_only=down_post_only + ) + order_list = [buy_up_leg, buy_down_leg] + + ### ADD CHECK FOR MKT MOVED AWAY FROM OPPORTNITY ### + + if ORDER_LOCK: + logging.info(f'BUY ORDER BLOCKED BY LOCK: {order_list}') + + else: + logging.info(f'Attempting BUY Order {order_list}') + await post_order( + CLIENT = CLIENT, + tick_size = POLY_CLOB['tick_size'], + neg_risk = POLY_CLOB['neg_risk'], + OrderArgs_list = order_list + ) + ORDER_LOCK = ORDER_LOCK + 1 + +async def active_orders_no_positions_route(): + global LOCAL_ACTIVE_ORDERS + + if len(LOCAL_ACTIVE_ORDERS) > 2: + logging.critical('More than two active orders, shutting down') + await kill_algo() + b_c = 0 + s_c = 0 + + active_buy_up = False + active_buy_down = False + + active_sell_up = False + active_sell_down = False + + for o in LOCAL_ACTIVE_ORDERS: + if o['side'] == 'BUY': + if o['token_id']==POLY_CLOB['token_id_up']: + active_buy_up = True + else: # o['token_id']==POLY_CLOB['token_id_down'] + active_buy_down = True + + b_c = b_c + 1 + elif o['side'] == 'SELL': + if o['token_id']==POLY_CLOB['token_id_up']: + active_sell_up = True + else: # o['token_id']==POLY_CLOB['token_id_down'] + active_sell_down = True + + s_c = s_c + 1 + + if (b_c > 2) or (s_c > 2): + logging.critical(f'More than two active buys or more than two active sells: b_c {b_c}; s_c{s_c}') + await kill_algo() + + for o in LOCAL_ACTIVE_ORDERS: + logging.info(f'Working on order ({o['side']}): {o['orderID']}') + + if o.get('status').upper() == 'MATCHED': + # logging.info('Order is matched, awaiting confirm or kickback') + if active_buy_up and active_buy_down: + logging.info('BUY UP AND BUY DOWN ACTIVE/MATCHED - WAITING FOR CONFIRMS') + continue + logging.info('Order is matched, ordering inverse side') + order_matched=True + elif o.get('status').upper() == 'FAILED': + order_matched=True + raise ValueError(f'Trade FAILED after matching: {o}') + elif o.get('status').upper() == 'RETRYING': + order_matched=True + raise ValueError(f'Trade RETRYING after matching: {o}') + else: + order_matched = False + + orig_px = float(o['price']) + orig_size = float(o['size']) + + ### BUY + if o['side'] == 'BUY': + if POLY_CLOB['token_id_up'] == o['token_id']: + clob_px = float(POLY_CLOB['price']) + else: + clob_px = float(POLY_CLOB_DOWN['price']) + + if (clob_px >= orig_px) or order_matched: + if (clob_px >= orig_px): + logging.info(f"Market px: ({clob_px} is above buy order px: {orig_px:.2f})") + + if (o.get('max_price', 0) > clob_px) or order_matched: + if (o.get('max_price', 0) > clob_px): + logging.info(f"Market px: ({clob_px} has moved too far away from original target, cancelling and resetting algo: {o.get('max_price', 0) :.2f})") + + if not order_matched: + order_matched = await cancel_single_order_by_id(CLIENT=CLIENT, order_id=o['orderID']) + + if order_matched: + o['status'] = 'MATCHED' + + + if order_matched and ( active_buy_up and active_buy_down ): + logging.info('BUY UP AND BUY DOWN MATCHED - WAITING FOR CONFIRMS (IN LOOP)') + continue + else: + token_id = o['token_id'] + px = clob_px+0.01 + max_price = o['max_price'] + post_only = False + + await post_order( + CLIENT = CLIENT, + tick_size = POLY_CLOB['tick_size'], + neg_risk = POLY_CLOB['neg_risk'], + OrderArgs_list = [Custom_OrderArgs( + token_id=token_id, + price=px, + size=orig_size, + side=BUY, + max_price=max_price, + post_only=post_only + + )] + ) + else: + await cancel_single_order_by_id(CLIENT=CLIENT, order_id=o['orderID']) + ### SELL + elif o['side'] == 'SELL': + if POLY_CLOB['token_id_up'] == o['token_id']: + clob_px = float(POLY_CLOB['price']) + else: + clob_px = float(POLY_CLOB_DOWN['price']) + + if (clob_px <= orig_px) or order_matched: + if (clob_px <= orig_px): + logging.info(f"Market px: ({clob_px} is below sell order px: {orig_px:.2f})") + + if not order_matched: + order_matched = await cancel_single_order_by_id(CLIENT=CLIENT, order_id=o['orderID']) + + if order_matched: + o['status'] = 'MATCHED' + + if order_matched and ( active_buy_up and active_buy_down ): + logging.info('SELL UP AND SELL DOWN MATCHED - WAITING FOR CONFIRMS (IN LOOP)') + continue + + if not order_matched: + await post_order( + CLIENT = CLIENT, + tick_size = POLY_CLOB['tick_size'], + neg_risk = POLY_CLOB['neg_risk'], + OrderArgs_list = [Custom_OrderArgs( + token_id=o['token_id'], + price=orig_px-0.01, + size=orig_size, + side=SELL, + max_price = 0.00 + )] + ) + else: + await cancel_single_order_by_id(CLIENT=CLIENT, order_id=o['orderID']) + + +async def no_orders_active_positions_route(): + ''' + Succesful Buy, now neeed to take profit and exit + ''' + global LOCAL_TOKEN_BALANCES + + OrderArgs_list = [] + + logging.warning(f'LOCAL_TOKEN_BALANCES: {LOCAL_TOKEN_BALANCES}') + + for k, v in LOCAL_TOKEN_BALANCES.items(): + size = await get_balance_by_token_id(CLIENT=CLIENT, token_id=k) + if size >= MIN_ORDER_SIZE: + if POLY_CLOB['token_id_up'] == k: + clob_px = float(POLY_CLOB['price']) + else: + clob_px = float(POLY_CLOB_DOWN['price']) + + OrderArgs_list.append( + Custom_OrderArgs( + token_id=k, + price=clob_px + TGT_PROFIT_CENTS, + size=size, + side='SELL', + ) + ) + else: + LOCAL_TOKEN_BALANCES[k] = 0.00 + logging.info(f'Wants to flatten small amount, skipping: {v}') + + if OrderArgs_list: + logging.info(f'Posting orders to close: {OrderArgs_list}') + await post_order( + CLIENT = CLIENT, + tick_size = POLY_CLOB['tick_size'], + neg_risk = POLY_CLOB['neg_risk'], + OrderArgs_list = OrderArgs_list + ) + +async def active_orders_active_positions_route(): + pass + +async def kill_algo(msg: str = 'No kill msg provided'): + logging.info('Killing algo...') + await cancel_all_orders(CLIENT=CLIENT) + await flatten_open_positions( + CLIENT=CLIENT, + token_id_up = POLY_CLOB.get('token_id_up', None), + token_id_down = POLY_CLOB.get('token_id_down', None), + ) + logging.info(f'...algo killed: {msg}') + raise Exception(f'Algo Killed: {msg}') + +async def run_algo(): + global POLY_BINANCE + global POLY_REF + global POLY_CLOB + global POLY_CLOB_DOWN + global USER_TRADES + global USER_ORDERS + + global SLOPE_HIST + global ACTIVE_BALANCES_EXIST + + global LOCAL_ACTIVE_ORDERS + global LOCAL_TOKEN_BALANCES + # global LOCAL_ACTIVE_POSITIONS + + + print(f'token_id_up: {POLY_CLOB.get('token_id_up', None)}') + print(f'token_id_down: {POLY_CLOB.get('token_id_down', None)}') + + + POLY_CLOB = json.loads(VAL_KEY.get('poly_5min_btcusd')) + + ### Check for missing target px (Poly 5min Target BTC Px Target) ### + if POLY_CLOB.get('target_price', 0) <= 1.00: + kill_algo('') + + ACTIVE_BALANCES_EXIST = await check_for_open_positions( + CLIENT=CLIENT, + token_id_up=POLY_CLOB.get('token_id_up', None), + token_id_down=POLY_CLOB.get('token_id_down', None), + ) + + try: + while True: + loop_start = time.time() + print('__________Start___________') + POLY_BINANCE = json.loads(VAL_KEY.get('poly_binance_btcusd')) + POLY_REF = json.loads(VAL_KEY.get('poly_rtds_cl_btcusd')) + POLY_CLOB = json.loads(VAL_KEY.get('poly_5min_btcusd')) + POLY_CLOB_DOWN = json.loads(VAL_KEY.get('poly_5min_btcusd_down')) + USER_TRADES = VAL_KEY.get('poly_user_trades') + USER_TRADES = json.loads(USER_TRADES) if USER_TRADES is not None else [] + USER_ORDERS = VAL_KEY.get('poly_user_orders') + USER_ORDERS = json.loads(USER_ORDERS) if USER_ORDERS is not None else [] + + ### Manage Local vs User Stream Orders ### + # print(f'LOCAL_ACTIVE_ORDERS: {LOCAL_ACTIVE_ORDERS}') + # print(f'USER_TRADES: {USER_TRADES}') + print(f'Len of Active Orders/Matched: {len(LOCAL_ACTIVE_ORDERS)}; User Trades: {len(USER_TRADES)}') + for idx, o in enumerate(LOCAL_ACTIVE_ORDERS): + user_order = next((item for item in USER_ORDERS if item["id"] == o['orderID']), None) + user_trade = next( ( item for item in USER_TRADES if ( o['orderID'] == item['taker_order_id'] ) or ( o["orderID"] == json.loads(item['maker_orders'])[0]['order_id'] ) ), None ) + ### ^ ASSUMPTION BROKEN - MANY IN THIS LIST SO CANT ASSUME FIRST MAKER + + print(f'*****USER TRADE: {user_trade}') + + if user_trade is not None: + trade_status = str(user_trade['status']).upper() + logging.info(f'Updated Trade Status: {o['status']} --> {trade_status}; {o['orderID']}') + if trade_status == 'CONFIRMED': + LOCAL_ACTIVE_ORDERS.pop(idx) + + token_id = user_trade['asset_id'] + current_balance = float(LOCAL_TOKEN_BALANCES.get(token_id, 0.00)) + + if user_trade['side'] == 'BUY': + size = float(user_trade['size']) + else: + size = float(user_trade['size']) * -1 + + LOCAL_TOKEN_BALANCES[token_id] = current_balance + size + + # px = user_trade['price'] + # LOCAL_ACTIVE_POSITIONS.append({ + # 'token_id': token_id, + # 'order_id': o['orderID'], + # 'associate_trades': user_order['associate_trades'], + # 'size_matched': user_order['size_matched'], + # 'price': px, + # 'timestamp_value': user_order['timestamp'], + # }) + logging.info('Order FILLED!') + elif trade_status == 'MATCHED': + logging.info(f'Order Matched...awaiting confirm: {trade_status}') + elif trade_status == 'MINED': + logging.info(f'Order Mined ...awaiting confirm: {trade_status}') + else: + logging.info(f'Trade status but not filled: trade= {user_trade}; order={o}') + + elif user_order is not None: + order_status = str(user_order['status']).upper() + o['status'] = order_status + logging.info(f'Updated Order Status: {o['status']} --> {order_status}; {o['orderID']}') + + if order_status == 'MATCHED': + logging.info('Order MATCHED, awaiting confirm') + + elif order_status == 'CANCELED': + LOCAL_ACTIVE_ORDERS.pop(idx) + logging.info('Order Canceled') + else: + logging.info('Order Live') + + token_id_up = POLY_CLOB.get('token_id_up', 0) + token_id_down = POLY_CLOB.get('token_id_down', 0) + + if (token_id_up is None) or (token_id_down is None): + print('Missing Token Ids for Market, sleeping 1 sec and retrying...') + time.sleep(1) + ACTIVE_BALANCES_EXIST = {} + continue + else: + if (LOCAL_TOKEN_BALANCES.get(token_id_up) is None): + LOCAL_TOKEN_BALANCES[token_id_up] = 0.00 + if (LOCAL_TOKEN_BALANCES.get(token_id_down) is None): + LOCAL_TOKEN_BALANCES[token_id_down] = 0.00 + ACTIVE_BALANCES_EXIST = (LOCAL_TOKEN_BALANCES.get(token_id_up) > 0) or (LOCAL_TOKEN_BALANCES.get(token_id_down) > 0) + + ### Check for Endtime Buffer ### + if ENDTIME_BUFFER_SEC > POLY_CLOB.get('sec_remaining', 0): + if LOCAL_ACTIVE_ORDERS: + print('buffer zone - orders cancel') + await cancel_all_orders(CLIENT=CLIENT) + if ACTIVE_BALANCES_EXIST: + print('buffer zone - flatten positions') + await flatten_open_positions( + CLIENT=CLIENT, + token_id_up = POLY_CLOB.get('token_id_up', None), + token_id_down = POLY_CLOB.get('token_id_down', None), + ) + + print('buffer zone, sleeping until next session') + time.sleep(1) + continue + + ### Execution Route ### + if not(LOCAL_ACTIVE_ORDERS) and not(ACTIVE_BALANCES_EXIST): # No Orders, No Positions + print('ROUTE: no_orders_no_positions_route') + await no_orders(entry_or_exit='ENTRY') + + ### Open Orders Route ### + elif LOCAL_ACTIVE_ORDERS and not(ACTIVE_BALANCES_EXIST): # Orders, No Positions + print('ROUTE: active_orders_no_positions_route') + # await active_orders_no_positions_route() + + ### Open Positions Route ### + elif not(LOCAL_ACTIVE_ORDERS) and ACTIVE_BALANCES_EXIST: # No Orders, Positions + print('ROUTE: no_orders_active_positions_route') + # await no_orders(entry_or_exit='EXIT') + + ### Open Orders and Open Positions Route ### + else: + print('ROUTE: active_orders_active_positions_route - BETA') + # await active_orders_no_positions_route() # Orders and Positions + + print(f'__________________________ (Algo Engine ms: {(time.time() - loop_start)*1000})') + time.sleep(1) + except KeyboardInterrupt: + print('...algo stopped') + await cancel_all_orders(CLIENT=CLIENT) + except Exception as e: + logging.critical(f'*** ALGO ENGINE CRASHED: {e}') + logging.error(traceback.format_exc()) + await cancel_all_orders(CLIENT=CLIENT) + + +async def main(): + global CLIENT + global VAL_KEY + global CON + + CLIENT = api.create_client() + VAL_KEY = valkey.Valkey(host='localhost', port=6379, db=0, decode_responses=True) + engine = create_async_engine('mysql+asyncmy://root:pwd@localhost/polymarket') + + async with engine.connect() as CON: + await create_executions_orders_table(CON=CON) + await run_algo() + +if __name__ == '__main__': + START_TIME = round(datetime.now().timestamp()*1000) + + logging.info(f'Log FilePath: {LOG_FILEPATH}') + + logging.basicConfig( + force=True, + filename=LOG_FILEPATH, + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + filemode='w' + ) + logging.info(f"STARTED: {START_TIME}") + + asyncio.run(main()) + \ No newline at end of file diff --git a/main.py b/main.py new file mode 100644 index 0000000..5c4cf55 --- /dev/null +++ b/main.py @@ -0,0 +1,1049 @@ +import asyncio +import json +from dataclasses import dataclass, asdict +import logging +import math +import os +import time +from datetime import datetime, timezone +from typing import AsyncContextManager +import traceback +import numpy as np +import pandas as pd +import requests +import talib +import valkey +from dotenv import load_dotenv +from py_clob_client.clob_types import ( + OrderArgs, + OrderType, + PartialCreateOrderOptions, + PostOrdersArgs, + BalanceAllowanceParams, + OpenOrderParams, + MarketOrderArgs +) +from py_clob_client.order_builder.constants import BUY, SELL +from sqlalchemy import text +from sqlalchemy.ext.asyncio import create_async_engine +from functools import wraps +import modules.api as api + +### Custom Order Args ### +# @dataclass +# class Custom_OrderArgs(OrderArgs): +# max_price: float = 0.00 +# post_only: bool = False + +@dataclass +class Custom_PostOrdersArgs(PostOrdersArgs): + token_id: str = '' + price: float = 0.00 + # max_price: float = 0.00 + size: float = 0.00 + side: str = '' + + +### Database ### +CLIENT = None +CON: AsyncContextManager | None = None +VAL_KEY = None + +### Logging ### +load_dotenv() +LOG_FILEPATH: str = os.getenv("LOGS_PATH") + '/Polymarket_Algo.log' + +### ALGO CONFIG / CONSTANTS ### +SLOPE_YES_THRESH = 0.00750 # In Percent % Chg (e.g. 0.02 == 0.02%) +SLOPE_YES_THRESH_1 = 0.00750 # In Percent % Chg (e.g. 0.02 == 0.02%) +ENDTIME_BUFFER_SEC = 30 # Stop trading, cancel all open orders and exit positions this many seconds before mkt settles. +TGT_PX_INDEX_DIFF_THRESH = 0.05 # In Percent % Chg (e.g. 0.02 == 0.02%) +DEFAULT_ORDER_SIZE = 6 # In USDe +MIN_ORDER_SIZE = 5 # In USDe +TGT_PROFIT_CENTS = 0.08 +# CHASE_TO_BUY_CENTS = 0.05 +MAX_ALLOWED_POLY_PX = 0.90 +MAX_LEG_LIVE_SEC = 3.25 + + +### GLOBALS ### +LOOP_LAST_ROUTE: str = '' +ORDER_LOCK = 0 +FIRST_LOOP_NEW_MKT = False + +SLUG_END_TIME = 0 + +FREE_CASH: float = 0 + +POLY_BINANCE = {} +POLY_REF = {} +POLY_CLOB = {} +POLY_CLOB_DOWN = {} +USER_TRADES = {} +USER_ORDERS = {} +SLOPE_HIST = [] + +LOCAL_ACTIVE_ORDERS = [] +LOCAL_MATCHED_ORDERS = [] +LOCAL_TOKEN_BALANCES = {} +# LOCAL_ACTIVE_POSITIONS = [] + +ACTIVE_BALANCES_EXIST = False ### REMOVE + + +### Decorators ### +def async_timeit(func): + @wraps(func) + async def wrapper(*args, **kwargs): + start_time = time.perf_counter() + try: + return await func(*args, **kwargs) + finally: + end_time = time.perf_counter() + total_time = (end_time - start_time)*1000 + logging.info(f"Function '{func.__name__}' executed in {total_time:.4f} ms") + + return wrapper + +### Database Funcs ### +# @async_timeit +async def create_executions_orders_table( + CON: AsyncContextManager, + engine: str = 'mysql', # mysql | duckdb + ) -> None: + if CON is None: + logging.info("NO DB CONNECTION, SKIPPING Create Statements") + else: + if engine == 'mysql': + logging.info('Creating Table if Does Not Exist: executions_orders') + await CON.execute(text(""" + CREATE TABLE IF NOT EXISTS executions_orders ( + timestamp_sent BIGINT, + token_id VARCHAR(100), + limit_price DOUBLE, + size DOUBLE, + side VARCHAR(8), + order_type VARCHAR(8), + post_only BOOL, + resp_errorMsg VARCHAR(100), + resp_orderID VARCHAR(100), + resp_takingAmount DOUBLE, + resp_makingAmount DOUBLE, + resp_status VARCHAR(20), + resp_success BOOL + ); + """)) + await CON.commit() + else: + raise ValueError('Only MySQL engine is implemented') + +# @async_timeit +async def insert_executions_orders_table( + timestamp_sent: int, + token_id: str, + limit_price: float, + size: float, + side: str, + order_type: str, + post_only: bool, + resp_errorMsg: str, + resp_orderID: str, + resp_takingAmount: float, + resp_makingAmount: float, + resp_status: str, + resp_success: bool, + CON: AsyncContextManager, + engine: str = 'mysql', # mysql | duckdb + ) -> None: + params={ + 'timestamp_sent': timestamp_sent, + 'token_id': token_id, + 'limit_price': limit_price, + 'size': size, + 'side': side, + 'order_type': order_type, + 'post_only': post_only, + 'resp_errorMsg': resp_errorMsg, + 'resp_orderID': resp_orderID, + 'resp_takingAmount': resp_takingAmount, + 'resp_makingAmount': resp_makingAmount, + 'resp_status': resp_status, + 'resp_success': resp_success, + } + if CON is None: + logging.info("NO DB CONNECTION, SKIPPING Insert Statements") + else: + if engine == 'mysql': + await CON.execute(text(""" + INSERT INTO executions_orders + ( + timestamp_sent, + token_id, + limit_price, + size, + side, + order_type, + post_only, + resp_errorMsg, + resp_orderID, + resp_takingAmount, + resp_makingAmount, + resp_status, + resp_success + ) + VALUES + ( + :timestamp_sent, + :token_id, + :limit_price, + :size, + :side, + :order_type, + :post_only, + :resp_errorMsg, + :resp_orderID, + :resp_takingAmount, + :resp_makingAmount, + :resp_status, + :resp_success + ) + """), + parameters=params + ) + await CON.commit() + else: + raise ValueError('Only MySQL engine is implemented') + +### Functions ### +# @async_timeit +def upsert_list_of_dicts_by_id(list_of_dicts, new_dict, id='id'): + for index, item in enumerate(list_of_dicts): + if item.get(id) == new_dict.get(id): + list_of_dicts[index] = new_dict + return list_of_dicts + + list_of_dicts.append(new_dict) + return list_of_dicts + +# @async_timeit +async def slope_decision(slope_yes_thresh) -> list[bool, str]: + hist_trades = np.array(POLY_BINANCE.get('hist_trades', [])) + + min_trade_hist_ts = np.min(hist_trades[:, 0]) + max_trade_hist_ts = np.max(hist_trades[:, 0] ) + if ( max_trade_hist_ts ) - ( min_trade_hist_ts ) < 5: + logging.info(f'Max - Min Trade In History is < 5 Seconds Apart: {max_trade_hist_ts} - {min_trade_hist_ts}') + return False, '' + + last_px = POLY_BINANCE['value'] + last_px_ts = POLY_BINANCE['timestamp_value'] + + ts_min_1_sec = last_px_ts - 1000 + price_min_1_sec_index = (np.abs(hist_trades[:, 0] - ts_min_1_sec)).argmin() + price_min_1_sec = hist_trades[:, 1][price_min_1_sec_index] + + ts_min_5_sec = last_px_ts - 5000 + price_min_5_sec_index = (np.abs(hist_trades[:, 0] - ts_min_5_sec)).argmin() + price_min_5_sec = hist_trades[:, 1][price_min_5_sec_index] + + slope = (last_px - price_min_1_sec) / price_min_1_sec + slope_5 = (last_px - price_min_5_sec) / price_min_5_sec + SLOPE_HIST.append(slope) + + # print(f'Avg Binance: {np.mean(hist_trades[:, 1])}') + # print(f'Len Hist : {len(hist_trades[:, 1])}') + # print(f'First Hist : {pd.to_datetime(np.min(hist_trades[:, 0]), unit='ms')}') + # print(f'Latest Hist: {pd.to_datetime(np.max(hist_trades[:, 0]), unit='ms')}') + # print(f'Slope Hist Avg: {np.mean(SLOPE_HIST):.4%}') + # print(f'Slope Hist Max: {np.max(SLOPE_HIST):.4%}') + # print(f'Slope Hist Std: {np.std(SLOPE_HIST):.4%}') + slope_1_buy = abs(slope) >= ( slope_yes_thresh / 100) + slope_5_buy = abs(slope_5) >= ( slope_yes_thresh / 100) + + ### DECISION ### + if slope_1_buy and slope_5_buy: + side = 'UP' if slope > 0.00 else 'DOWN' + print(f'🤑 {round(datetime.now().timestamp()*1000)}: Slope_1: {slope_5:.4%}; Slope_5: {slope_5:.4%}; SIDE: {side}') + logging.info(f'🤑 {round(datetime.now().timestamp()*1000)}: Slope_1: {slope_5:.4%}; Slope_5: {slope_5:.4%}; SIDE: {side}') + return True, side + elif abs(slope) >= ( 0.001 / 100): + print(f'{round(datetime.now().timestamp()*1000)}: SLOPE_1: {slope:.4%}; SLOPE_5: {slope_5:.4%};') + return False, '' + else: + return False, '' + +# @async_timeit +async def cancel_all_orders(CLIENT): + logging.info('Attempting to Cancel All Orders') + cxl_resp = CLIENT.cancel_all() + if bool(cxl_resp.get('not_canceled', True)): + logging.warning(f'*** Cancel Request FAILED, trying again and shutting down: {cxl_resp}') + cxl_resp = CLIENT.cancel_all() + raise Exception('*** Cancel Request FAILED') + logging.info(f'Cancel Successful: {cxl_resp}') + +# @async_timeit +async def cancel_single_order_by_id(CLIENT, order_id): + global LOCAL_ACTIVE_ORDERS + + logging.info(f'Attempting to Cancel Single Order: {order_id}') + cxl_resp = CLIENT.cancel(order_id=order_id) + + for idx, o in enumerate(LOCAL_ACTIVE_ORDERS): + if o.get('orderID') == order_id: + if bool(cxl_resp.get('not_canceled', True)): + if cxl_resp.get('not_canceled', {}).get(order_id, None) == "matched orders can't be canceled": + # LOCAL_ACTIVE_ORDERS[idx]['status'] = 'MATCHED' + local_local = LOCAL_ACTIVE_ORDERS.copy() + local_local = local_local[idx] + local_local['status'] = 'MATCHED' + LOCAL_ACTIVE_ORDERS = upsert_list_of_dicts_by_id(LOCAL_ACTIVE_ORDERS, local_local) + logging.info(f'Cancel request failed b/c already matched: {cxl_resp}') + return True + elif cxl_resp.get('not_canceled', {}).get(order_id, None) == "order can't be found - already canceled or matched": + logging.info(f'Cancel request failed b/c already matched or cancelled: {cxl_resp}') + # GET ORDER STATUS + order_status = CLIENT.get_orders( + OpenOrderParams(id=o['orderID']) + )[0]['status'].upper() + logging.info(f'Fetched status from CLOB: {order_status} for order: {o['orderID']}') + if order_status == 'MATCHED': + logging.info('Order is MATCHED') + return True + elif order_status == 'CANCELED': + logging.info('Order is CANCELED') + LOCAL_ACTIVE_ORDERS.pop(idx) + return False + else: + raise ValueError(f'ORDER CXL FAILED AND ORDER STILL SHOWS AS LIVE: {cxl_resp}; STATUS: {order_status}; ID: {o.get('orderID')}') + else: + logging.warning(f'*** Cancel Request FAILED, shutting down: {cxl_resp}') + raise Exception('*** Cancel Request FAILED - SHUTDONW') + else: + LOCAL_ACTIVE_ORDERS.pop(idx) + logging.info(f'Cancel Successful: {cxl_resp}') + return False + +# @async_timeit +async def flatten_open_positions(CLIENT, token_id_up, token_id_down): + up_size = await get_balance_by_token_id(CLIENT=CLIENT, token_id=token_id_up) + down_size = await get_balance_by_token_id(CLIENT=CLIENT, token_id=token_id_down) + + up_size = round(up_size, 2) + down_size = round(up_size, 2) + + logging.info('*********FLATTENING*********') + logging.info(f'UP BALANCE = {up_size}') + logging.info(f'DOWN BALANCE = {down_size}') + + ### Submit orders to flatten outstanding balances ### + order_list = [] + + if up_size: + logging.info(f'Flattening Up Position: {up_size}') + # up_px_worst = round(float(POLY_CLOB['price'])-0.05, 2) + order_list.append(MarketOrderArgs( + token_id = token_id_up, + amount = up_size, + # size = up_size, + price = 0.01, + # max_price = 0.99, + side = SELL, + order_type = OrderType.FAK + )) + if down_size: + order_list.append(MarketOrderArgs( + token_id = token_id_down, + amount = down_size, + # size = down_size, + price = 0.01, + # max_price = 0.99, + side = SELL, + order_type = OrderType.FAK + )) + logging.info(f'Flattening Down Position: {down_size}') + if order_list: + await post_order( + CLIENT = CLIENT, + PostOrdersArgs_list = order_list, + is_mkt_order_list = True + ) + +# @async_timeit +async def get_balance_by_token_id(CLIENT, token_id): + collateral = CLIENT.get_balance_allowance( + BalanceAllowanceParams( + asset_type='CONDITIONAL', + token_id=token_id, + ) + ) + balance = float(collateral['balance']) / 1_000_000 + logging.info(f'Balance: {balance}; Collateral: {collateral}') + balance = balance if balance >= 0.01 else 0.00 + return balance + +# @async_timeit +async def get_usde_balance(CLIENT): + collateral = CLIENT.get_balance_allowance( + BalanceAllowanceParams( + asset_type='COLLATERAL' + ) + ) + return int(collateral['balance']) / 1_000_000 + +@async_timeit +async def check_for_open_positions(CLIENT, token_id_up, token_id_down): + global LOCAL_TOKEN_BALANCES + + if token_id_up is None or token_id_down is None: + logging.critical('Token Id is None, Exiting') + raise ValueError('Token Id is None, Exiting') + # return False + up = await get_balance_by_token_id(CLIENT=CLIENT, token_id=token_id_up) + down = await get_balance_by_token_id(CLIENT=CLIENT, token_id=token_id_down) + + LOCAL_TOKEN_BALANCES = { + token_id_up: up if up else 0, + token_id_down: down if down else 0, + } + + logging.info(f'LOCAL_TOKEN_BALANCES: {LOCAL_TOKEN_BALANCES}') + + if ( abs(up) > 0 ) or ( abs(down) > 0 ): + return True + else: + return False + +@async_timeit +async def post_order(CLIENT, PostOrdersArgs_list: list[Custom_PostOrdersArgs], is_mkt_order_list: bool = False) -> list[dict]: # Returns order response dict + global LOCAL_ACTIVE_ORDERS + global LOCAL_MATCHED_ORDERS + global LOCAL_TOKEN_BALANCES + + ### POST + if is_mkt_order_list: + timestamp_post_sent = round(datetime.now().timestamp()*1000) + response = [] + for o in PostOrdersArgs_list: + response.append(CLIENT.post_order(o)) + else: + timestamp_post_sent = round(datetime.now().timestamp()*1000) + response = CLIENT.post_orders(PostOrdersArgs_list) + for idx, d in enumerate(response): + if d['errorMsg'] == '': + d['timestamp_post_sent'] = timestamp_post_sent + d['timestamp_post_resp_rec'] = round(datetime.now().timestamp()*1000) + + d['token_id'] = PostOrdersArgs_list[idx].token_id + if d['token_id'] == POLY_CLOB['token_id_up']: + d['outcome'] = "UP" + elif d['token_id'] == POLY_CLOB['token_id_down']: + d['outcome'] = "DOWN" + else: + d['outcome'] = "UNKNOWN" + raise ValueError(f'UNKNOWN outcome for order: {d}') + + d['price'] = PostOrdersArgs_list[idx].price + + # d['max_price'] = PostOrdersArgs_list[idx].max_price + if is_mkt_order_list: + d['size'] = PostOrdersArgs_list[idx].amount + else: + d['size'] = PostOrdersArgs_list[idx].size + d['side'] = str(PostOrdersArgs_list[idx].side).upper() + + if d['status'].upper() =='MATCHED': + ### Order Immediately Matched, Can Put in Offsetting Order Depending on State ### + logging.info('******** ORDER APPEND TO LOCAL - MATCHED ********* ') + LOCAL_MATCHED_ORDERS.append(d) + elif d['status'].upper() == 'CONFIRMED': + current_balance = float(LOCAL_TOKEN_BALANCES.get(d['token_id'], 0.00)) + if d['side'] == 'BUY': + size = float(d['size']) + else: + size = float(d['size']) * -1 + + LOCAL_TOKEN_BALANCES[d['token_id']] = current_balance + size + logging.info('******** TRADE FILLED, BAL UPDATED ********* ') + else: + logging.info('******** ORDER APPEND TO LOCAL - LIVE ********* ') + LOCAL_ACTIVE_ORDERS.append(d) + elif d['errorMsg'] == "invalid post-only order: order crosses book": + await cancel_all_orders(CLIENT=CLIENT) + logging.info(f'invalid post-only order: order crosses book. posted: {PostOrdersArgs_list[idx].price}') + else: + await cancel_all_orders(CLIENT=CLIENT) + raise ValueError(f'Order entry failed: {d}') + + logging.info(f'🚨 Order Posted Resp: {response}') + return response + + +### Routes ### +# @async_timeit +async def no_orders_route(entry_or_exit: str = 'ENTRY'): + global ORDER_LOCK + + ### Check for Price Bands ### + up_last_px = float(POLY_CLOB.get('price', 0)) + down_last_px = float(POLY_CLOB_DOWN.get('price', 0)) + + if entry_or_exit == 'ENTRY': + if (up_last_px > MAX_ALLOWED_POLY_PX) or (down_last_px > MAX_ALLOWED_POLY_PX): + logging.info(f'Outside max allowed px: {MAX_ALLOWED_POLY_PX}') + return False + + if entry_or_exit == 'ENTRY': + ### Check for Index vs. Target Px ### + tgt_px = float(POLY_CLOB.get('target_price', 0)) + ref_px = float(POLY_REF.get('value')) + tgt_px_diff_to_index = ( abs( tgt_px - ref_px ) / tgt_px) + if tgt_px_diff_to_index > (TGT_PX_INDEX_DIFF_THRESH / 100): + logging.info(f'Tgt Diff to Index Outside Limit ({TGT_PX_INDEX_DIFF_THRESH}%); Diff {tgt_px_diff_to_index:.4%}; Index: {ref_px:.2f}; Tgt: {tgt_px:.2f}') + return False + + token_id_up = POLY_CLOB.get('token_id_up', None) + token_id_down = POLY_CLOB.get('token_id_down', None) + + if entry_or_exit == 'EXIT': + # size_up = await get_balance_by_token_id(CLIENT=CLIENT, token_id=token_id_up) + size_up = LOCAL_TOKEN_BALANCES[token_id_up] + # size_down = await get_balance_by_token_id(CLIENT=CLIENT, token_id=token_id_down) + size_down = LOCAL_TOKEN_BALANCES[token_id_down] + size_up = round(size_up, 6) + size_down = round(size_down, 6) + size_less_than_min = (size_up < MIN_ORDER_SIZE) or (size_down < MIN_ORDER_SIZE) + # logging.info(f"EXITING: size_less_than_min: {size_less_than_min}; up: {size_up}; down: {size_down};") + else: + size_less_than_min = False + + if not size_less_than_min: + ### Check Slope ### + slope_bool, slope_side = await slope_decision(slope_yes_thresh=SLOPE_YES_THRESH) + if not slope_bool: + # logging.info('Failed Slope Check') + return False + else: + slope_bool, slope_side = False, 'MKT' + ### Order Entry ### + if slope_side == 'MKT': + side = SELL + size = min([size_up, size_down]) + up_px = 0.01 + down_px = 0.01 + up_post_only = False + down_post_only = False # T + order_type = OrderType.FAK + logging.info(f'Flattening Residuals - Mkt Order: ({size} ({side}) ({slope_side}))') + elif slope_side == 'UP': + if entry_or_exit == 'ENTRY': + side = BUY + size = DEFAULT_ORDER_SIZE + up_px = round(up_last_px + 0.01, 2) + down_px = round(down_last_px - TGT_PROFIT_CENTS + 0.01, 2) + up_post_only = False + down_post_only = False # T + order_type = OrderType.GTC + else: # entry_or_exit == 'EXIT' + side = SELL + size = size_up + logging.info(f'Flattening Residuals - Limit Order: ({size} ({side}) ({slope_side}))') + up_px = round(up_last_px + TGT_PROFIT_CENTS + 0.01, 2) + down_px = round(down_last_px - 0.01, 2) + order_type = OrderType.GTC + up_post_only = False # T + down_post_only = False + else: # slope_side == 'DOWN' + if entry_or_exit == 'ENTRY': + side = BUY + size = DEFAULT_ORDER_SIZE + up_px = round(up_last_px - TGT_PROFIT_CENTS + 0.01, 2) + down_px = round(down_last_px + 0.01, 2) + up_post_only = False # T + down_post_only = False + order_type = OrderType.GTC + else: # entry_or_exit == 'EXIT' + side = SELL + size = size_down + logging.info(f'Flattening Residuals - Limit Order: ({size} ({side}) ({slope_side}))') + up_px = round(up_last_px - 0.01, 2) + down_px = round(down_last_px + TGT_PROFIT_CENTS + 0.01, 2) + order_type = OrderType.GTC + up_post_only = False + down_post_only = False # T + + up_leg = Custom_PostOrdersArgs( + order=CLIENT.create_order( + order_args=OrderArgs( + token_id=token_id_up, + price=up_px, + size=size, + side=side, + ), + options=PartialCreateOrderOptions( + tick_size=str(POLY_CLOB['tick_size']), + neg_risk=POLY_CLOB['neg_risk'] + ), + ), + orderType = order_type, + postOnly = up_post_only, + token_id = token_id_up, + price = up_px, + # max_price = 0.99, + size = size, + side = side + ) + down_leg = Custom_PostOrdersArgs( + order=CLIENT.create_order( + order_args=OrderArgs( + token_id=token_id_down, + price=down_px, + size=size, + side=side, + ), + options=PartialCreateOrderOptions( + tick_size=str(POLY_CLOB['tick_size']), + neg_risk=POLY_CLOB['neg_risk'] + ), + ), + orderType = order_type, + postOnly = down_post_only, + token_id = token_id_down, + price = down_px, + # max_price = 0.99, + size = size, + side = side + ) + + ### ADD CHECK FOR MKT MOVED AWAY FROM OPPORTUNITY ### + if slope_side == 'MKT': + order_list = [up_leg, down_leg] + elif slope_side == 'UP': + order_list = [up_leg, down_leg] + vk_px = float( json.loads( VAL_KEY.get('poly_5min_btcusd') )['price'] ) + if up_px < vk_px: + logging.info(f'ABANDONED BUY ORDERS: UP px moved from {up_px} -> {vk_px}') + return False + else: + logging.info(f'NOT ABANDONED BUY ORDERS: UP px moved from {up_px} -> {vk_px}') + else: + order_list = [down_leg, up_leg] + vk_px = float( json.loads( VAL_KEY.get('poly_5min_btcusd_down') )['price'] ) + if down_px < vk_px: + logging.info(f'ABANDONED BUY ORDERS: DOWN px moved from {down_px} -> {vk_px}') + return False + else: + logging.info(f'NOT ABANDONED BUY ORDERS: UP px moved from {up_px} -> {vk_px}') + + logging.info('PRICES AT TIME OF ORDER:') + logging.info(f'Current TS: {round(datetime.now().timestamp()*1000)}') + logging.info(f'up_last_px: {up_last_px}; order up_px: {up_px}') + logging.info(f'down_last_px: {down_last_px}; order down_px: {down_px}') + logging.info(f'TGT_PROFIT_CENTS: {TGT_PROFIT_CENTS}') + + if ORDER_LOCK: + logging.info(f'BUY ORDER BLOCKED BY LOCK: {order_list}') + + else: + logging.info(f'Attempting {entry_or_exit} Orders {order_list}') + await post_order( + CLIENT = CLIENT, + PostOrdersArgs_list = order_list + ) + # ORDER_LOCK = ORDER_LOCK + 1 + +# @async_timeit +async def active_orders_route(): + global LOCAL_ACTIVE_ORDERS + global LOCAL_MATCHED_ORDERS + + if len(LOCAL_ACTIVE_ORDERS) > 2: + logging.critical('More than two active orders, shutting down') + await kill_algo('More than two active orders, shutting down') + + if len(LOCAL_MATCHED_ORDERS) > 2: + logging.critical('More than two matched orders, shutting down') + await kill_algo('More than two matched orders, shutting down') + + for idx, o in enumerate(LOCAL_ACTIVE_ORDERS): + replace_w_order_at_mkt = False + + if o.get('status').upper() == 'MATCHED': + logging.info(f'Active Order MATCHED. Moving to LOCAL_MATCHED_ORDERS. Order Id: {o['orderID']}') + LOCAL_MATCHED_ORDERS.append(o) + LOCAL_ACTIVE_ORDERS.pop(idx) + continue + elif o.get('status').upper() == 'LIVE': + ts_now = round(datetime.now().timestamp()*1000) + ts_order_post = o['timestamp_post_sent'] + sec_order_live = (ts_now - ts_order_post) / 1000 + logging.info(f'Working on order ({o['side']}) ({o['outcome']}): {o['orderID']}; SEC ALIVE: {sec_order_live:.2f}') + + ### Check Conditions to Immediately Replace Order at Mkt (Abandon Target Px) ### + if (sec_order_live > MAX_LEG_LIVE_SEC): # Abandon if lived longer than x seconds + logging.info(f'Order live > max sec ({sec_order_live} > {MAX_LEG_LIVE_SEC}); {o['side']}) ({o['outcome']}): {o['orderID']}') + replace_w_order_at_mkt = True + + if not replace_w_order_at_mkt: + slope_bool, slope_side = await slope_decision(slope_yes_thresh=SLOPE_YES_THRESH / 2) # Abandon if slope has reversed + if slope_bool and (slope_side == o['outcome']): + logging.info(f'SLOPE MOVED AWAY FROM TGT ORDER, replacing at mkt; ({o['side']}) ({o['outcome']}): {o['orderID']}') + replace_w_order_at_mkt = True + + if replace_w_order_at_mkt: + order_matched = await cancel_single_order_by_id(CLIENT=CLIENT, order_id=o['orderID']) + if order_matched: + logging.info(f'Order is MATCHED after being worked: {o}') + LOCAL_MATCHED_ORDERS.append(o) + LOCAL_ACTIVE_ORDERS.pop(idx) + continue + else: + token_id = o['token_id'] + if POLY_CLOB['token_id_up'] == token_id: + clob_px = float(POLY_CLOB['price']) + else: + clob_px = float(POLY_CLOB_DOWN['price']) + + ### BUY + if o['side'] == 'BUY': + px = round(clob_px + 0.02, 2) + side = BUY + # max_price = o['max_price'] + size = float(o['size']) + post_only = False + + ### SELL + elif o['side'] == 'SELL': + px = round(clob_px - 0.02, 2) + side = SELL + # max_price = o['max_price'] + size = float(o['size']) + post_only = False + + if size < MIN_ORDER_SIZE: + order_type = OrderType.FAK + else: + order_type = OrderType.GTC + + logging.info(f'REPLACING ORDER. Orig Px {o['price']} -> Mkt Px: {clob_px}; {o['side']}) ({o['outcome']}): {o['orderID']}') + + order = Custom_PostOrdersArgs( + order=CLIENT.create_order( + order_args=OrderArgs( + token_id=token_id, + price=px, + size=size, + side=side, + ), + options=PartialCreateOrderOptions( + tick_size=str(POLY_CLOB['tick_size']), + neg_risk=POLY_CLOB['neg_risk'] + ), + ), + orderType = order_type, + postOnly = post_only, + token_id = token_id, + price = px, + # max_price = max_price, + size = size, + side = side + ) + + await post_order( + CLIENT = CLIENT, + PostOrdersArgs_list = [order] + ) + elif o.get('status').upper() == 'FAILED': + raise ValueError(f'Trade FAILED after matching: {o}') + elif o.get('status').upper() == 'RETRYING': + raise ValueError(f'Trade RETRYING after matching: {o}') + else: + raise ValueError(f'Unexpected Order Status: {o}') + +@async_timeit +async def kill_algo(msg: str = 'No kill msg provided'): + logging.info('Killing algo...') + await cancel_all_orders(CLIENT=CLIENT) + await flatten_open_positions( + CLIENT=CLIENT, + token_id_up = POLY_CLOB.get('token_id_up', None), + token_id_down = POLY_CLOB.get('token_id_down', None), + ) + logging.info(f'...algo killed: {msg}') + raise Exception(f'Algo Killed: {msg}') + +@async_timeit +async def clob_client_caching(token_id): + # logging.info('CLIENT CACHING') + tick_size = CLIENT.get_tick_size(token_id=token_id) + # logging.info(f'Tick Size: {tick_size}') + neg_risk = CLIENT.get_neg_risk(token_id=token_id) + # logging.info(f'Is Negative Risk: {neg_risk}') + fee_rate_bps = CLIENT.get_fee_rate_bps(token_id=token_id) + # logging.info(f'Fee Rate Bps: {fee_rate_bps}') + # logging.info('CLIENT CACHING COMPLETE') + +# @async_timeit +async def loop_check_route_switch(route_name): + global LOOP_LAST_ROUTE + + if LOOP_LAST_ROUTE != route_name: + print(f'SWITCHING ROUTES: {LOOP_LAST_ROUTE} -> {route_name}') + logging.info(f'SWITCHING ROUTES: {LOOP_LAST_ROUTE} -> {route_name}') + if route_name == 'no_orders_route_EXIT': + await check_for_open_positions( + CLIENT=CLIENT, + token_id_up=POLY_CLOB.get('token_id_up', None), + token_id_down=POLY_CLOB.get('token_id_down', None), + ) + LOOP_LAST_ROUTE = route_name + + +async def run_algo(): + global POLY_BINANCE + global POLY_REF + global POLY_CLOB + global POLY_CLOB_DOWN + global USER_TRADES + global USER_ORDERS + + global SLOPE_HIST + global ACTIVE_BALANCES_EXIST + global FIRST_LOOP_NEW_MKT + + global LOCAL_ACTIVE_ORDERS + global LOCAL_MATCHED_ORDERS + global LOCAL_TOKEN_BALANCES + # global LOCAL_ACTIVE_POSITIONS + + POLY_CLOB = json.loads(VAL_KEY.get('poly_5min_btcusd')) + + ### Get Token IDs ### + token_id_up = POLY_CLOB.get('token_id_up', None) + token_id_down = POLY_CLOB.get('token_id_down', None) + + if (token_id_up is None) or (token_id_down is None): + raise ValueError(f'Token ID is None: UP: {token_id_up}; DOWN: {token_id_down}') + + logging.info(f'token_id_up: {POLY_CLOB.get('token_id_up', None)}') + logging.info(f'token_id_down: {POLY_CLOB.get('token_id_down', None)}') + + ### CLOB Client Caching ### + await clob_client_caching(token_id=token_id_up) + await clob_client_caching(token_id=token_id_down) + + ### Get Initial Balances ### + ACTIVE_BALANCES_EXIST = await check_for_open_positions( + CLIENT=CLIENT, + token_id_up=POLY_CLOB.get('token_id_up', None), + token_id_down=POLY_CLOB.get('token_id_down', None), + ) + + ### Check for missing target px (Poly 5min Target BTC Px Target) ### + if POLY_CLOB.get('target_price', 0) <= 1.00: + kill_algo('Missing target_price, check CLOB feedhandler') + + try: + while True: + # loop_start = time.time() + # print('__________Start___________') + POLY_BINANCE = json.loads(VAL_KEY.get('poly_binance_btcusd')) + POLY_REF = json.loads(VAL_KEY.get('poly_rtds_cl_btcusd')) + POLY_CLOB = json.loads(VAL_KEY.get('poly_5min_btcusd')) + POLY_CLOB_DOWN = json.loads(VAL_KEY.get('poly_5min_btcusd_down')) + USER_TRADES = VAL_KEY.get('poly_user_trades') + USER_TRADES = json.loads(USER_TRADES) if USER_TRADES is not None else [] + USER_ORDERS = VAL_KEY.get('poly_user_orders') + USER_ORDERS = json.loads(USER_ORDERS) if USER_ORDERS is not None else [] + + ### Check for Token Id + token_id_up = POLY_CLOB.get('token_id_up', None) + token_id_down = POLY_CLOB.get('token_id_down', None) + + if (token_id_up is None) or (token_id_down is None): + logging.info(f'Missing Token Ids for Market (token_id_up: {token_id_up}; token_id_down: {token_id_down}), sleeping 1 sec and retrying...') + time.sleep(1) + ACTIVE_BALANCES_EXIST = {} + continue + + if FIRST_LOOP_NEW_MKT: + ### CLOB Client Caching ### + await clob_client_caching(token_id=token_id_up) + await clob_client_caching(token_id=token_id_down) + FIRST_LOOP_NEW_MKT = False + + for idx, o in enumerate(LOCAL_ACTIVE_ORDERS): + if USER_TRADES: + for t in USER_TRADES: + if t['trader_side']=='MAKER': + user_trade = next( ( item for item in json.loads(t['maker_orders']) if ( o['orderID'] == item['order_id'] ) ), None ) + if user_trade: + user_trade['status'] = t['status'] + user_trade['size'] = float(user_trade['matched_amount']) + # logging.info(f'********** MAKER TRADE IN USER TRADES: {user_trade} *******') + elif t['taker_order_id'] == o["orderID"]: + user_trade = t + else: + user_trade = None + + if user_trade: + trade_status = str(user_trade['status']).upper() + if trade_status != o['status'].upper(): + logging.info(f'Updated Trade Status: {o['status'].upper()} --> {trade_status}; {o['orderID']}') + logging.info(f'Trade Details: {user_trade}') + o['status'] = trade_status + + if trade_status == 'CONFIRMED': + LOCAL_ACTIVE_ORDERS.pop(idx) + + token_id = user_trade['asset_id'] + current_balance = float(LOCAL_TOKEN_BALANCES.get(token_id, 0.00)) + + if user_trade['side'] == 'BUY': + size = float(user_trade['size']) + else: + size = float(user_trade['size']) * -1 + + LOCAL_TOKEN_BALANCES[token_id] = current_balance + size + + logging.info('Order FILLED! - IN LOCAL_ACTIVE_ORDERS') + elif trade_status == 'MATCHED': + logging.info(f'Order MATCHED. Moving to LOCAL_MATCHED_ORDERS. Trade Status: {trade_status}') + LOCAL_MATCHED_ORDERS.append(o) + LOCAL_ACTIVE_ORDERS.pop(idx) + elif trade_status == 'MINED': + logging.info(f'Order Mined ...awaiting confirm: {trade_status}') + else: + logging.info(f'Trade status but not filled: trade= {user_trade}; order={o}') + + for idx, o in enumerate(LOCAL_MATCHED_ORDERS): + if USER_TRADES: + for t in USER_TRADES: + if t['trader_side']=='MAKER': + user_trade = next( ( item for item in json.loads(t['maker_orders']) if ( o['orderID'] == item['order_id'] ) ), None ) + if user_trade: + user_trade['status'] = t['status'] + user_trade['size'] = float(user_trade['matched_amount']) + # logging.info(f'********** MAKER TRADE IN USER TRADES: {user_trade} *******') + elif t['taker_order_id'] == o["orderID"]: + user_trade = t + else: + user_trade = None + + if user_trade: + trade_status = str(user_trade['status']).upper() + if trade_status != o['status'].upper(): + logging.info(f'Updated Trade Status: {o['status']} --> {trade_status}; {o['orderID']}') + o['status'] = trade_status + + if trade_status == 'CONFIRMED': + LOCAL_MATCHED_ORDERS.pop(idx) + + token_id = user_trade['asset_id'] + current_balance = float(LOCAL_TOKEN_BALANCES.get(token_id, 0.00)) + + if user_trade['side'] == 'BUY': + size = float(user_trade['size']) + else: + size = float(user_trade['size']) * -1 + + LOCAL_TOKEN_BALANCES[token_id] = current_balance + size + + logging.info('Matched order CONFIRMED! - IN LOCAL_MATCHED_ORDERS') + elif trade_status == 'MATCHED': + # logging.info(f'Order Matched...awaiting confirm: {trade_status}') + pass + elif trade_status == 'MINED': + # logging.info(f'Order Mined...awaiting confirm: {trade_status}') + pass + else: + logging.info(f'Trade status but not filled: trade= {user_trade}; order={o}') + + + ### CHECK BALANCES ### + if (LOCAL_TOKEN_BALANCES.get(token_id_up) is None): + LOCAL_TOKEN_BALANCES[token_id_up] = 0.00 + if (LOCAL_TOKEN_BALANCES.get(token_id_down) is None): + LOCAL_TOKEN_BALANCES[token_id_down] = 0.00 + ACTIVE_BALANCES_EXIST = (abs(LOCAL_TOKEN_BALANCES.get(token_id_up)) > 0) or abs((LOCAL_TOKEN_BALANCES.get(token_id_down)) > 0) + + ### Check for Endtime Buffer ### + if ENDTIME_BUFFER_SEC > POLY_CLOB.get('sec_remaining', 0): + FIRST_LOOP_NEW_MKT = True + if LOCAL_ACTIVE_ORDERS: + print('buffer zone - orders cancel') + await cancel_all_orders(CLIENT=CLIENT) + if ACTIVE_BALANCES_EXIST: + print('buffer zone - flatten positions') + await flatten_open_positions( + CLIENT=CLIENT, + token_id_up = POLY_CLOB.get('token_id_up', None), + token_id_down = POLY_CLOB.get('token_id_down', None), + ) + print('buffer zone, sleeping until next session') + time.sleep(1) + continue + + ### ENTRY Route ### + if not(LOCAL_ACTIVE_ORDERS) and not(LOCAL_MATCHED_ORDERS) and not(ACTIVE_BALANCES_EXIST): # No Orders, No Matched, No Positions + await loop_check_route_switch('no_orders_route_ENTRY') + await no_orders_route(entry_or_exit='ENTRY') + + ### Open Orders Route ### + elif LOCAL_ACTIVE_ORDERS: # Any Active Orders + await loop_check_route_switch('active_orders_route') + await active_orders_route() + + ### Matched Orders Route - Waiting ### + elif LOCAL_MATCHED_ORDERS: # Any Active Orders + await loop_check_route_switch('matched_orders_awaiting_confirms') + # await active_orders_route() + + ### Open Positions Route - EXIT ### + elif not(LOCAL_ACTIVE_ORDERS) and not(LOCAL_MATCHED_ORDERS) and ACTIVE_BALANCES_EXIST: # No Orders, No Matches, Positions + await loop_check_route_switch('no_orders_route_EXIT') + await no_orders_route(entry_or_exit='EXIT') + # time.sleep(0.5) + else: + print('ROUTE: NOT IMPLEMENTED') + + # print(f'__________________________ (Algo Engine ms: {(time.time() - loop_start)*1000})') + # time.sleep(3) + except KeyboardInterrupt: + print('...algo stopped') + await cancel_all_orders(CLIENT=CLIENT) + except Exception as e: + logging.critical(f'*** ALGO ENGINE CRASHED: {e}') + logging.error(traceback.format_exc()) + await cancel_all_orders(CLIENT=CLIENT) + + +async def main(): + global CLIENT + global VAL_KEY + global CON + + CLIENT = api.create_client() + VAL_KEY = valkey.Valkey(host='localhost', port=6379, db=0, decode_responses=True) + engine = create_async_engine('mysql+asyncmy://root:pwd@localhost/polymarket') + + async with engine.connect() as CON: + await create_executions_orders_table(CON=CON) + await run_algo() + +if __name__ == '__main__': + START_TIME = round(datetime.now().timestamp()*1000) + + logging.info(f'Log FilePath: {LOG_FILEPATH}') + + logging.basicConfig( + force=True, + filename=LOG_FILEPATH, + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + filemode='w' + ) + logging.info(f"STARTED: {START_TIME}") + + asyncio.run(main()) + \ No newline at end of file diff --git a/modules/__pycache__/api.cpython-313.pyc b/modules/__pycache__/api.cpython-313.pyc index 41ce6d9..3f48f5c 100644 Binary files a/modules/__pycache__/api.cpython-313.pyc and b/modules/__pycache__/api.cpython-313.pyc differ diff --git a/ng.py b/ng.py new file mode 100644 index 0000000..2d44cd8 --- /dev/null +++ b/ng.py @@ -0,0 +1,117 @@ +import os +from nicegui import ui, app +from sqlalchemy import create_engine +# import requests +import json +# import time +# import re +import valkey +# import asyncio +# import datetime as dt +# from random import random +# from nicegui_modules import data +# from nicegui_modules import ui_components +# from glide import GlideClient, NodeAddress, GlideClientConfiguration + + +LISTENING_CLIENT = None +LH_PAIR = 'BTC' +RH_PAIR = 'USD' + +DEFAULT_TO_DARKMODE: bool = True +ALLOW_BODY_SCROLL: bool = True +LOOKBACK: int = 60 +LOOKBACK_RT_TV_MAX_POINTS: int = 3000 +REFRESH_INTERVAL_SEC: int = 10 +REFRESH_INTERVAL_RT_SEC: int = 1/30 + +ENGINE = create_engine('mysql+pymysql://root:pwd@localhost/polymarket') +VALKEY_R = valkey.Valkey(host='localhost', port=6379, db=0, decode_responses=True) +# VALKEY_P = VALKEY_R.pubsub() +# VALKEY_P.subscribe('mexc_mkt_bookTicker') + + +def root(): + app.add_static_files(max_cache_age=0, url_path='/static', local_directory=os.path.join(os.path.dirname(__file__), 'nicegui_modules/static')) + ui.add_head_html(''' + + + + + ''' + ) + + # ui.add_head_html('') + update_body_scroll(bool_override=ALLOW_BODY_SCROLL) + + ui.sub_pages({ + '/': rt_chart_page, + }).classes('w-full') + + +async def update_tv(): + series_update = json.loads(VALKEY_R.get('poly_rtds_cl_btcusd')) + series_update_b = json.loads(VALKEY_R.get('poly_binance_btcusd')) + series_update_c = json.loads(VALKEY_R.get('poly_5min_btcusd')) + timestamp = round( ( series_update['timestamp_arrival'] / 1000 ) , 2) + timestamp_b = round( ( series_update_b['timestamp_arrival'] / 1000 ) , 2) + timestamp_c = round( ( series_update_c['timestamp_arrival'] / 1000 ) , 2) + value = float(series_update['value']) + value_b = float(series_update_b['value']) + value_c = float(series_update_c['price']) + + data_dict = { + 'timestamp': timestamp, + 'timestamp_b': timestamp_b, + 'timestamp_c': timestamp_c, + 'value': value, + 'value_b': value_b, + 'value_c': value_c, + 'target': series_update_c['target_price'], + 'LOOKBACK_RT_TV_MAX_POINTS': LOOKBACK_RT_TV_MAX_POINTS, + } + + ui.run_javascript(f'await update_tv(data_dict={data_dict});') + + +def update_body_scroll(e=None, bool_override=False): + if e is None: + if bool_override: + ui.query('body').style('height: 100%; overflow-y: auto;') + else: + ui.query('body').style('height: 100%; overflow-y: hidden;') + else: + if e.value: + ui.query('body').style('height: 100%; overflow-y: auto;') + else: + ui.query('body').style('height: 100%; overflow-y: hidden;') + +# async def refresh_lookback_funcs(lookback: int = LOOKBACK): +# lookback = app.storage.user.get('lookback', lookback) + +# await data.trades_pnl_graph.refresh(ENGINE=ENGINE, LH_PAIR=LH_PAIR, RH_PAIR=RH_PAIR, lookback=lookback) +# await ui_components.er_table.refresh(ENGINE=ENGINE, lookback=lookback) +# await ui_components.trades_table.refresh(ENGINE=ENGINE, lookback=lookback) +# await ui_components.er_stats.refresh(ENGINE=ENGINE, lookback=lookback) + +async def rt_chart_page(): + global LOOKBACK + + LOOKBACK = app.storage.user.get('lookback', LOOKBACK) + timer = ui.timer(REFRESH_INTERVAL_RT_SEC, update_tv) + + with ui.row(): + with ui.column(): + ui.switch('☸︎', value=ALLOW_BODY_SCROLL, on_change=lambda e: update_body_scroll(e)) + with ui.column(): + ui.switch('▶️', value=True).bind_value_to(timer, 'active') + with ui.column().style('position: absolute; right: 20px; font-family: monospace; align-self: center;'): + ui.label('Atwater Trading: Orderbook') + + with ui.grid(columns=16).classes('w-full gap-0 auto-fit'): + with ui.card().tight().classes('w-full col-span-full no-shadow border border-black-200').style('overflow: auto;'): + ui.html('
', sanitize=False).classes('w-full') + ui.run_javascript('await create_tv();') + + +ui.run(root, storage_secret="123ABC", reload=True, dark=True, title='Atwater Trading') \ No newline at end of file diff --git a/ng/Dockerfile b/ng/Dockerfile new file mode 100644 index 0000000..042d9b9 --- /dev/null +++ b/ng/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3.13-slim + +RUN apt-get update && \ + apt-get install -y build-essential + +RUN gcc --version +RUN rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt . + +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +# Finally, run gunicorn. +CMD [ "python", "ng.py"] +# CMD [ "gunicorn", "--workers=5", "--threads=1", "-b 0.0.0.0:8000", "app:server"] \ No newline at end of file diff --git a/nicegui_modules/static/script.js b/nicegui_modules/static/script.js new file mode 100644 index 0000000..aedf603 --- /dev/null +++ b/nicegui_modules/static/script.js @@ -0,0 +1,223 @@ +async function waitForVariable(variableName, timeout = 5000) { + const startTime = Date.now(); + while (typeof window[variableName] === 'undefined') { + if (Date.now() - startTime > timeout) { + throw new Error(`Variable '${variableName}' not defined within ${timeout}ms`); + } + await new Promise(resolve => setTimeout(resolve, 100)); + } + console.log(`Variable '${variableName}' is now defined.`); +} + +async function update_tv(data_dict) { + + window.data.push({ time: data_dict.timestamp, value: data_dict.value }); + window.data_b.push({ time: data_dict.timestamp_b, value: data_dict.value_b }); + window.data_c.push({ time: data_dict.timestamp_c, value: data_dict.value_c }); + window.data_tgt.push({ time: data_dict.timestamp_c, value: data_dict.target }); + window.lineSeries.update({ time: data_dict.timestamp, value: data_dict.value }); + window.lineSeries_b.update({ time: data_dict.timestamp_b, value: data_dict.value_b }); + window.lineSeries_c.update({ time: data_dict.timestamp_c, value: data_dict.value_c }); + window.lineSeries_tgt.update({ time: data_dict.timestamp_c, value: data_dict.target }); + + // midPriceLine.applyOptions({ + // price: data_dict.mid_px, + // color: '#c78228', + // lineWidth: 3, + // lineStyle: LightweightCharts.LineStyle.Dashed, + // axisLabelVisible: true, + // }); + + window.chart.timeScale().scrollToRealTime(); + // const currentRange = window.chart.timeScale().getVisibleLogicalRange(); + // window.chart.timeScale().fitContent(); + // window.chart.timeScale().setVisibleLogicalRange(currentRange); + + const MAX_DATA_POINTS = data_dict.LOOKBACK_RT_TV_MAX_POINTS; + if (window.lineSeries.data().length > MAX_DATA_POINTS) { + window.lineSeries.setData(lineSeries.data().slice(-MAX_DATA_POINTS)); + } + if (window.lineSeries_b.data().length > MAX_DATA_POINTS) { + window.lineSeries_b.setData(lineSeries_b.data().slice(-MAX_DATA_POINTS)); + } + if (window.lineSeries_c.data().length > MAX_DATA_POINTS) { + window.lineSeries_c.setData(lineSeries_c.data().slice(-MAX_DATA_POINTS)); + } + if (window.lineSeries_tgt.data().length > MAX_DATA_POINTS) { + window.lineSeries_tgt.setData(lineSeries_tgt.data().slice(-MAX_DATA_POINTS)); + } +}; + + +async function create_tv() { + window.chart = LightweightCharts.createChart(document.getElementById('tv'), + { + autoSize: true, + toolbox: true, + timeScale: { + timeVisible: true, // Shows HH:mm on x-axis + secondsVisible: true // Optional: show seconds + }, + rightPriceScale: { + visible: true, + autoScale: true + }, + leftPriceScale: { + visible: true + }, + + layout: { + background: { type: 'solid', color: '#222' }, + textColor: '#DDD', + }, + grid: { + vertLines: { + color: '#e1e1e1', // Set vertical line color + visible: true, + style: 2, // 0: Solid, 1: Dashed, 2: Dotted, 3: LargeDashed, 4: SparseDotted + }, + horzLines: { + color: '#e1e1e1', // Set horizontal line color + visible: true, + style: 2, + }, + }, + + crosshair: { mode: LightweightCharts.CrosshairMode.Normal }, + } + ); + window.lineSeries = chart.addSeries(LightweightCharts.LineSeries, { + color: '#94fcdf', + priceScaleId: 'right' + // topColor: '#94fcdf', + // bottomColor: 'rgba(112, 171, 249, 0.28)', + // invertFilledArea: false + }); + window.lineSeries_b = chart.addSeries(LightweightCharts.LineSeries, { + color: '#dd7525', + priceScaleId: 'right' + // topColor: '#94fcdf', + // bottomColor: 'rgba(112, 171, 249, 0.28)', + // invertFilledArea: false + }); + window.lineSeries_c = chart.addSeries(LightweightCharts.LineSeries, { + color: '#ea0707', + priceScaleId: 'left', + autoscaleInfoProvider: () => ({ + priceRange: { + minValue: 0.0, + maxValue: 1.0 + } + }) + // topColor: '#94fcdf', + // bottomColor: 'rgba(112, 171, 249, 0.28)', + // invertFilledArea: false + }); + window.lineSeries_tgt = chart.addSeries(LightweightCharts.LineSeries, { + color: '#ffffff', + priceScaleId: 'right', + lineStyle: LightweightCharts.LineStyle.Dashed + // topColor: '#94fcdf', + // bottomColor: 'rgba(112, 171, 249, 0.28)', + // invertFilledArea: false + }); + // window.midPriceLine_Config = { + // price: 0, + // color: '#c78228', + // lineWidth: 3, + // lineStyle: LightweightCharts.LineStyle.Dashed, + // axisLabelVisible: false, + // }; + // window.midPriceLine = window.lineSeries.createPriceLine(midPriceLine_Config); + window.data = []; + window.data_b = []; + window.data_c = []; + window.data_tgt = []; + window.lineSeries.setData(window.data); + window.lineSeries_b.setData(window.data_b); + window.lineSeries_c.setData(window.data_c); + window.lineSeries_tgt.setData(window.data_tgt); + + // Create and style the tooltip html element + const container = document.getElementById('tv'); + + window.toolTipWidth = 200; + + const toolTip = document.createElement('div'); + toolTip.style = `width: ${window.toolTipWidth}px; height: 100%; position: absolute; display: none; padding: 8px; box-sizing: border-box; font-size: 12px; text-align: left; z-index: 1000; top: 12px; left: 12px; pointer-events: none; border-radius: 4px 4px 0px 0px; border-bottom: none; box-shadow: 0 2px 5px 0 rgba(117, 134, 150, 0.45);font-family: -apple-system, BlinkMacSystemFont, 'Trebuchet MS', Roboto, Ubuntu, sans-serif; -webkit-font-smoothing: antialiased; -moz-osx-font-smoothing: grayscale;`; + toolTip.style.background = `rgba(${'0, 0, 0'}, 0.25)`; + toolTip.style.color = 'white'; + toolTip.style.borderColor = 'rgba( 239, 83, 80, 1)'; + container.appendChild(toolTip); + + // update tooltip + window.chart.subscribeCrosshairMove(async param => { + + if ( + param.point === undefined || + !param.time || + param.point.x < 0 || + param.point.x > container.clientWidth || + param.point.y < 0 || + param.point.y > container.clientHeight + ) { + toolTip.style.display = 'none'; + } else { + + // toolTip.style.height = '100%'; + toolTip.style.alignContent = 'center'; + + const dateStr = new Date(param.time*1000).toISOString(); + + let data = await param.seriesData.get(window.lineSeries); + if (data === undefined) { + data = {} + data.value = 0 + console.log('data is UNDEFINED, SETTING TO 0') + }; + + let data_b = await param.seriesData.get(window.lineSeries_b); + if (data_b === undefined) { + data_b = {} + data_b.value = 0 + console.log('data is UNDEFINED, SETTING TO 0') + }; + + const value_px = data.value + const value_px_b = window.data_b.value + const value_px_c = window.data_c.value + const value_px_tgt = window.data_tgt.value + + toolTip.style.display = 'block'; + //
+ // Atwater Trading + //
+ toolTip.innerHTML = ` +
+ Chainlink: ${Math.round(100 * value_px) / 100} + Binance: ${Math.round(100 * value_px_b) / 100} +
+
+ ${dateStr} +
+ `; + + let left = param.point.x; // relative to timeScale + const timeScaleWidth = chart.timeScale().width(); + const priceScaleWidth = chart.priceScale('left').width(); + const halfTooltipWidth = toolTipWidth / 2; + left += priceScaleWidth - halfTooltipWidth; + left = Math.min(left, priceScaleWidth + timeScaleWidth - toolTipWidth); + left = Math.max(left, priceScaleWidth); + + toolTip.style.left = left + 'px'; + toolTip.style.top = 0 + 'px'; + } + }); + + + + window.chart.timeScale().fitContent(); + + console.log("TV Created!") +}; \ No newline at end of file diff --git a/nicegui_modules/static/styles.css b/nicegui_modules/static/styles.css new file mode 100644 index 0000000..7dc00b2 --- /dev/null +++ b/nicegui_modules/static/styles.css @@ -0,0 +1,33 @@ +/* Sticky Quasar Table for Dark Mode */ +.table-sticky-dark .q-table__top, +.table-sticky-dark .q-table__bottom, +.table-sticky-dark thead tr:first-child th { + background-color: black; +} +.table-sticky-dark thead tr th { + position: sticky; + z-index: 1; +} +.table-sticky-dark thead tr:first-child th { + top: 0; +} +.table-sticky-dark tbody { + scroll-margin-top: 48px; +} + +/* Sticky Quasar Table for Light Mode */ +/* .table-sticky-light .q-table__top, +.table-sticky-light .q-table__bottom, +.table-sticky-light thead tr:first-child th { + background-color: rgb(229, 223, 223); +} +.table-sticky-light thead tr th { + position: sticky; + z-index: 1; +} +.table-sticky-light thead tr:first-child th { + top: 0; +} +.table-sticky-light tbody { + scroll-margin-top: 48px; +} */ \ No newline at end of file diff --git a/order_entry.ipynb b/order_entry.ipynb index 2b6a579..e0628c4 100644 --- a/order_entry.ipynb +++ b/order_entry.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 64, + "execution_count": 1, "id": "c0bfb3b5", "metadata": {}, "outputs": [], @@ -15,13 +15,13 @@ "import json\n", "from dataclasses import dataclass\n", "\n", - "from py_clob_client.clob_types import OrderArgs, OrderType, PostOrdersArgs, PartialCreateOrderOptions\n", + "from py_clob_client.clob_types import OrderArgs, OrderType, PostOrdersArgs, PartialCreateOrderOptions, BalanceAllowanceParams, OpenOrderParams\n", "from py_clob_client.order_builder.constants import BUY, SELL\n" ] }, { "cell_type": "code", - "execution_count": 65, + "execution_count": 2, "id": "7d7dc787", "metadata": {}, "outputs": [], @@ -49,17 +49,17 @@ }, { "cell_type": "code", - "execution_count": 66, + "execution_count": 3, "id": "c3e07e21", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "Timestamp('2026-03-27 03:15:00')" + "Timestamp('2026-04-04 05:15:00')" ] }, - "execution_count": 66, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -74,22 +74,123 @@ }, { "cell_type": "code", - "execution_count": 67, + "execution_count": 4, + "id": "10671da4", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'id': '1840714',\n", + " 'question': 'Bitcoin Up or Down - April 4, 1:15AM-1:20AM ET',\n", + " 'conditionId': '0xa71ed436d160cc45bf182b9004e0c10b16322ea1b41375182b747c2629223ecc',\n", + " 'slug': 'btc-updown-5m-1775279700',\n", + " 'resolutionSource': 'https://data.chain.link/streams/btc-usd',\n", + " 'endDate': '2026-04-04T05:20:00Z',\n", + " 'liquidity': '19009.3694',\n", + " 'startDate': '2026-04-03T05:23:41.149918Z',\n", + " 'image': 'https://polymarket-upload.s3.us-east-2.amazonaws.com/BTC+fullsize.png',\n", + " 'icon': 'https://polymarket-upload.s3.us-east-2.amazonaws.com/BTC+fullsize.png',\n", + " 'description': 'This market will resolve to \"Up\" if the Bitcoin price at the end of the time range specified in the title is greater than or equal to the price at the beginning of that range. Otherwise, it will resolve to \"Down\".\\nThe resolution source for this market is information from Chainlink, specifically the BTC/USD data stream available at https://data.chain.link/streams/btc-usd.\\nPlease note that this market is about the price according to Chainlink data stream BTC/USD, not according to other sources or spot markets.',\n", + " 'outcomes': '[\"Up\", \"Down\"]',\n", + " 'outcomePrices': '[\"0.505\", \"0.495\"]',\n", + " 'volume': '1149.1770909999996',\n", + " 'active': True,\n", + " 'closed': False,\n", + " 'marketMakerAddress': '',\n", + " 'createdAt': '2026-04-03T05:22:26.236646Z',\n", + " 'updatedAt': '2026-04-04T05:14:58.886334Z',\n", + " 'new': False,\n", + " 'featured': False,\n", + " 'archived': False,\n", + " 'restricted': True,\n", + " 'groupItemThreshold': '0',\n", + " 'questionID': '0x1f7b9fd2711422d90794895b99edcb93a81246afd56b08ee4736c9b57565f8f7',\n", + " 'enableOrderBook': True,\n", + " 'orderPriceMinTickSize': 0.01,\n", + " 'orderMinSize': 5,\n", + " 'volumeNum': 1149.1770909999996,\n", + " 'liquidityNum': 19009.3694,\n", + " 'endDateIso': '2026-04-04',\n", + " 'startDateIso': '2026-04-03',\n", + " 'hasReviewedDates': True,\n", + " 'volume24hr': 1149.1770909999998,\n", + " 'volume1wk': 1149.1770909999998,\n", + " 'volume1mo': 1149.1770909999998,\n", + " 'volume1yr': 1149.1770909999998,\n", + " 'clobTokenIds': '[\"111612048087962925846397645788043113901565915142888535086349545305985085081594\", \"12874695654084258339187997850118078073676562138805309130550135898707856234061\"]',\n", + " 'volume24hrClob': 1149.1770909999998,\n", + " 'volume1wkClob': 1149.1770909999998,\n", + " 'volume1moClob': 1149.1770909999998,\n", + " 'volume1yrClob': 1149.1770909999998,\n", + " 'volumeClob': 1149.1770909999996,\n", + " 'liquidityClob': 19009.3694,\n", + " 'makerBaseFee': 1000,\n", + " 'takerBaseFee': 1000,\n", + " 'acceptingOrders': True,\n", + " 'negRisk': False,\n", + " 'ready': False,\n", + " 'funded': False,\n", + " 'acceptingOrdersTimestamp': '2026-04-03T05:22:35Z',\n", + " 'cyom': False,\n", + " 'competitive': 0.9999750006249843,\n", + " 'pagerDutyNotificationEnabled': False,\n", + " 'approved': True,\n", + " 'rewardsMinSize': 50,\n", + " 'rewardsMaxSpread': 4.5,\n", + " 'spread': 0.01,\n", + " 'lastTradePrice': 0.51,\n", + " 'bestBid': 0.5,\n", + " 'bestAsk': 0.51,\n", + " 'automaticallyActive': True,\n", + " 'clearBookOnStart': False,\n", + " 'showGmpSeries': False,\n", + " 'showGmpOutcome': False,\n", + " 'manualActivation': False,\n", + " 'negRiskOther': False,\n", + " 'umaResolutionStatuses': '[]',\n", + " 'pendingDeployment': False,\n", + " 'deploying': False,\n", + " 'rfqEnabled': False,\n", + " 'eventStartTime': '2026-04-04T05:15:00Z',\n", + " 'holdingRewardsEnabled': False,\n", + " 'feesEnabled': True,\n", + " 'requiresTranslation': False,\n", + " 'makerRebatesFeeShareBps': 10000,\n", + " 'feeType': 'crypto_fees_v2',\n", + " 'feeSchedule': {'exponent': 1,\n", + " 'rate': 0.072,\n", + " 'takerOnly': True,\n", + " 'rebateRate': 0.2}}" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "market" + ] + }, + { + "cell_type": "code", + "execution_count": 5, "id": "5ba43ffc", "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'Up': '97875487643168796351669326324566509161830383659944117871160601839654217457417',\n", - " 'Down': '96344823573113580705457152659674776966355813491715728702490170635510049560213',\n", + "{'Up': '111612048087962925846397645788043113901565915142888535086349545305985085081594',\n", + " 'Down': '12874695654084258339187997850118078073676562138805309130550135898707856234061',\n", " 'isActive': False,\n", " 'MinTickSize': 0.01,\n", " 'isNegRisk': False,\n", - " 'ConditionId': '0x071d8568d3d736502bd450e150ef93481992d1d26df0c094cc119246d8931a23'}" + " 'ConditionId': '0xa71ed436d160cc45bf182b9004e0c10b16322ea1b41375182b747c2629223ecc'}" ] }, - "execution_count": 67, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -100,7 +201,7 @@ }, { "cell_type": "code", - "execution_count": 61, + "execution_count": 6, "id": "5d356d3b", "metadata": {}, "outputs": [ @@ -109,7 +210,7 @@ "output_type": "stream", "text": [ "creating client...\n", - "You've made 41 trades\n", + "You've made 297 trades\n", "client created successfully!\n" ] } @@ -120,7 +221,128 @@ }, { "cell_type": "code", - "execution_count": 62, + "execution_count": null, + "id": "22eb81de", + "metadata": {}, + "outputs": [], + "source": [ + "# Filtered by market\n", + "order_status = client.get_orders(\n", + " OpenOrderParams(id=\"0x9249ce4a8bc67de355b487b00eaa6ce25c1b451867f9350672b25eaa1de08494\")\n", + ")[0]['status']" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "fb3b8151", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'CANCELED'" + ] + }, + "execution_count": 14, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "order" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "de5ccc3a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.0" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "collateral = client.get_balance_allowance(\n", + " BalanceAllowanceParams(\n", + " asset_type='CONDITIONAL',\n", + " token_id='29663568421665501825278796284809925893978140634751674792176179244450686939029',\n", + " )\n", + ")\n", + "a = collateral['balance']\n", + "a = int(a) / 1_000_000\n", + "a" + ] + }, + { + "cell_type": "code", + "execution_count": 129, + "id": "40323f8d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'balance': '7682900',\n", + " 'allowances': {'0x4bFb41d5B3570DeFd03C39a9A4D8dE6Bd8B8982E': '115792089237316195398462578067141184799968521174335529155754622898352762650625',\n", + " '0xC5d563A36AE78145C45a50134d48A1215220f80a': '115792089237316195398462578067141184799968521174335529155754622898352762650625',\n", + " '0xd91E80cF2E7be2e162c6513ceD06f1dD0dA35296': '115792089237316195398462578067141184799968521174335529155754622898352762650625'}}" + ] + }, + "execution_count": 129, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "collateral" + ] + }, + { + "cell_type": "code", + "execution_count": 119, + "id": "45fe4f53", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'balance': '24494337',\n", + " 'allowances': {'0x4bFb41d5B3570DeFd03C39a9A4D8dE6Bd8B8982E': '115792089237316195423570985008687907853269984665640564039457584007912981179951',\n", + " '0xC5d563A36AE78145C45a50134d48A1215220f80a': '115792089237316195423570985008687907853269984665640564039457584007913129639935',\n", + " '0xd91E80cF2E7be2e162c6513ceD06f1dD0dA35296': '115792089237316195423570985008687907853269984665640564039457584007913129639935'}}" + ] + }, + "execution_count": 119, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "collateral" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6cf19a23", + "metadata": {}, + "outputs": [], + "source": [ + "client.cancel(order_id=)" + ] + }, + { + "cell_type": "code", + "execution_count": 106, "id": "bebb53eb", "metadata": {}, "outputs": [ @@ -128,7 +350,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "{'price': '0.84', 'side': 'BUY'}\n" + "{'price': '0.3', 'side': 'BUY'}\n" ] } ], @@ -139,15 +361,38 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "bae5e6a9", + "execution_count": 109, + "id": "1c5a6c9a", "metadata": {}, "outputs": [], - "source": [] + "source": [ + "d = client.cancel_all()" + ] }, { "cell_type": "code", - "execution_count": 63, + "execution_count": null, + "id": "7bc2b7e7", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "False" + ] + }, + "execution_count": 111, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "bool(d['not_canceled'])" + ] + }, + { + "cell_type": "code", + "execution_count": null, "id": "52c0c38a", "metadata": {}, "outputs": [ @@ -155,17 +400,19 @@ "name": "stdout", "output_type": "stream", "text": [ - "[{'errorMsg': '', 'orderID': '0x4e8e8b193d91c2d4e7b455d3d654a26697ee3289399a9140b2b1888dda7b9a16', 'takingAmount': '10', 'makingAmount': '1.7', 'status': 'matched', 'transactionsHashes': ['0x4f66978cc001a819d9ba266f708148f0512414627a59ffd9a48d8e4b92e5a716'], 'success': True}, {'errorMsg': '', 'orderID': '0x6c5aad6b231c8a6d7a91c1260fe31955e3715e6557aaa773029bd9ec42f26917', 'takingAmount': '', 'makingAmount': '', 'status': 'live', 'success': True}]\n" + "[{'errorMsg': '', 'orderID': '0x0c701329ddd7881505648c5ebdd03cd6f86ae3a70b0cf150bcb53946e043e6bf', 'takingAmount': '', 'makingAmount': '', 'status': 'live', 'success': True}]\n" ] } ], "source": [ + "%%time\n", + "### POST \n", "response = client.post_orders([\n", " PostOrdersArgs(\n", " order=client.create_order(\n", " order_args=OrderArgs(\n", " token_id=market_details['Up'],\n", - " price=0.90,\n", + " price=0.1,\n", " size=10,\n", " side=BUY,\n", " ),\n", @@ -177,11 +424,67 @@ " orderType=OrderType.GTC,\n", " postOnly=False,\n", " ),\n", + " # PostOrdersArgs(\n", + " # order=client.create_order(\n", + " # order_args=OrderArgs(\n", + " # token_id=market_details['Down'],\n", + " # price=0.10,\n", + " # size=10,\n", + " # side=BUY,\n", + " # ),\n", + " # options=PartialCreateOrderOptions(\n", + " # tick_size=str(market_details['MinTickSize']),\n", + " # neg_risk=market_details['isNegRisk']\n", + " # ),\n", + " # ),\n", + " # orderType=OrderType.GTC,\n", + " # postOnly=True,\n", + " # ),\n", + "])\n", + "\n", + "print(response)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "52a7229b", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'errorMsg': '',\n", + " 'orderID': '0x483e094216d6453f0398875094a29a29a913b103fb1180164e092e3ee65209fe',\n", + " 'takingAmount': '',\n", + " 'makingAmount': '',\n", + " 'status': 'live',\n", + " 'success': True}]" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "response" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bbed2536", + "metadata": {}, + "outputs": [], + "source": [ + "### POST \n", + "response = client.post_orders([\n", " PostOrdersArgs(\n", " order=client.create_order(\n", " order_args=OrderArgs(\n", - " token_id=market_details['Down'],\n", - " price=0.10,\n", + " token_id=market_details['Up'],\n", + " price=0.1,\n", " size=10,\n", " side=BUY,\n", " ),\n", @@ -191,17 +494,59 @@ " ),\n", " ),\n", " orderType=OrderType.GTC,\n", - " postOnly=True,\n", + " postOnly=False,\n", " ),\n", + " # PostOrdersArgs(\n", + " # order=client.create_order(\n", + " # order_args=OrderArgs(\n", + " # token_id=market_details['Down'],\n", + " # price=0.10,\n", + " # size=10,\n", + " # side=BUY,\n", + " # ),\n", + " # options=PartialCreateOrderOptions(\n", + " # tick_size=str(market_details['MinTickSize']),\n", + " # neg_risk=market_details['isNegRisk']\n", + " # ),\n", + " # ),\n", + " # orderType=OrderType.GTC,\n", + " # postOnly=True,\n", + " # ),\n", "])\n", "\n", "print(response)" ] }, + { + "cell_type": "code", + "execution_count": 138, + "id": "9ec69680", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'errorMsg': '',\n", + " 'orderID': '0x575c43f35b1f5e3e01779df5293d987dc81a347a4e09423a856ecb45555e30cb',\n", + " 'takingAmount': '',\n", + " 'makingAmount': '',\n", + " 'status': 'live',\n", + " 'success': True}]" + ] + }, + "execution_count": 138, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "response" + ] + }, { "cell_type": "code", "execution_count": null, - "id": "52a7229b", + "id": "b7234035", "metadata": {}, "outputs": [], "source": [] @@ -209,7 +554,6 @@ { "cell_type": "code", "execution_count": null, - "id": "fb5f066a", "metadata": {}, "outputs": [], "source": [] @@ -234,167 +578,84 @@ }, { "cell_type": "code", - "execution_count": 69, - "id": "f608378f", + "execution_count": null, + "id": "2fe32a82", "metadata": {}, "outputs": [ { "data": { - "application/vnd.microsoft.datawrangler.viewer.v0+json": { - "columns": [ - { - "name": "index", - "rawType": "int64", - "type": "integer" - }, - { - "name": "asset_id", - "rawType": "object", - "type": "string" - }, - { - "name": "price", - "rawType": "object", - "type": "string" - }, - { - "name": "size", - "rawType": "object", - "type": "string" - }, - { - "name": "side", - "rawType": "object", - "type": "string" - }, - { - "name": "hash", - "rawType": "object", - "type": "string" - }, - { - "name": "best_bid", - "rawType": "object", - "type": "string" - }, - { - "name": "best_ask", - "rawType": "object", - "type": "string" - } - ], - "ref": "310c4763-20db-4051-931a-ef52e1b6513b", - "rows": [ - [ - "0", - "97987758532314346863331680319607711694838937984814950023901315671390566048932", - "0.37", - "429.41", - "BUY", - "999d5b73d83a840c0df7dc2d817ae55a242845d5", - "0.37", - "0.38" - ], - [ - "1", - "92959981857766705879127008770062050214089835506649207585188324269480756219695", - "0.63", - "429.41", - "SELL", - "ead5af5a55f8b125b1e50f1c80a783c3c2d33187", - "0.62", - "0.63" - ] - ], - "shape": { - "columns": 7, - "rows": 2 - } - }, - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
asset_idpricesizesidehashbest_bidbest_ask
09798775853231434686333168031960771169483893798...0.37429.41BUY999d5b73d83a840c0df7dc2d817ae55a242845d50.370.38
19295998185776670587912700877006205021408983550...0.63429.41SELLead5af5a55f8b125b1e50f1c80a783c3c2d331870.620.63
\n", - "
" - ], "text/plain": [ - " asset_id price size side \\\n", - "0 9798775853231434686333168031960771169483893798... 0.37 429.41 BUY \n", - "1 9295998185776670587912700877006205021408983550... 0.63 429.41 SELL \n", - "\n", - " hash best_bid best_ask \n", - "0 999d5b73d83a840c0df7dc2d817ae55a242845d5 0.37 0.38 \n", - "1 ead5af5a55f8b125b1e50f1c80a783c3c2d33187 0.62 0.63 " + "Timestamp('2026-03-30 23:32:03')" ] }, - "execution_count": 69, + "execution_count": 80, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "pd.DataFrame(d)" + "hist_trades_lookback_ts = round(datetime.now().timestamp() - 10)*1000\n", + "pd.to_datetime(hist_trades_lookback_ts*1000, unit='ms')" + ] + }, + { + "cell_type": "code", + "execution_count": 81, + "id": "2a3b19f2", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Timestamp('2026-03-30 04:22:23.585000')" + ] + }, + "execution_count": 81, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pd.to_datetime(1774844543585, unit='ms')" + ] + }, + { + "cell_type": "code", + "execution_count": 79, + "id": "b5139910", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1774913409000" + ] + }, + "execution_count": 79, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "hist_trades_lookback_ts*1000" ] }, { "cell_type": "code", "execution_count": null, - "id": "2fe32a82", + "id": "d6d3c1bb", "metadata": {}, "outputs": [], - "source": [] + "source": [ + "import numpy as np\n", + "n = np.empty((0, 3))\n", + "np.append(n, [1,2,3])" + ] }, { "cell_type": "code", "execution_count": null, - "id": "7603be6c", + "id": "1e5cba44", "metadata": {}, "outputs": [], "source": [] @@ -626,19 +887,524 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 140, "id": "008cb5c9", "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'Up': '13157292356296687506747919717798752029699544499054519087985411865141996614822',\n", + " 'Down': '70507961363566124468475538524172170043725846352735387705515911177933084557518',\n", + " 'isActive': False,\n", + " 'MinTickSize': 0.01,\n", + " 'isNegRisk': False,\n", + " 'ConditionId': '0xd1773b412dacad884c202a7b14f0197918b1e22028ce2b5737fbd659bbe150f0'}" + ] + }, + "execution_count": 140, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "market_details" + ] + }, + { + "cell_type": "code", + "execution_count": 130, + "id": "f6c647b7", + "metadata": {}, + "outputs": [], + "source": [ + "d = {\"id\":\"0x73c6d7c0cba705a06d840c185ef188a195b3e36472dceab61cee23202ec7a9a0\",\"owner\":\"00e5d36c-6c46-77e1-a436-0f0a4bfbfdd8\",\"market\":\"0xf856ff89eb9ed2ab393a1bbc496b5db539b4d7f9cec91202b13c97379e9e58d6\",\"asset_id\":\"39378292107289994981363071337831788209917841322683175778910172170676449806535\",\"side\":\"BUY\",\"order_owner\":\"00e5d36c-6c46-77e1-a436-0f0a4bfbfdd8\",\"original_size\":\"10\",\"size_matched\":\"0\",\"price\":\"0.2\",\"associate_trades\":[],\"outcome\":\"Up\",\"type\":\"PLACEMENT\",\"created_at\":\"1774818626\",\"expiration\":\"0\",\"order_type\":\"GTC\",\"status\":\"LIVE\",\"maker_address\":\"0xb2967A7e578E700E27611238B7F762BdADC72CcB\",\"timestamp\":\"1774818626080\",\"event_type\":\"order\"}\n", + "e = {\"id\":\"0x73c6d7c0cba705a06d840c185ef188a195b3e36472dceab61cee23202ec7a9a0\",\"owner\":\"00e5d36c-6c46-77e1-a436-0f0a4bfbfdd8\",\"market\":\"0xf856ff89eb9ed2ab393a1bbc496b5db539b4d7f9cec91202b13c97379e9e58d6\",\"asset_id\":\"39378292107289994981363071337831788209917841322683175778910172170676449806535\",\"side\":\"BUY\",\"order_owner\":\"00e5d36c-6c46-77e1-a436-0f0a4bfbfdd8\",\"original_size\":\"10\",\"size_matched\":\"0\",\"price\":\"0.2\",\"associate_trades\":[],\"outcome\":\"Up\",\"type\":\"PLACEMENT\",\"created_at\":\"1774818626\",\"expiration\":\"0\",\"order_type\":\"GTC\",\"status\":\"LIVE\",\"maker_address\":\"0xb2967A7e578E700E27611238B7F762BdADC72CcB\",\"timestamp\":\"9774818626080\",\"event_type\":\"order\"}" + ] + }, + { + "cell_type": "code", + "execution_count": 131, + "id": "00c55bf5", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'id': '0x73c6d7c0cba705a06d840c185ef188a195b3e36472dceab61cee23202ec7a9a0',\n", + " 'owner': '00e5d36c-6c46-77e1-a436-0f0a4bfbfdd8',\n", + " 'market': '0xf856ff89eb9ed2ab393a1bbc496b5db539b4d7f9cec91202b13c97379e9e58d6',\n", + " 'asset_id': '39378292107289994981363071337831788209917841322683175778910172170676449806535',\n", + " 'side': 'BUY',\n", + " 'order_owner': '00e5d36c-6c46-77e1-a436-0f0a4bfbfdd8',\n", + " 'original_size': '10',\n", + " 'size_matched': '0',\n", + " 'price': '0.2',\n", + " 'associate_trades': [],\n", + " 'outcome': 'Up',\n", + " 'type': 'PLACEMENT',\n", + " 'created_at': '1774818626',\n", + " 'expiration': '0',\n", + " 'order_type': 'GTC',\n", + " 'status': 'LIVE',\n", + " 'maker_address': '0xb2967A7e578E700E27611238B7F762BdADC72CcB',\n", + " 'timestamp': '1774818626080',\n", + " 'event_type': 'order'}" + ] + }, + "execution_count": 131, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "d" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "cf38819c", + "metadata": {}, + "outputs": [], + "source": [ + "f = {\"id\":\"0x73c6d7c0cba705a06d840c185ef188a195b3e36472dceab61cee23202ec7a9a0\",\"owner\":\"00e5d36c-6c46-77e1-a436-0f0a4bfbfdd8\",\"market\":\"0xf856ff89eb9ed2ab393a1bbc496b5db539b4d7f9cec91202b13c97379e9e58d6\",\"asset_id\":\"39378292107289994981363071337831788209917841322683175778910172170676449806535\",\"side\":\"BUY\",\"order_owner\":\"00e5d36c-6c46-77e1-a436-0f0a4bfbfdd8\",\"original_size\":\"10\",\"size_matched\":\"0\",\"price\":\"0.2\",\"associate_trades\":[],\"outcome\":\"Up\",\"type\":\"CANCELLATION\",\"created_at\":\"1774818626\",\"expiration\":\"0\",\"order_type\":\"GTC\",\"status\":\"CANCELED\",\"maker_address\":\"0xb2967A7e578E700E27611238B7F762BdADC72CcB\",\"timestamp\":\"1774818630291\",\"event_type\":\"order\"}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ea67fb64", + "metadata": {}, + "outputs": [], + "source": [ + "a = json.dumps(f['associate_trades']) if len(f['associate_trades']) > 0 else None" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "59521b6e", + "metadata": {}, "outputs": [], "source": [] }, { "cell_type": "code", "execution_count": null, - "id": "f6c647b7", + "id": "6b4ee964", "metadata": {}, "outputs": [], - "source": [] + "source": [ + "def live_orders_only(live_orders, new_msg):\n", + " return [d for d in live_orders if d.get('status')=='live']" + ] + }, + { + "cell_type": "code", + "execution_count": 117, + "id": "715d8606", + "metadata": {}, + "outputs": [], + "source": [ + "def upsert_list_of_dicts_by_id(list_of_dicts, new_dict):\n", + " for index, item in enumerate(list_of_dicts):\n", + " if item.get('id') == new_dict.get('id'):\n", + " list_of_dicts[index] = new_dict\n", + " return list_of_dicts\n", + " \n", + " list_of_dicts.append(new_dict)\n", + " return list_of_dicts" + ] + }, + { + "cell_type": "code", + "execution_count": 113, + "id": "4c00fdb7", + "metadata": {}, + "outputs": [], + "source": [ + "ORDERS_STATUS = []\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d8a2bbbf", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'id': '0x73c6d7c0cba705a06d840c185ef188a195b3e36472dceab61cee23202ec7a9a0',\n", + " 'owner': '00e5d36c-6c46-77e1-a436-0f0a4bfbfdd8',\n", + " 'market': '0xf856ff89eb9ed2ab393a1bbc496b5db539b4d7f9cec91202b13c97379e9e58d6',\n", + " 'asset_id': '39378292107289994981363071337831788209917841322683175778910172170676449806535',\n", + " 'side': 'BUY',\n", + " 'order_owner': '00e5d36c-6c46-77e1-a436-0f0a4bfbfdd8',\n", + " 'original_size': '10',\n", + " 'size_matched': '0',\n", + " 'price': '0.2',\n", + " 'associate_trades': [],\n", + " 'outcome': 'Up',\n", + " 'type': 'PLACEMENT',\n", + " 'created_at': '1774818626',\n", + " 'expiration': '0',\n", + " 'order_type': 'GTC',\n", + " 'status': 'LIVE',\n", + " 'maker_address': '0xb2967A7e578E700E27611238B7F762BdADC72CcB',\n", + " 'timestamp': '1774818626080',\n", + " 'event_type': 'order'}]" + ] + }, + "execution_count": 114, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ORDERS_STATUS = upsert_list_of_dicts_by_id(ORDERS_STATUS, d)\n", + "ORDERS_STATUS" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6e8be4ab", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'id': '0x73c6d7c0cba705a06d840c185ef188a195b3e36472dceab61cee23202ec7a9a0',\n", + " 'owner': '00e5d36c-6c46-77e1-a436-0f0a4bfbfdd8',\n", + " 'market': '0xf856ff89eb9ed2ab393a1bbc496b5db539b4d7f9cec91202b13c97379e9e58d6',\n", + " 'asset_id': '39378292107289994981363071337831788209917841322683175778910172170676449806535',\n", + " 'side': 'BUY',\n", + " 'order_owner': '00e5d36c-6c46-77e1-a436-0f0a4bfbfdd8',\n", + " 'original_size': '10',\n", + " 'size_matched': '0',\n", + " 'price': '0.2',\n", + " 'associate_trades': [],\n", + " 'outcome': 'Up',\n", + " 'type': 'PLACEMENT',\n", + " 'created_at': '1774818626',\n", + " 'expiration': '0',\n", + " 'order_type': 'GTC',\n", + " 'status': 'LIVE',\n", + " 'maker_address': '0xb2967A7e578E700E27611238B7F762BdADC72CcB',\n", + " 'timestamp': '9774818626080',\n", + " 'event_type': 'order'}]" + ] + }, + "execution_count": 115, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ORDERS_STATUS = upsert_list_of_dicts_by_id(ORDERS_STATUS, e)\n", + "ORDERS_STATUS" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f3b0a0c2", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'id': '0x73c6d7c0cba705a06d840c185ef188a195b3e36472dceab61cee23202ec7a9a0',\n", + " 'owner': '00e5d36c-6c46-77e1-a436-0f0a4bfbfdd8',\n", + " 'market': '0xf856ff89eb9ed2ab393a1bbc496b5db539b4d7f9cec91202b13c97379e9e58d6',\n", + " 'asset_id': '39378292107289994981363071337831788209917841322683175778910172170676449806535',\n", + " 'side': 'BUY',\n", + " 'order_owner': '00e5d36c-6c46-77e1-a436-0f0a4bfbfdd8',\n", + " 'original_size': '10',\n", + " 'size_matched': '0',\n", + " 'price': '0.2',\n", + " 'associate_trades': [],\n", + " 'outcome': 'Up',\n", + " 'type': 'CANCELLATION',\n", + " 'created_at': '1774818626',\n", + " 'expiration': '0',\n", + " 'order_type': 'GTC',\n", + " 'status': 'CANCELED',\n", + " 'maker_address': '0xb2967A7e578E700E27611238B7F762BdADC72CcB',\n", + " 'timestamp': '1774818630291',\n", + " 'event_type': 'order'}]" + ] + }, + "execution_count": 116, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ORDERS_STATUS = upsert_list_of_dicts_by_id(ORDERS_STATUS, f)\n", + "ORDERS_STATUS" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6be0d1e8", + "metadata": {}, + "outputs": [], + "source": [ + "{\n", + " \"event_type\": \"order\",\n", + " \"id\": \"0xff354cd7ca7539dfa9c28d90943ab5779a4eac34b9b37a757d7b32bdfb11790b\",\n", + " \"owner\": \"9180014b-33c8-9240-a14b-bdca11c0a465\",\n", + " \"market\": \"0xbd31dc8a20211944f6b70f31557f1001557b59905b7738480ca09bd4532f84af\",\n", + " \"asset_id\": \"52114319501245915516055106046884209969926127482827954674443846427813813222426\",\n", + " \"side\": \"SELL\",\n", + " \"order_owner\": \"9180014b-33c8-9240-a14b-bdca11c0a465\",\n", + " \"original_size\": \"10\",\n", + " \"size_matched\": \"0\",\n", + " \"price\": \"0.57\",\n", + " \"associate_trades\": null,\n", + " \"outcome\": \"YES\",\n", + " \"type\": \"PLACEMENT\",\n", + " \"created_at\": \"1672290687\",\n", + " \"expiration\": \"1234567\",\n", + " \"order_type\": \"GTD\",\n", + " \"status\": \"LIVE\",\n", + " \"maker_address\": \"0x1234...\",\n", + " \"timestamp\": \"1672290687\"\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "761e4f8a", + "metadata": {}, + "outputs": [], + "source": [ + "z = {\n", + " \"event_type\": \"trade\",\n", + " \"type\": \"TRADE\",\n", + " \"id\": \"28c4d2eb-bbea-40e7-a9f0-b2fdb56b2c2e\",\n", + " \"taker_order_id\": \"0x06bc63e346ed4ceddce9efd6b3af37c8f8f440c92fe7da6b2d0f9e4ccbc50c42\",\n", + " \"market\": \"0xbd31dc8a20211944f6b70f31557f1001557b59905b7738480ca09bd4532f84af\",\n", + " \"asset_id\": \"52114319501245915516055106046884209969926127482827954674443846427813813222426\",\n", + " \"side\": \"BUY\",\n", + " \"size\": \"10\",\n", + " \"price\": \"0.57\",\n", + " \"fee_rate_bps\": \"0\",\n", + " \"status\": \"MATCHED\",\n", + " \"matchtime\": \"1672290701\",\n", + " \"last_update\": \"1672290701\",\n", + " \"outcome\": \"YES\",\n", + " \"owner\": \"9180014b-33c8-9240-a14b-bdca11c0a465\",\n", + " \"trade_owner\": \"9180014b-33c8-9240-a14b-bdca11c0a465\",\n", + " \"maker_address\": \"0x1234...\",\n", + " \"transaction_hash\": \"\",\n", + " \"bucket_index\": 0,\n", + " \"maker_orders\": [\n", + " {\n", + " \"order_id\": \"0xff354cd7ca7539dfa9c28d90943ab5779a4eac34b9b37a757d7b32bdfb11790b\",\n", + " \"owner\": \"9180014b-33c8-9240-a14b-bdca11c0a465\",\n", + " \"maker_address\": \"0x5678...\",\n", + " \"matched_amount\": \"10\",\n", + " \"price\": \"0.57\",\n", + " \"fee_rate_bps\": \"0\",\n", + " \"asset_id\": \"52114319501245915516055106046884209969926127482827954674443846427813813222426\",\n", + " \"outcome\": \"YES\",\n", + " \"side\": \"SELL\"\n", + " }\n", + " ],\n", + " \"trader_side\": \"TAKER\",\n", + " \"timestamp\": \"1672290701\"\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "3c5a0922", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'event_type': 'trade',\n", + " 'type': 'TRADE',\n", + " 'id': '28c4d2eb-bbea-40e7-a9f0-b2fdb56b2c2e',\n", + " 'taker_order_id': '0x06bc63e346ed4ceddce9efd6b3af37c8f8f440c92fe7da6b2d0f9e4ccbc50c42',\n", + " 'market': '0xbd31dc8a20211944f6b70f31557f1001557b59905b7738480ca09bd4532f84af',\n", + " 'asset_id': '52114319501245915516055106046884209969926127482827954674443846427813813222426',\n", + " 'side': 'BUY',\n", + " 'size': '10',\n", + " 'price': '0.57',\n", + " 'fee_rate_bps': '0',\n", + " 'status': 'MATCHED',\n", + " 'matchtime': '1672290701',\n", + " 'last_update': '1672290701',\n", + " 'outcome': 'YES',\n", + " 'owner': '9180014b-33c8-9240-a14b-bdca11c0a465',\n", + " 'trade_owner': '9180014b-33c8-9240-a14b-bdca11c0a465',\n", + " 'maker_address': '0x1234...',\n", + " 'transaction_hash': '',\n", + " 'bucket_index': 0,\n", + " 'maker_orders': [{'order_id': '0xff354cd7ca7539dfa9c28d90943ab5779a4eac34b9b37a757d7b32bdfb11790b',\n", + " 'owner': '9180014b-33c8-9240-a14b-bdca11c0a465',\n", + " 'maker_address': '0x5678...',\n", + " 'matched_amount': '10',\n", + " 'price': '0.57',\n", + " 'fee_rate_bps': '0',\n", + " 'asset_id': '52114319501245915516055106046884209969926127482827954674443846427813813222426',\n", + " 'outcome': 'YES',\n", + " 'side': 'SELL'}],\n", + " 'trader_side': 'TAKER',\n", + " 'timestamp': '1672290701'}" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "z" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'[{\"order_id\": \"0xff354cd7ca7539dfa9c28d90943ab5779a4eac34b9b37a757d7b32bdfb11790b\", \"owner\": \"9180014b-33c8-9240-a14b-bdca11c0a465\", \"maker_address\": \"0x5678...\", \"matched_amount\": \"10\", \"price\": \"0.57\", \"fee_rate_bps\": \"0\", \"asset_id\": \"52114319501245915516055106046884209969926127482827954674443846427813813222426\", \"outcome\": \"YES\", \"side\": \"SELL\"}]'" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import json\n", + "json.dumps(z['maker_orders'])" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "z['maker_orders'] = json.dumps(z['maker_orders'])" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'event_type': 'trade',\n", + " 'type': 'TRADE',\n", + " 'id': '28c4d2eb-bbea-40e7-a9f0-b2fdb56b2c2e',\n", + " 'taker_order_id': '0x06bc63e346ed4ceddce9efd6b3af37c8f8f440c92fe7da6b2d0f9e4ccbc50c42',\n", + " 'market': '0xbd31dc8a20211944f6b70f31557f1001557b59905b7738480ca09bd4532f84af',\n", + " 'asset_id': '52114319501245915516055106046884209969926127482827954674443846427813813222426',\n", + " 'side': 'BUY',\n", + " 'size': '10',\n", + " 'price': '0.57',\n", + " 'fee_rate_bps': '0',\n", + " 'status': 'MATCHED',\n", + " 'matchtime': '1672290701',\n", + " 'last_update': '1672290701',\n", + " 'outcome': 'YES',\n", + " 'owner': '9180014b-33c8-9240-a14b-bdca11c0a465',\n", + " 'trade_owner': '9180014b-33c8-9240-a14b-bdca11c0a465',\n", + " 'maker_address': '0x1234...',\n", + " 'transaction_hash': '',\n", + " 'bucket_index': 0,\n", + " 'maker_orders': '[{\"order_id\": \"0xff354cd7ca7539dfa9c28d90943ab5779a4eac34b9b37a757d7b32bdfb11790b\", \"owner\": \"9180014b-33c8-9240-a14b-bdca11c0a465\", \"maker_address\": \"0x5678...\", \"matched_amount\": \"10\", \"price\": \"0.57\", \"fee_rate_bps\": \"0\", \"asset_id\": \"52114319501245915516055106046884209969926127482827954674443846427813813222426\", \"outcome\": \"YES\", \"side\": \"SELL\"}]',\n", + " 'trader_side': 'TAKER',\n", + " 'timestamp': '1672290701'}" + ] + }, + "execution_count": 19, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "z" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "id": "0dbb8fa9", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "1774899574594" + ] + }, + "execution_count": 29, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from datetime import datetime\n", + "import pandas as pd\n", + "round(datetime.now().timestamp()*1000)" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Timestamp('2026-03-30 19:39:16.685000')" + ] + }, + "execution_count": 30, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pd.to_datetime(1774899556685, unit='ms')" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Timestamp('2026-03-30 19:39:06.685000')" + ] + }, + "execution_count": 31, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "pd.to_datetime(1774899556685-10*1000, unit='ms')" + ] }, { "cell_type": "code", @@ -648,6 +1414,200 @@ "outputs": [], "source": [] }, + { + "cell_type": "code", + "execution_count": null, + "id": "6765c7e6", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 50, + "id": "d367f55b", + "metadata": {}, + "outputs": [], + "source": [ + "d = {\"type\":\"TRADE\",\"id\":\"08045b74-925f-4982-bc5b-577104db6530\",\"taker_order_id\":\"0xf6c77c6ef4c229c3edf7ba0a2183c4cd1193211787e2fa7fb0e7bf132189b303\",\"market\":\"0x7cdfcb753bebd87214e719497aa1d1b217582dc22746c8543b74e1e53539a9f5\",\"asset_id\":\"56806427206990113501155420994096888900269904953230745602350514810276549822457\",\"side\":\"BUY\",\"size\":\"7.042251\",\"fee_rate_bps\":\"1000\",\"price\":\"0.71\",\"status\":\"MATCHED\",\"match_time\":\"1774900601\",\"last_update\":\"1774900601\",\"outcome\":\"Up\",\"owner\":\"00e5d36c-6c46-77e1-a436-0f0a4bfbfdd8\",\"trade_owner\":\"00e5d36c-6c46-77e1-a436-0f0a4bfbfdd8\",\"maker_address\":\"0xb2967A7e578E700E27611238B7F762BdADC72CcB\",\"transaction_hash\":\"0x454cbaad0da15d095be778773c362af90de43beb6660c78ff155b3877d283882\",\"bucket_index\":0,\"maker_orders\":[{\"order_id\":\"0x4efdbcd86c81df9d6a02d7e35ff92411af76aa3ca6f445ba9d6e3773e812f706\",\"owner\":\"579ba9d5-cc34-1b66-3b70-d5bb5d1ccc18\",\"maker_address\":\"0x5Bde889dC26B097b5eAa2F1F027e01712EBCcbB7\",\"matched_amount\":\"7.042251\",\"price\":\"0.2900000298200107\",\"fee_rate_bps\":\"1000\",\"asset_id\":\"95594709608392853199445131908660268785353291802775571968225275873296398106331\",\"outcome\":\"Down\",\"outcome_index\":0,\"side\":\"BUY\"}],\"trader_side\":\"TAKER\",\"timestamp\":\"1774900601865\",\"event_type\":\"trade\"}" + ] + }, + { + "cell_type": "code", + "execution_count": 51, + "id": "3e8a71a7", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'type': 'TRADE',\n", + " 'id': '08045b74-925f-4982-bc5b-577104db6530',\n", + " 'taker_order_id': '0xf6c77c6ef4c229c3edf7ba0a2183c4cd1193211787e2fa7fb0e7bf132189b303',\n", + " 'market': '0x7cdfcb753bebd87214e719497aa1d1b217582dc22746c8543b74e1e53539a9f5',\n", + " 'asset_id': '56806427206990113501155420994096888900269904953230745602350514810276549822457',\n", + " 'side': 'BUY',\n", + " 'size': '7.042251',\n", + " 'fee_rate_bps': '1000',\n", + " 'price': '0.71',\n", + " 'status': 'MATCHED',\n", + " 'match_time': '1774900601',\n", + " 'last_update': '1774900601',\n", + " 'outcome': 'Up',\n", + " 'owner': '00e5d36c-6c46-77e1-a436-0f0a4bfbfdd8',\n", + " 'trade_owner': '00e5d36c-6c46-77e1-a436-0f0a4bfbfdd8',\n", + " 'maker_address': '0xb2967A7e578E700E27611238B7F762BdADC72CcB',\n", + " 'transaction_hash': '0x454cbaad0da15d095be778773c362af90de43beb6660c78ff155b3877d283882',\n", + " 'bucket_index': 0,\n", + " 'maker_orders': [{'order_id': '0x4efdbcd86c81df9d6a02d7e35ff92411af76aa3ca6f445ba9d6e3773e812f706',\n", + " 'owner': '579ba9d5-cc34-1b66-3b70-d5bb5d1ccc18',\n", + " 'maker_address': '0x5Bde889dC26B097b5eAa2F1F027e01712EBCcbB7',\n", + " 'matched_amount': '7.042251',\n", + " 'price': '0.2900000298200107',\n", + " 'fee_rate_bps': '1000',\n", + " 'asset_id': '95594709608392853199445131908660268785353291802775571968225275873296398106331',\n", + " 'outcome': 'Down',\n", + " 'outcome_index': 0,\n", + " 'side': 'BUY'}],\n", + " 'trader_side': 'TAKER',\n", + " 'timestamp': '1774900601865',\n", + " 'event_type': 'trade'}" + ] + }, + "execution_count": 51, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "d" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "009fab36", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 35, + "id": "4d524867", + "metadata": {}, + "outputs": [], + "source": [ + "o = {'orderID': '0xaebd4053bd167eb7a7fc48ae29036829582787a73d8cc7b8b1afa7294c474972',\n", + " 'owner': '00e5d36c-6c46-77e1-a436-0f0a4bfbfdd8',\n", + " 'market': '0xf856ff89eb9ed2ab393a1bbc496b5db539b4d7f9cec91202b13c97379e9e58d6',\n", + " 'asset_id': '39378292107289994981363071337831788209917841322683175778910172170676449806535',\n", + " 'side': 'BUY',\n", + " 'order_owner': '00e5d36c-6c46-77e1-a436-0f0a4bfbfdd8',\n", + " 'original_size': '10',\n", + " 'size_matched': '0',\n", + " 'price': '0.2',\n", + " 'associate_trades': [],\n", + " 'outcome': 'Up',\n", + " 'type': 'CANCELLATION',\n", + " 'created_at': '1774818626',\n", + " 'expiration': '0',\n", + " 'order_type': 'GTC',\n", + " 'status': 'CANCELED',\n", + " 'maker_address': '0xb2967A7e578E700E27611238B7F762BdADC72CcB',\n", + " 'timestamp': '1774818630291',\n", + " 'event_type': 'order'}" + ] + }, + { + "cell_type": "code", + "execution_count": 47, + "id": "43a68ee8", + "metadata": {}, + "outputs": [], + "source": [ + "ut = {'trade_side':'MAKER', 'taker_order_id': None, 'maker_orders': [{\"side\": \"BUY\", \"owner\": \"f44394d9-7782-e5e5-af99-dd515edd7fd9\", \"price\": \"0.44\", \"outcome\": \"Up\", \"asset_id\": \"55598909213399021321632159985071802901933471635502091907595118948537230110477\", \"order_id\": \"0x7eed6fa75cd49680b738e1563fc9e5dc0f7095c8a7dd3ee1a496c01f4ad2f610\", \"fee_rate_bps\": \"1000\", \"maker_address\": \"0xE29042f5D913DCC4015aaB3455C13C58514CA33F\", \"outcome_index\": 0, \"matched_amount\": \"15.25\"}, {\"side\": \"BUY\", \"owner\": \"520cb8f0-a71b-1c49-8dd6-751e682ee7f8\", \"price\": \"0.44\", \"outcome\": \"Up\", \"asset_id\": \"55598909213399021321632159985071802901933471635502091907595118948537230110477\", \"order_id\": \"0xf7c0a9e322fbf232dce6290f36bbe5e484ccec7fda599ddeb98867f241df6e37\", \"fee_rate_bps\": \"1000\", \"maker_address\": \"0x74a6364297292774c7f9a16B925207E77eEE262D\", \"outcome_index\": 0, \"matched_amount\": \"5\"}, {\"side\": \"BUY\", \"owner\": \"ae0f07ea-bcbe-3d54-6841-f220e89794ae\", \"price\": \"0.44\", \"outcome\": \"Up\", \"asset_id\": \"55598909213399021321632159985071802901933471635502091907595118948537230110477\", \"order_id\": \"0xe321494520de4737980a15160bfae94cb48791a2dc978aa843e7a8504e815771\", \"fee_rate_bps\": \"1000\", \"maker_address\": \"0xDba9C86F8d20ac73BcBf4dedaA6ADbd26A0a1303\", \"outcome_index\": 0, \"matched_amount\": \"5\"}, {\"side\": \"BUY\", \"owner\": \"00e5d36c-6c46-77e1-a436-0f0a4bfbfdd8\", \"price\": \"0.44\", \"outcome\": \"Up\", \"asset_id\": \"55598909213399021321632159985071802901933471635502091907595118948537230110477\", \"order_id\": \"0xaebd4053bd167eb7a7fc48ae29036829582787a73d8cc7b8b1afa7294c474972\", \"fee_rate_bps\": \"1000\", \"maker_address\": \"0xb2967A7e578E700E27611238B7F762BdADC72CcB\", \"outcome_index\": 0, \"matched_amount\": \"4.75\"}]}" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "id": "ae50f082", + "metadata": {}, + "outputs": [], + "source": [ + "USER_TRADES = [ut]\n", + "user_trade = next( ( item for item in USER_TRADES if ( o['orderID'] == item['taker_order_id'] ) or ( o[\"orderID\"] == item['maker_orders'][0]['order_id'] ) ), None )" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11c111eb", + "metadata": {}, + "outputs": [], + "source": [ + "for t in USER_TRADES:\n", + " if t['trade_side']=='MAKER':\n", + " pass\n", + " elif t['taker_order_id'] == o[\"orderID\"]:\n", + " pass\n" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "id": "de38feda", + "metadata": {}, + "outputs": [], + "source": [ + "user_trade" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "67b7b730", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d15d92d3", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "code", "execution_count": null, @@ -659,7 +1619,7 @@ ], "metadata": { "kernelspec": { - "display_name": "py313", + "display_name": "py_313", "language": "python", "name": "python3" }, @@ -673,7 +1633,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.13.2" + "version": "3.13.12" } }, "nbformat": 4, diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..70e7b45 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,22 @@ +pandas +rel +websockets +pyarrow +plotly +mysql-connector-python +sqlalchemy +requests +pymysql +scipy +asyncmy +cryptography +TA-Lib +valkey +nicegui +py_clob_client +# google +# google-api-core==2.30.0 +# google-api-python-client==2.190.0 +# googleapis-common-protos==1.72.0 +# grpcio==1.76.0 +# grpcio-tools==1.76.0 \ No newline at end of file diff --git a/test.py b/test.py new file mode 100644 index 0000000..717c72b --- /dev/null +++ b/test.py @@ -0,0 +1,63 @@ +import asyncio +import json +import websockets +import time + +# Credentials +API_KEY = "019d2ad3-3755-744b-ace8-ad0f08c958dd" +API_SECRET = "vXT1UeliaP89z9vcxDtdv47422mftijJkrJYE7CFqvA=" +API_PASSPHRASE = "57e703b801f22333d1a66a48c3a71773d3d3a42825ddcf330c3325856bc99756" +WS_URL = "wss://ws-subscriptions-clob.polymarket.com/ws/user" + +async def heartbeat(websocket): + """Sends a heartbeat every 10 seconds to keep the connection alive.""" + while True: + try: + await asyncio.sleep(10) + await websocket.send(json.dumps({})) + # print("Heartbeat sent") + except Exception: + break + +async def connect_polymarket_user_ws(): + while True: # Outer loop for reconnection + try: + async with websockets.connect(WS_URL) as websocket: + subscribe_message = { + "type": "user", + "auth": { + "apiKey": API_KEY, + "secret": API_SECRET, + "passphrase": API_PASSPHRASE + }, + "markets": [] + } + + await websocket.send(json.dumps(subscribe_message)) + print(f"[{time.strftime('%H:%M:%S')}] Subscription sent...") + + # Start the heartbeat task in the background + heartbeat_task = asyncio.create_task(heartbeat(websocket)) + + async for message in websocket: + data = json.loads(message) + + # Log the specific reason if it's an error message + if data.get("type") == "error": + print(f"Server Error: {data.get('message')}") + break + + if data: # Ignore empty heartbeat responses from server + print(f"Update: {data}") + + heartbeat_task.cancel() + + except Exception as e: + print(f"Connection lost: {e}. Retrying in 5s...") + await asyncio.sleep(5) + +if __name__ == "__main__": + try: + asyncio.run(connect_polymarket_user_ws()) + except KeyboardInterrupt: + print("Stopped by user.") \ No newline at end of file diff --git a/ws.py b/ws.py deleted file mode 100644 index 461e516..0000000 --- a/ws.py +++ /dev/null @@ -1,150 +0,0 @@ -import asyncio -import json -import math -import pandas as pd -import os -from datetime import datetime, timezone -import websockets -import numpy as np -import talib -import requests - -WSS_URL = "wss://ws-subscriptions-clob.polymarket.com/ws/market" -SLUG_END_TIME = 0 - -HIST_TRADES = np.empty((0, 2)) - -def format_timestamp(total_seconds) -> str: - minutes, seconds = divmod(total_seconds, 60) - return f"{minutes} minutes and {seconds} seconds" - -def time_round_down(dt, interval_mins=5) -> int: # returns timestamp in seconds - interval_secs = interval_mins * 60 - seconds = dt.timestamp() - rounded_seconds = math.floor(seconds / interval_secs) * interval_secs - - return rounded_seconds - -def get_mkt_details_by_slug(slug: str) -> dict[str, str, str]: # {'Up' : 123, 'Down': 456, 'isActive': True, 'MinTickSize': 0.01, 'isNegRisk': False} - r = requests.get(f"https://gamma-api.polymarket.com/events/slug/{slug}") - market = r.json()['markets'][0] - token_ids = json.loads(market.get("clobTokenIds", "[]")) - outcomes = json.loads(market.get("outcomes", "[]")) - d = dict(zip(outcomes, token_ids)) - d['isActive'] = market['negRisk'] - d['MinTickSize'] = market['orderPriceMinTickSize'] - d['isNegRisk'] = market['negRisk'] - d['ConditionId'] = market['conditionId'] - d['EndDateTime'] = market['endDate'] - - return d, market - -def gen_slug(): - slug_prefix = 'btc-updown-5m-' - slug_time_id = time_round_down(dt=datetime.now(timezone.utc)) - return slug_prefix + str(slug_time_id) - - -async def polymarket_stream(): - global SLUG_END_TIME - global HIST_TRADES - - slug_full = gen_slug() - market_details, market = get_mkt_details_by_slug(slug_full) - TARGET_ASSET_ID = market_details['Up'] - SLUG_END_TIME = round(datetime.strptime(market_details['EndDateTime'], '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone.utc).timestamp()) - print(f'********* NEW MKT - END DATETIME: {pd.to_datetime(SLUG_END_TIME, unit='s')} *********') - - async with websockets.connect(WSS_URL) as websocket: - print(f"Connected to {WSS_URL}") - - subscribe_msg = { - "assets_ids": [TARGET_ASSET_ID], - "type": "market", - "custom_feature_enabled": True - } - - await websocket.send(json.dumps(subscribe_msg)) - print(f"Subscribed to Asset: {TARGET_ASSET_ID}") - - try: - async for message in websocket: - current_ts = round(datetime.now().timestamp()) - sec_remaining = SLUG_END_TIME - current_ts - - if sec_remaining <= 0: - HIST_TRADES = np.empty((0, 2)) - - print('*** Attempting to unsub from past 5min') - update_unsub_msg = { - "operation": 'unsubscribe', - "assets_ids": [TARGET_ASSET_ID], - "custom_feature_enabled": True - } - await websocket.send(json.dumps(update_unsub_msg)) - - print('*** Attempting to SUB to new 5min') - slug_full = gen_slug() - market_details, market = get_mkt_details_by_slug(slug_full) - TARGET_ASSET_ID = market_details['Up'] - SLUG_END_TIME = round(datetime.strptime(market_details['EndDateTime'], '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone.utc).timestamp()) - - update_sub_msg = { - "operation": 'subscribe', - "assets_ids": [TARGET_ASSET_ID], - "custom_feature_enabled": True - } - await websocket.send(json.dumps(update_sub_msg)) - - - if isinstance(message, str): - data = json.loads(message) - - if isinstance(data, dict): - # print(data.get("event_type", None)) - pass - elif isinstance(data, list): - print('initial book: ') - print(data) - continue - else: - raise ValueError(f'Type: {type(data)} not expected: {message}') - - event_type = data.get("event_type", None) - - if event_type == "price_change": - # print("📈 Price Change") - # print(pd.DataFrame(data['price_changes'])) - pass - elif event_type == "best_bid_ask": - # print(pd.DataFrame([data])) - pass - elif event_type == "last_trade_price": - px = float(data['price']) - qty = float(data['size']) - HIST_TRADES = np.append(HIST_TRADES, np.array([[px, qty]]), axis=0) - SMA = talib.ROC(HIST_TRADES[:,0], timeperiod=10)[-1] - print(f"✨ Last Px: {px:.2f}; ROC: {SMA:.4f}; Qty: {qty:6.2f}; Sec Left: {sec_remaining}") - elif event_type == "book": - pass - elif event_type == "new_market": - print('Received new_market') - elif event_type == "market_resolved": - print(f"Received: {event_type}") - print(data) - elif event_type == "tick_size_change": # may want for CLOB order routing - print(f"Received: {event_type}") - print(data) - else: - print(f"Received: {event_type}") - print(data) - - except websockets.ConnectionClosed: - print("Connection closed by server.") - - -if __name__ == '__main__': - try: - asyncio.run(polymarket_stream()) - except KeyboardInterrupt: - print("Stream stopped.") \ No newline at end of file diff --git a/ws_binance.py b/ws_binance.py new file mode 100644 index 0000000..42645b7 --- /dev/null +++ b/ws_binance.py @@ -0,0 +1,214 @@ +import asyncio +import json +import logging +import socket +import traceback +from datetime import datetime +from typing import AsyncContextManager + +import numpy as np +import pandas as pd +import requests.packages.urllib3.util.connection as urllib3_cn # type: ignore +from sqlalchemy import text +import websockets +from sqlalchemy.ext.asyncio import create_async_engine +import valkey +import os +from dotenv import load_dotenv + + +### Allow only ipv4 ### +def allowed_gai_family(): + return socket.AF_INET +urllib3_cn.allowed_gai_family = allowed_gai_family + +### Database ### +USE_DB: bool = True +USE_VK: bool = True +VK_CHANNEL = 'poly_binance_btcusd' +CON: AsyncContextManager | None = None +VAL_KEY = None + +### Logging ### +load_dotenv() +LOG_FILEPATH: str = os.getenv("LOGS_PATH") + '/Polymarket_Binance_Trades.log' + +### Globals ### +WSS_URL = "wss://stream.binance.com:9443/ws/BTCUSDT@aggTrade" +HIST_TRADES = np.empty((0, 3)) +HIST_TRADES_LOOKBACK_SEC = 6 + +### Database Funcs ### +async def create_rtds_btcusd_table( + CON: AsyncContextManager, + engine: str = 'mysql', # mysql | duckdb + ) -> None: + if CON is None: + logging.info("NO DB CONNECTION, SKIPPING Create Statements") + else: + if engine == 'mysql': + logging.info('Creating Table if Does Not Exist: binance_btcusd_trades') + await CON.execute(text(""" + CREATE TABLE IF NOT EXISTS binance_btcusd_trades ( + timestamp_arrival BIGINT, + timestamp_msg BIGINT, + timestamp_value BIGINT, + value DOUBLE, + qty DOUBLE + ); + """)) + await CON.commit() + else: + raise ValueError('Only MySQL engine is implemented') + +async def insert_rtds_btcusd_table( + timestamp_arrival: int, + timestamp_msg: int, + timestamp_value: int, + value: float, + qty: float, + CON: AsyncContextManager, + engine: str = 'mysql', # mysql | duckdb + ) -> None: + params={ + 'timestamp_arrival': timestamp_arrival, + 'timestamp_msg': timestamp_msg, + 'timestamp_value': timestamp_value, + 'value': value, + 'qty': qty, + } + if CON is None: + logging.info("NO DB CONNECTION, SKIPPING Insert Statements") + else: + if engine == 'mysql': + await CON.execute(text(""" + INSERT INTO binance_btcusd_trades + ( + timestamp_arrival, + timestamp_msg, + timestamp_value, + value, + qty + ) + VALUES + ( + :timestamp_arrival, + :timestamp_msg, + :timestamp_value, + :value, + :qty + ) + """), + parameters=params + ) + await CON.commit() + else: + raise ValueError('Only MySQL engine is implemented') + +### Websocket ### +async def binance_trades_stream(): + global HIST_TRADES + + async for websocket in websockets.connect(WSS_URL): + logging.info(f"Connected to {WSS_URL}") + + subscribe_msg = { + "method": "SUBSCRIBE", + "params": ["btcusdt@aggTrade"], + "id": 1 + } + + await websocket.send(json.dumps(subscribe_msg)) + + try: + async for message in websocket: + ts_arrival = round(datetime.now().timestamp()*1000) + if isinstance(message, str): + try: + data = json.loads(message) + if data.get('T', None) is not None: + timestamp_msg = data['E'] + timestamp_value = data['T'] + last_px = float(data['p']) + qty = float(data['q']) + # print(f'🤑 BTC Binance Last Px: {last_px:_.4f}; TS: {pd.to_datetime(data['T'], unit='ms')}') + # HIST_TRADES = np.append(HIST_TRADES, np.array([[timestamp_value, last_px, qty]]), axis=0) + HIST_TRADES = np.append(HIST_TRADES, np.array([[ts_arrival, last_px, qty]]), axis=0) + hist_trades_lookback_ts_ms = round(datetime.now().timestamp() - HIST_TRADES_LOOKBACK_SEC)*1000 + HIST_TRADES = HIST_TRADES[HIST_TRADES[:, 0] >= hist_trades_lookback_ts_ms] + VAL_KEY_OBJ = json.dumps({ + 'timestamp_arrival': ts_arrival, + 'timestamp_msg': timestamp_msg, + 'timestamp_value': timestamp_value, + 'value': last_px, + 'qty': qty, + 'hist_trades': HIST_TRADES.tolist() + }) + # VAL_KEY.publish(VK_CHANNEL, VAL_KEY_OBJ) + VAL_KEY.set(VK_CHANNEL, VAL_KEY_OBJ) + await insert_rtds_btcusd_table( + CON=CON, + timestamp_arrival=ts_arrival, + timestamp_msg=timestamp_msg, + timestamp_value=timestamp_value, + value=last_px, + qty=qty, + ) + else: + logging.info(f'Initial or unexpected data struct, skipping: {data}') + continue + except (json.JSONDecodeError, ValueError): + logging.warning(f'Message not in JSON format, skipping: {message}') + continue + else: + raise ValueError(f'Type: {type(data)} not expected: {message}') + except websockets.ConnectionClosed as e: + logging.error(f'Connection closed: {e}') + logging.error(traceback.format_exc()) + continue + except Exception as e: + logging.error(f'Connection closed: {e}') + logging.error(traceback.format_exc()) + + +async def main(): + global VAL_KEY + global CON + + if USE_VK: + VAL_KEY = valkey.Valkey(host='localhost', port=6379, db=0) + # published_count = VAL_KEY.publish(VK_CHANNEL,f"Hola, starting to publish to valkey: {VK_CHANNEL} @ {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + # logging.info(f"Valkey message published to {published_count} subscribers of {VK_CHANNEL}") + else: + VAL_KEY = None + logging.warning("VALKEY NOT BEING USED, NO DATA WILL BE PUBLISHED") + + if USE_DB: + engine = create_async_engine('mysql+asyncmy://root:pwd@localhost/polymarket') + async with engine.connect() as CON: + await create_rtds_btcusd_table(CON=CON) + await binance_trades_stream() + else: + CON = None + logging.warning("DATABASE NOT BEING USED, NO DATA WILL BE RECORDED") + await binance_trades_stream() + + +if __name__ == '__main__': + START_TIME = round(datetime.now().timestamp()*1000) + + logging.info(f'Log FilePath: {LOG_FILEPATH}') + + logging.basicConfig( + force=True, + filename=LOG_FILEPATH, + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + filemode='w' + ) + logging.info(f"STARTED: {START_TIME}") + + try: + asyncio.run(main()) + except KeyboardInterrupt: + logging.info("Stream stopped") \ No newline at end of file diff --git a/ws_binance/Dockerfile b/ws_binance/Dockerfile new file mode 100644 index 0000000..f2899de --- /dev/null +++ b/ws_binance/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3.13-slim + +RUN apt-get update && \ + apt-get install -y build-essential + +RUN gcc --version +RUN rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt . + +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +# Finally, run gunicorn. +CMD [ "python", "ws_binance.py"] +# CMD [ "gunicorn", "--workers=5", "--threads=1", "-b 0.0.0.0:8000", "app:server"] \ No newline at end of file diff --git a/ws_clob.py b/ws_clob.py new file mode 100644 index 0000000..ed381ea --- /dev/null +++ b/ws_clob.py @@ -0,0 +1,380 @@ +import asyncio +import json +import math +import logging +import pandas as pd +import os +from datetime import datetime, timezone +import websockets +import numpy as np +import talib +import requests +from typing import AsyncContextManager +from sqlalchemy.ext.asyncio import create_async_engine +from sqlalchemy import text +import valkey +import time + +import os +from dotenv import load_dotenv + +### Database ### +USE_DB: bool = True +USE_VK: bool = True +VK_CHANNEL = 'poly_5min_btcusd' +VK_CHANNEL_DOWN = 'poly_5min_btcusd_down' +CON: AsyncContextManager | None = None +VAL_KEY = None + +### Logging ### +load_dotenv() +LOG_FILEPATH: str = os.getenv("LOGS_PATH") + '/Polymarket_CLOB.log' + +WSS_URL = "wss://ws-subscriptions-clob.polymarket.com/ws/market" +SLUG_END_TIME = 0 + +HIST_TRADES = np.empty((0, 2)) +HIST_TRADES_DOWN = np.empty((0, 2)) +MIN_TICK_SIZE = 0.01 +NEG_RISK = False + +TARGET_PX = 0 + +TARGET_ASSET_ID = None +TARGET_ASSET_ID_DOWN = None + +def format_timestamp(total_seconds) -> str: + minutes, seconds = divmod(total_seconds, 60) + + return f"{minutes} minutes and {seconds} seconds" + +def time_round_down(dt, interval_mins=5) -> int: # returns timestamp in seconds + interval_secs = interval_mins * 60 + seconds = dt.timestamp() + rounded_seconds = math.floor(seconds / interval_secs) * interval_secs + + return rounded_seconds + +def get_mkt_details_by_slug(slug: str) -> dict[str, str, str]: # {'Up' : 123, 'Down': 456, 'isActive': True, 'MinTickSize': 0.01, 'isNegRisk': False} + r = requests.get(f"https://gamma-api.polymarket.com/events/slug/{slug}") + market = r.json()['markets'][0] + token_ids = json.loads(market.get("clobTokenIds", "[]")) + outcomes = json.loads(market.get("outcomes", "[]")) + d = dict(zip(outcomes, token_ids)) + d['isActive'] = market['negRisk'] + d['MinTickSize'] = market['orderPriceMinTickSize'] + d['OrderMinSize'] = market['orderMinSize'] + d['isNegRisk'] = market['negRisk'] + d['ConditionId'] = market['conditionId'] + d['EndDateTime'] = market['endDate'] + # d['Liquidity'] = market['liquidity'] + # d['LiquidityClob'] = market['liquidityClob'] + # d['VolumeNum'] = market['volumeNum'] + # d['Volume24hr'] = market['volume24hr'] + logging.info(f'MARKET CHANGED: {market}') + + return d, market + +def gen_slug(): + slug_prefix = 'btc-updown-5m-' + slug_time_id = time_round_down(dt=datetime.now(timezone.utc)) + + return slug_prefix + str(slug_time_id) + + +### Database Funcs ### +async def create_poly_btcusd_trades_table( + CON: AsyncContextManager, + engine: str = 'mysql', # mysql | duckdb + ) -> None: + if CON is None: + logging.info("NO DB CONNECTION, SKIPPING Create Statements") + else: + if engine == 'mysql': + logging.info('Creating Table if Does Not Exist: poly_btcusd_trades') + await CON.execute(text(""" + CREATE TABLE IF NOT EXISTS poly_btcusd_trades ( + timestamp_arrival BIGINT, + timestamp_msg BIGINT, + timestamp_value BIGINT, + price DOUBLE, + qty DOUBLE, + side_taker VARCHAR(8), + up_or_down VARCHAR(8) + ); + """)) + await CON.commit() + else: + raise ValueError('Only MySQL engine is implemented') + +async def insert_poly_btcusd_trades_table( + timestamp_arrival: int, + timestamp_msg: int, + timestamp_value: int, + price: float, + qty: float, + side_taker: str, + up_or_down: str, + CON: AsyncContextManager, + engine: str = 'mysql', # mysql | duckdb + ) -> None: + params={ + 'timestamp_arrival': timestamp_arrival, + 'timestamp_msg': timestamp_msg, + 'timestamp_value': timestamp_value, + 'price': price, + 'qty': qty, + 'side_taker': side_taker, + 'up_or_down': up_or_down, + } + if CON is None: + logging.info("NO DB CONNECTION, SKIPPING Insert Statements") + else: + if engine == 'mysql': + await CON.execute(text(""" + INSERT INTO poly_btcusd_trades + ( + timestamp_arrival, + timestamp_msg, + timestamp_value, + price, + qty, + side_taker, + up_or_down + ) + VALUES + ( + :timestamp_arrival, + :timestamp_msg, + :timestamp_value, + :price, + :qty, + :side_taker, + :up_or_down + ) + """), + parameters=params + ) + await CON.commit() + else: + raise ValueError('Only MySQL engine is implemented') + + +async def polymarket_stream(): + global SLUG_END_TIME + global TARGET_PX + global HIST_TRADES + global HIST_TRADES_DOWN + global MIN_TICK_SIZE + global NEG_RISK + global TARGET_ASSET_ID + global TARGET_ASSET_ID_DOWN + + slug_full = gen_slug() + market_details, _ = get_mkt_details_by_slug(slug_full) + CONDITION_ID = market_details['ConditionId'] + TARGET_ASSET_ID = market_details['Up'] + TARGET_ASSET_ID_DOWN = market_details['Down'] + MIN_TICK_SIZE = market_details['MinTickSize'] + NEG_RISK = market_details['isNegRisk'] + SLUG_END_TIME = round(datetime.strptime(market_details['EndDateTime'], '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone.utc).timestamp()) + print(f'********* NEW MKT - END DATETIME: {pd.to_datetime(SLUG_END_TIME, unit='s')} *********') + + async for websocket in websockets.connect(WSS_URL): + print(f"Connected to {WSS_URL}") + + subscribe_msg = { + "assets_ids": [TARGET_ASSET_ID, TARGET_ASSET_ID_DOWN], + "type": "market", + "custom_feature_enabled": False + } + + await websocket.send(json.dumps(subscribe_msg)) + print(f"Subscribed to Assets: Up {TARGET_ASSET_ID}; Down: {TARGET_ASSET_ID_DOWN}") + + try: + async for message in websocket: + ts_arrival = round(datetime.now().timestamp()*1000) + sec_remaining = SLUG_END_TIME - round(datetime.now().timestamp()) + + if sec_remaining <= 0: + time.sleep(0.1) + ref_data = json.loads(VAL_KEY.get('poly_rtds_cl_btcusd')) + TARGET_PX = float(ref_data['value']) + HIST_TRADES = np.empty((0, 2)) + + print('*** Attempting to unsub from past 5min') + update_unsub_msg = { + "operation": 'unsubscribe', + "assets_ids": [TARGET_ASSET_ID, TARGET_ASSET_ID_DOWN], + "custom_feature_enabled": False + } + await websocket.send(json.dumps(update_unsub_msg)) + + print('*** Attempting to SUB to new 5min') + slug_full = gen_slug() + market_details, market = get_mkt_details_by_slug(slug_full) + CONDITION_ID = market_details['ConditionId'] + TARGET_ASSET_ID = market_details['Up'] + TARGET_ASSET_ID_DOWN = market_details['Down'] + MIN_TICK_SIZE = market_details['MinTickSize'] + NEG_RISK = market_details['isNegRisk'] + SLUG_END_TIME = round(datetime.strptime(market_details['EndDateTime'], '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone.utc).timestamp()) + + update_sub_msg = { + "operation": 'subscribe', + "assets_ids": [TARGET_ASSET_ID, TARGET_ASSET_ID_DOWN], + "custom_feature_enabled": False + } + await websocket.send(json.dumps(update_sub_msg)) + + if isinstance(message, str): + data = json.loads(message) + if isinstance(data, list): + print('initial book:') + print(data) + continue + + event_type = data.get("event_type", None) + + if event_type == "price_change": + # print("📈 Price Change") + # print(pd.DataFrame(data['price_changes'])) + continue + elif event_type == "best_bid_ask": + # print(pd.DataFrame([data])) + continue + elif event_type == "last_trade_price": + token_id = data['asset_id'] + ts_msg = int(data['timestamp']) + ts_value = int(ts_msg) + px = float(data['price']) + qty = float(data['size']) + side_taker = data['side'] + if token_id == TARGET_ASSET_ID: + up_or_down = 'UP' + HIST_TRADES = np.append(HIST_TRADES, np.array([[px, qty]]), axis=0) + # print(f"✨ Last Px: {px:.2f}; Qty: {qty:6.2f}; Sec Left: {sec_remaining}") + # print(f'Up: {TARGET_ASSET_ID}') + # print(f'Down: {TARGET_ASSET_ID_DOWN}') + # SMA = talib.ROC(HIST_TRADES[:,0], timeperiod=10)[-1] + # print(f"✨ Last Px: {px:.2f}; ROC: {SMA:.4f}; Qty: {qty:6.2f}; Sec Left: {sec_remaining}") + if USE_VK: + VAL_KEY_OBJ = json.dumps({ + 'timestamp_arrival': ts_arrival, + 'timestamp_msg': ts_msg, + 'timestamp_value': ts_value, + 'price': px, + 'qty': qty, + 'side_taker': side_taker, + 'sec_remaining': sec_remaining, + 'target_price': TARGET_PX, + 'condition_id': CONDITION_ID, + 'token_id_up': TARGET_ASSET_ID, + 'token_id_down': TARGET_ASSET_ID_DOWN, + 'tick_size': MIN_TICK_SIZE, + 'neg_risk': NEG_RISK, + }) + VAL_KEY.set(VK_CHANNEL, VAL_KEY_OBJ) + elif token_id == TARGET_ASSET_ID_DOWN: + up_or_down = 'DOWN' + HIST_TRADES_DOWN = np.append(HIST_TRADES_DOWN, np.array([[px, qty]]), axis=0) + if USE_VK: + VAL_KEY_OBJ = json.dumps({ + 'timestamp_arrival': ts_arrival, + 'timestamp_msg': ts_msg, + 'timestamp_value': ts_value, + 'price': px, + 'qty': qty, + 'side_taker': side_taker, + 'sec_remaining': sec_remaining, + 'target_price': TARGET_PX, + 'condition_id': CONDITION_ID, + 'token_id_up': TARGET_ASSET_ID, + 'token_id_down': TARGET_ASSET_ID_DOWN, + 'tick_size': MIN_TICK_SIZE, + 'neg_risk': NEG_RISK, + }) + VAL_KEY.set(VK_CHANNEL_DOWN, VAL_KEY_OBJ) + else: + logging.warning('Token Id from Market Does Not Match Pricing Data Id') + + if USE_DB: + await insert_poly_btcusd_trades_table( + CON=CON, + timestamp_arrival=ts_arrival, + timestamp_msg=ts_msg, + timestamp_value=ts_value, + price=px, + qty=qty, + side_taker=side_taker, + up_or_down=up_or_down + ) + + elif event_type == "book": + continue + elif event_type == "new_market": + print('Received new_market') + continue + elif event_type == "market_resolved": + print(f"Received: {event_type}") + # print(data) + continue + elif event_type == "tick_size_change": # may want for CLOB order routing + print(f"Received: {event_type}") + # print(data) + continue + else: + print(f"*********** REC UNMAPPED EVENT: {event_type}") + # print(data) + continue + elif isinstance(data, dict): + continue + else: + raise ValueError(f'Type: {type(data)} not expected: {message}') + + except websockets.ConnectionClosed as e: + print(f"Connection closed by server. Exception: {e}") + continue + +async def main(): + global VAL_KEY + global CON + + if USE_VK: + VAL_KEY = valkey.Valkey(host='localhost', port=6379, db=0) + # published_count = VAL_KEY.publish(VK_CHANNEL,f"Hola, starting to publish to valkey: {VK_CHANNEL} @ {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + # logging.info(f"Valkey message published to {published_count} subscribers of {VK_CHANNEL}") + else: + VAL_KEY = None + logging.warning("VALKEY NOT BEING USED, NO DATA WILL BE PUBLISHED") + + if USE_DB: + engine = create_async_engine('mysql+asyncmy://root:pwd@localhost/polymarket') + async with engine.connect() as CON: + await create_poly_btcusd_trades_table(CON=CON) + await polymarket_stream() + else: + CON = None + logging.warning("DATABASE NOT BEING USED, NO DATA WILL BE RECORDED") + await polymarket_stream() + + +if __name__ == '__main__': + START_TIME = round(datetime.now().timestamp()*1000) + + logging.info(f'Log FilePath: {LOG_FILEPATH}') + + logging.basicConfig( + force=True, + filename=LOG_FILEPATH, + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + filemode='w' + ) + logging.info(f"STARTED: {START_TIME}") + + try: + asyncio.run(main()) + except KeyboardInterrupt as e: + print(f"Stream stopped: {e}") \ No newline at end of file diff --git a/ws_clob/Dockerfile b/ws_clob/Dockerfile new file mode 100644 index 0000000..748b47f --- /dev/null +++ b/ws_clob/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3.13-slim + +RUN apt-get update && \ + apt-get install -y build-essential + +RUN gcc --version +RUN rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt . + +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +# Finally, run gunicorn. +CMD [ "python", "ws_clob.py"] +# CMD [ "gunicorn", "--workers=5", "--threads=1", "-b 0.0.0.0:8000", "app:server"] \ No newline at end of file diff --git a/ws_coinbase.py b/ws_coinbase.py new file mode 100644 index 0000000..93d5758 --- /dev/null +++ b/ws_coinbase.py @@ -0,0 +1,227 @@ +import asyncio +import json +import logging +import socket +import traceback +from datetime import datetime +from typing import AsyncContextManager +import os +# import numpy as np +import pandas as pd +import requests.packages.urllib3.util.connection as urllib3_cn # type: ignore +from sqlalchemy import text +import websockets +from sqlalchemy.ext.asyncio import create_async_engine +import valkey +import os +from dotenv import load_dotenv + +### Allow only ipv4 ### +def allowed_gai_family(): + return socket.AF_INET +urllib3_cn.allowed_gai_family = allowed_gai_family + +### Database ### +USE_DB: bool = True +USE_VK: bool = True +VK_CHANNEL = 'poly_coinbase_btcusd' +CON: AsyncContextManager | None = None +VAL_KEY = None + +### Logging ### +load_dotenv() +LOG_FILEPATH: str = os.getenv("LOGS_PATH") + '/Polymarket_coinbase_Trades.log' + +### Globals ### +WSS_URL = "wss://ws-feed.exchange.coinbase.com" +# HIST_TRADES = np.empty((0, 2)) + +### Database Funcs ### +async def create_rtds_btcusd_table( + CON: AsyncContextManager, + engine: str = 'mysql', # mysql | duckdb + ) -> None: + if CON is None: + logging.info("NO DB CONNECTION, SKIPPING Create Statements") + else: + if engine == 'mysql': + logging.info('Creating Table if Does Not Exist: coinbase_btcusd_trades') + await CON.execute(text(""" + CREATE TABLE IF NOT EXISTS coinbase_btcusd_trades ( + timestamp_arrival BIGINT, + timestamp_msg BIGINT, + timestamp_value BIGINT, + value DOUBLE, + qty DOUBLE, + side VARCHAR(8) + ); + """)) + await CON.commit() + else: + raise ValueError('Only MySQL engine is implemented') + +async def insert_rtds_btcusd_table( + timestamp_arrival: int, + timestamp_msg: int, + timestamp_value: int, + value: float, + qty: float, + side: str, + CON: AsyncContextManager, + engine: str = 'mysql', # mysql | duckdb + ) -> None: + params={ + 'timestamp_arrival': timestamp_arrival, + 'timestamp_msg': timestamp_msg, + 'timestamp_value': timestamp_value, + 'value': value, + 'qty': qty, + 'side': side, + } + if CON is None: + logging.info("NO DB CONNECTION, SKIPPING Insert Statements") + else: + if engine == 'mysql': + await CON.execute(text(""" + INSERT INTO coinbase_btcusd_trades + ( + timestamp_arrival, + timestamp_msg, + timestamp_value, + value, + qty, + side + ) + VALUES + ( + :timestamp_arrival, + :timestamp_msg, + :timestamp_value, + :value, + :qty, + :side + ) + """), + parameters=params + ) + await CON.commit() + else: + raise ValueError('Only MySQL engine is implemented') + + +### Websocket ### +async def coinbase_trades_stream(): + global HIST_TRADES + + async with websockets.connect(WSS_URL) as websocket: + logging.info(f"Connected to {WSS_URL}") + + subscribe_msg = { + "type": "subscribe", + "product_ids": ["BTC-USD"], + "channels": [ + { + "name": "ticker", + "product_ids": ["BTC-USD"] + } + ] + } + + await websocket.send(json.dumps(subscribe_msg)) + + try: + async for message in websocket: + if isinstance(message, str) or isinstance(message, bytes): + try: + data = json.loads(message) + if data.get('price', None) is not None: + ts_arrival = round(datetime.now().timestamp()*1000) + ts_msg = round(datetime.strptime(data['time'], "%Y-%m-%dT%H:%M:%S.%fZ").timestamp()*1000) + ts_value = ts_msg + last_px = float(data['price']) + qty = float(data['last_size']) + side = data['side'] + print(f'🤑 BTC Coinbase Last Px: {last_px:_.4f}; TS: {pd.to_datetime(ts_value, unit='ms')}; Side: {side};') + if USE_VK: + VAL_KEY_OBJ = json.dumps({ + 'timestamp_arrival': ts_arrival, + 'timestamp_msg': ts_msg, + 'timestamp_value': ts_value, + 'value': last_px, + 'qty': qty, + 'side': side, + }) + VAL_KEY.publish(VK_CHANNEL, VAL_KEY_OBJ) + VAL_KEY.set(VK_CHANNEL, VAL_KEY_OBJ) + if USE_DB: + await insert_rtds_btcusd_table( + CON=CON, + timestamp_arrival=ts_arrival, + timestamp_msg=ts_msg, + timestamp_value=ts_value, + value=last_px, + qty=qty, + side=side, + ) + # elif data.get('op'): + # if data['op'] == 'PING': + # pong = {"op": "PONG", "timestamp": ts_arrival} + # await websocket.send(json.dumps(pong)) + # logging.info(f'PING RECEIVED: {data}; PONG SENT: {pong}') + else: + logging.info(f'Initial or unexpected data struct, skipping: {data}') + continue + except (json.JSONDecodeError, ValueError) as e: + logging.warning(f'Message not in JSON format, skipping: {message}; excepion: {e}') + continue + else: + raise ValueError(f'Type: {type(message)} not expected: {message}') + except websockets.ConnectionClosed as e: + logging.error(f'Connection closed: {e}') + logging.error(traceback.format_exc()) + except Exception as e: + logging.error(f'Connection closed: {e}') + logging.error(traceback.format_exc()) + + +async def main(): + global VAL_KEY + global CON + + if USE_VK: + VAL_KEY = valkey.Valkey(host='localhost', port=6379, db=0) + published_count = VAL_KEY.publish(VK_CHANNEL,f"Hola, starting to publish to valkey: {VK_CHANNEL} @ {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + logging.info(f"Valkey message published to {published_count} subscribers of {VK_CHANNEL}") + else: + VAL_KEY = None + logging.warning("VALKEY NOT BEING USED, NO DATA WILL BE PUBLISHED") + + if USE_DB: + engine = create_async_engine('mysql+asyncmy://root:pwd@localhost/polymarket') + async with engine.connect() as CON: + await create_rtds_btcusd_table(CON=CON) + await coinbase_trades_stream() + else: + CON = None + logging.warning("DATABASE NOT BEING USED, NO DATA WILL BE RECORDED") + await coinbase_trades_stream() + + +if __name__ == '__main__': + START_TIME = round(datetime.now().timestamp()*1000) + + logging.info(f'Log FilePath: {LOG_FILEPATH}') + + logging.basicConfig( + force=True, + filename=LOG_FILEPATH, + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + filemode='w' + ) + logging.info(f"STARTED: {START_TIME}") + + try: + asyncio.run(main()) + except KeyboardInterrupt: + logging.info("Stream stopped") \ No newline at end of file diff --git a/ws_coinbase/Dockerfile b/ws_coinbase/Dockerfile new file mode 100644 index 0000000..ceba70c --- /dev/null +++ b/ws_coinbase/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3.13-slim + +RUN apt-get update && \ + apt-get install -y build-essential + +RUN gcc --version +RUN rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt . + +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +# Finally, run gunicorn. +CMD [ "python", "ws_coinbase.py"] +# CMD [ "gunicorn", "--workers=5", "--threads=1", "-b 0.0.0.0:8000", "app:server"] \ No newline at end of file diff --git a/ws_pionex.py b/ws_pionex.py new file mode 100644 index 0000000..5c6efb4 --- /dev/null +++ b/ws_pionex.py @@ -0,0 +1,222 @@ +import asyncio +import json +import logging +import socket +import traceback +from datetime import datetime +from typing import AsyncContextManager + +import numpy as np +import pandas as pd +import requests.packages.urllib3.util.connection as urllib3_cn # type: ignore +from sqlalchemy import text +import websockets +from sqlalchemy.ext.asyncio import create_async_engine +import valkey +import os +from dotenv import load_dotenv + +### Allow only ipv4 ### +def allowed_gai_family(): + return socket.AF_INET +urllib3_cn.allowed_gai_family = allowed_gai_family + +### Database ### +USE_DB: bool = True +USE_VK: bool = True +VK_CHANNEL = 'poly_pionex_btcusd' +CON: AsyncContextManager | None = None +VAL_KEY = None + + +### Logging ### +load_dotenv() +LOG_FILEPATH: str = os.getenv("LOGS_PATH") + '/Polymarket_Pionex_Trades.log' + +### Globals ### +WSS_URL = "wss://ws.pionex.com/wsPub" +# HIST_TRADES = np.empty((0, 2)) + +### Database Funcs ### +async def create_rtds_btcusd_table( + CON: AsyncContextManager, + engine: str = 'mysql', # mysql | duckdb + ) -> None: + if CON is None: + logging.info("NO DB CONNECTION, SKIPPING Create Statements") + else: + if engine == 'mysql': + logging.info('Creating Table if Does Not Exist: pionex_btcusd_trades') + await CON.execute(text(""" + CREATE TABLE IF NOT EXISTS pionex_btcusd_trades ( + timestamp_msg BIGINT, + timestamp_value BIGINT, + value DOUBLE, + qty DOUBLE, + side VARCHAR(8) + ); + """)) + await CON.commit() + else: + raise ValueError('Only MySQL engine is implemented') + +async def insert_rtds_btcusd_table( + timestamp_msg: int, + timestamp_value: int, + value: float, + qty: float, + side: str, + CON: AsyncContextManager, + engine: str = 'mysql', # mysql | duckdb + ) -> None: + params={ + 'timestamp_msg': timestamp_msg, + 'timestamp_value': timestamp_value, + 'value': value, + 'qty': qty, + 'side': side, + } + if CON is None: + logging.info("NO DB CONNECTION, SKIPPING Insert Statements") + else: + if engine == 'mysql': + await CON.execute(text(""" + INSERT INTO pionex_btcusd_trades + ( + timestamp_msg, + timestamp_value, + value, + qty, + side + ) + VALUES + ( + :timestamp_msg, + :timestamp_value, + :value, + :qty, + :side + ) + """), + parameters=params + ) + await CON.commit() + else: + raise ValueError('Only MySQL engine is implemented') + + +### Websocket ### +async def pionex_trades_stream(): + global HIST_TRADES + + async with websockets.connect(WSS_URL) as websocket: + logging.info(f"Connected to {WSS_URL}") + + subscribe_msg = { + "op": "SUBSCRIBE", + "topic": "TRADE", + "symbol": "BTC_USDT" + } + + await websocket.send(json.dumps(subscribe_msg)) + + try: + async for message in websocket: + if isinstance(message, str) or isinstance(message, bytes): + try: + data = json.loads(message) + if data.get('data', None) is not None: + ts_msg = data['timestamp'] + data = data['data'] + ts_value = data[0]['timestamp'] + last_px = float(data[0]['price']) + qty = float(data[0]['size']) + side = data[0]['side'] + print(f'🤑 BTC Pionex Last Px: {last_px:_.4f}; TS: {pd.to_datetime(ts_value, unit='ms')}; Side: {side};') + print(ts_value) + if USE_VK: + VAL_KEY.publish(VK_CHANNEL, json.dumps({ + 'timestamp_msg': ts_msg, + 'timestamp_value': ts_value, + 'value': last_px, + 'qty': qty, + 'side': side, + })) + VAL_KEY.set(VK_CHANNEL, json.dumps({ + 'timestamp_msg': ts_msg, + 'timestamp_value': ts_value, + 'value': last_px, + 'qty': qty, + 'side': side, + })) + if USE_DB: + await insert_rtds_btcusd_table( + CON=CON, + timestamp_msg=ts_msg, + timestamp_value=ts_value, + value=last_px, + qty=qty, + side=side, + ) + elif data.get('op'): + if data['op'] == 'PING': + pong = {"op": "PONG", "timestamp": round(datetime.now().timestamp()*1000)} + await websocket.send(json.dumps(pong)) + logging.info(f'PING RECEIVED: {data}; PONG SENT: {pong}') + else: + logging.info(f'Initial or unexpected data struct, skipping: {data}') + continue + except (json.JSONDecodeError, ValueError) as e: + logging.warning(f'Message not in JSON format, skipping: {message}; excepion: {e}') + continue + else: + raise ValueError(f'Type: {type(message)} not expected: {message}') + except websockets.ConnectionClosed as e: + logging.error(f'Connection closed: {e}') + logging.error(traceback.format_exc()) + except Exception as e: + logging.error(f'Connection closed: {e}') + logging.error(traceback.format_exc()) + + +async def main(): + global VAL_KEY + global CON + + if USE_VK: + VAL_KEY = valkey.Valkey(host='localhost', port=6379, db=0) + published_count = VAL_KEY.publish(VK_CHANNEL,f"Hola, starting to publish to valkey: {VK_CHANNEL} @ {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + logging.info(f"Valkey message published to {published_count} subscribers of {VK_CHANNEL}") + else: + VAL_KEY = None + logging.warning("VALKEY NOT BEING USED, NO DATA WILL BE PUBLISHED") + + if USE_DB: + engine = create_async_engine('mysql+asyncmy://root:pwd@localhost/polymarket') + async with engine.connect() as CON: + await create_rtds_btcusd_table(CON=CON) + await pionex_trades_stream() + else: + CON = None + logging.warning("DATABASE NOT BEING USED, NO DATA WILL BE RECORDED") + await pionex_trades_stream() + + +if __name__ == '__main__': + START_TIME = round(datetime.now().timestamp()*1000) + + logging.info(f'Log FilePath: {LOG_FILEPATH}') + + logging.basicConfig( + force=True, + filename=LOG_FILEPATH, + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + filemode='w' + ) + logging.info(f"STARTED: {START_TIME}") + + try: + asyncio.run(main()) + except KeyboardInterrupt: + logging.info("Stream stopped") \ No newline at end of file diff --git a/ws_rtds.py b/ws_rtds.py index e860944..73663c9 100644 --- a/ws_rtds.py +++ b/ws_rtds.py @@ -1,24 +1,111 @@ import asyncio import json -import math -import pandas as pd -import os -from datetime import datetime, timezone -import websockets +import logging +import socket +import traceback +from datetime import datetime +from typing import AsyncContextManager + import numpy as np -import talib -import requests +import pandas as pd +import requests.packages.urllib3.util.connection as urllib3_cn # type: ignore +from sqlalchemy import text +import websockets +from sqlalchemy.ext.asyncio import create_async_engine +import valkey +import os +from dotenv import load_dotenv +### Allow only ipv4 ### +def allowed_gai_family(): + return socket.AF_INET +urllib3_cn.allowed_gai_family = allowed_gai_family + +### Database ### +USE_DB: bool = True +USE_VK: bool = True +VK_CHANNEL = 'poly_rtds_cl_btcusd' +CON: AsyncContextManager | None = None +VAL_KEY = None + + +### Logging ### +load_dotenv() +LOG_FILEPATH: str = os.getenv("LOGS_PATH") + '/Polymarket_RTDS.log' + +### Globals ### WSS_URL = "wss://ws-live-data.polymarket.com" +# HIST_TRADES = np.empty((0, 2)) -HIST_TRADES = np.empty((0, 2)) +### Database Funcs ### +async def create_rtds_btcusd_table( + CON: AsyncContextManager, + engine: str = 'mysql', # mysql | duckdb + ) -> None: + if CON is None: + logging.info("NO DB CONNECTION, SKIPPING Create Statements") + else: + if engine == 'mysql': + logging.info('Creating Table if Does Not Exist: poly_rtds_cl_btcusd') + await CON.execute(text(""" + CREATE TABLE IF NOT EXISTS poly_rtds_cl_btcusd ( + timestamp_arrival BIGINT, + timestamp_msg BIGINT, + timestamp_value BIGINT, + value DOUBLE + ); + """)) + await CON.commit() + else: + raise ValueError('Only MySQL engine is implemented') + +async def insert_rtds_btcusd_table( + timestamp_arrival: int, + timestamp_msg: int, + timestamp_value: int, + value: int, + CON: AsyncContextManager, + engine: str = 'mysql', # mysql | duckdb + ) -> None: + params={ + 'timestamp_arrival': timestamp_arrival, + 'timestamp_msg': timestamp_msg, + 'timestamp_value': timestamp_value, + 'value': value, + } + if CON is None: + logging.info("NO DB CONNECTION, SKIPPING Insert Statements") + else: + if engine == 'mysql': + await CON.execute(text(""" + INSERT INTO poly_rtds_cl_btcusd + ( + timestamp_arrival, + timestamp_msg, + timestamp_value, + value + ) + VALUES + ( + :timestamp_arrival, + :timestamp_msg, + :timestamp_value, + :value + ) + """), + parameters=params + ) + await CON.commit() + else: + raise ValueError('Only MySQL engine is implemented') +### Websocket ### async def rtds_stream(): global HIST_TRADES - async with websockets.connect(WSS_URL) as websocket: - print(f"Connected to {WSS_URL}") + async for websocket in websockets.connect(WSS_URL): + logging.info(f"Connected to {WSS_URL}") subscribe_msg = { "action": "subscribe", @@ -39,23 +126,79 @@ async def rtds_stream(): try: data = json.loads(message) if data['payload'].get('value', None) is not None: - print(f'🤑 BTC Chainlink Last Px: {data['payload']['value']:_.4f}; TS: {pd.to_datetime(data['timestamp'], unit='ms')}') + ts_arrival = round(datetime.now().timestamp()*1000) + # print(f'🤑 BTC Chainlink Last Px: {data['payload']['value']:_.4f}; TS: {pd.to_datetime(data['payload']['timestamp'], unit='ms')}') + VAL_KEY_OBJ = json.dumps({ + 'timestamp_arrival': ts_arrival, + 'timestamp_msg': data['timestamp'], + 'timestamp_value': data['payload']['timestamp'], + 'value': data['payload']['value'], + }) + # VAL_KEY.publish(VK_CHANNEL, VAL_KEY_OBJ) + VAL_KEY.set(VK_CHANNEL, VAL_KEY_OBJ) + await insert_rtds_btcusd_table( + CON=CON, + timestamp_arrival=ts_arrival, + timestamp_msg=data['timestamp'], + timestamp_value=data['payload']['timestamp'], + value=data['payload']['value'], + ) else: - print(f'Initial or unexpected data struct, skipping: {data}') + # logging.info(f'Initial or unexpected data struct, skipping: {data}') + logging.info('Initial or unexpected data struct, skipping') continue except (json.JSONDecodeError, ValueError): - print(f'Message not in JSON format, skipping: {message}') + logging.warning(f'Message not in JSON format, skipping: {message}') continue else: raise ValueError(f'Type: {type(data)} not expected: {message}') - + except websockets.ConnectionClosed as e: + logging.error(f'Connection closed: {e}') + logging.error(traceback.format_exc()) + continue + except Exception as e: + logging.error(f'Connection closed: {e}') + logging.error(traceback.format_exc()) - except websockets.ConnectionClosed: - print("Connection closed by server.") + +async def main(): + global VAL_KEY + global CON + + if USE_VK: + VAL_KEY = valkey.Valkey(host='localhost', port=6379, db=0) + # published_count = VAL_KEY.publish(VK_CHANNEL,f"Hola, starting to publish to valkey: {VK_CHANNEL} @ {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + # logging.info(f"Valkey message published to {published_count} subscribers of {VK_CHANNEL}") + else: + VAL_KEY = None + logging.warning("VALKEY NOT BEING USED, NO DATA WILL BE PUBLISHED") + + if USE_DB: + engine = create_async_engine('mysql+asyncmy://root:pwd@localhost/polymarket') + async with engine.connect() as CON: + await create_rtds_btcusd_table(CON=CON) + await rtds_stream() + else: + CON = None + logging.warning("DATABASE NOT BEING USED, NO DATA WILL BE RECORDED") + await rtds_stream() if __name__ == '__main__': + START_TIME = round(datetime.now().timestamp()*1000) + + logging.info(f'Log FilePath: {LOG_FILEPATH}') + + logging.basicConfig( + force=True, + filename=LOG_FILEPATH, + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + filemode='w' + ) + logging.info(f"STARTED: {START_TIME}") + try: - asyncio.run(rtds_stream()) + asyncio.run(main()) except KeyboardInterrupt: - print("Stream stopped.") \ No newline at end of file + logging.info("Stream stopped") \ No newline at end of file diff --git a/ws_rtds/Dockerfile b/ws_rtds/Dockerfile new file mode 100644 index 0000000..5cb3f84 --- /dev/null +++ b/ws_rtds/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3.13-slim + +RUN apt-get update && \ + apt-get install -y build-essential + +RUN gcc --version +RUN rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt . + +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +# Finally, run gunicorn. +CMD [ "python", "ws_rtds.py"] +# CMD [ "gunicorn", "--workers=5", "--threads=1", "-b 0.0.0.0:8000", "app:server"] \ No newline at end of file diff --git a/ws_user.py b/ws_user.py new file mode 100644 index 0000000..b1eeb88 --- /dev/null +++ b/ws_user.py @@ -0,0 +1,430 @@ +import asyncio +import json +import logging +import os +from datetime import datetime +from typing import AsyncContextManager + +import numpy as np +import valkey +import websockets +from dotenv import load_dotenv +from py_clob_client.client import ClobClient +from sqlalchemy import text +from sqlalchemy.ext.asyncio import create_async_engine + +### Database ### +USE_DB: bool = True +USE_VK: bool = True + +LOCAL_LIVE_ORDERS = [] +LOCAL_RECENT_TRADES = [] +LOCAL_RECENT_TRADES_LOOKBACK_SEC = 10 + +VK_LIVE_ORDERS = 'poly_user_orders' +VK_RECENT_TRADES = 'poly_user_trades' +CON: AsyncContextManager | None = None +VAL_KEY = None + +### Logging ### +load_dotenv() +LOG_FILEPATH: str = os.getenv("LOGS_PATH") + '/Polymarket_User.log' + +# https://docs.polymarket.com/market-data/websocket/user-channel +WSS_URL = "wss://ws-subscriptions-clob.polymarket.com/ws/user" +API_CREDS = {} + +HIST_TRADES = np.empty((0, 2)) +TARGET_PX = 0 + +### Database Funcs ### +async def create_user_trades_table( + CON: AsyncContextManager, + engine: str = 'mysql', # mysql | duckdb + ) -> None: + if CON is None: + logging.info("NO DB CONNECTION, SKIPPING Create Statements") + else: + if engine == 'mysql': + logging.info('Creating Table if Does Not Exist: user_stream_trades') + await CON.execute(text(""" + CREATE TABLE IF NOT EXISTS user_stream_trades ( + -- event_type VARCHAR(8), + timestamp_arrival BIGINT, + type VARCHAR(20), + id VARCHAR(100), + taker_order_id VARCHAR(100), + market VARCHAR(100), + asset_id VARCHAR(100), + side VARCHAR(8), + size DOUBLE, + price DOUBLE, + fee_rate_bps DOUBLE, + status VARCHAR(20), + matchtime BIGINT, + last_update BIGINT, + outcome VARCHAR(20), + owner VARCHAR(100), + trade_owner VARCHAR(100), + maker_address VARCHAR(100), + transaction_hash VARCHAR(100), + bucket_index INT, + maker_orders JSON NULL, + trader_side VARCHAR(8), + timestamp BIGINT + ); + """)) + await CON.commit() + else: + raise ValueError('Only MySQL engine is implemented') + +async def insert_user_trades_table( + params: dict, + CON: AsyncContextManager, + engine: str = 'mysql', # mysql | duckdb + ) -> None: + if CON is None: + logging.info("NO DB CONNECTION, SKIPPING Insert Statements") + else: + if engine == 'mysql': + await CON.execute(text(""" + INSERT INTO user_stream_trades + ( + timestamp_arrival, + type, + id, + taker_order_id, + market, + asset_id, + side, + size, + price, + fee_rate_bps, + status, + matchtime, + last_update, + outcome, + owner, + trade_owner, + maker_address, + transaction_hash, + bucket_index, + maker_orders, + trader_side, + timestamp + ) + VALUES + ( + :timestamp_arrival, + :type, + :id, + :taker_order_id, + :market, + :asset_id, + :side, + :size, + :price, + :fee_rate_bps, + :status, + :matchtime, + :last_update, + :outcome, + :owner, + :trade_owner, + :maker_address, + :transaction_hash, + :bucket_index, + :maker_orders, + :trader_side, + :timestamp + ) + """), + parameters=params + ) + await CON.commit() + else: + raise ValueError('Only MySQL engine is implemented') + + +async def create_user_orders_table( + CON: AsyncContextManager, + engine: str = 'mysql', # mysql | duckdb + ) -> None: + if CON is None: + logging.info("NO DB CONNECTION, SKIPPING Create Statements") + else: + if engine == 'mysql': + logging.info('Creating Table if Does Not Exist: user_stream_orders') + await CON.execute(text(""" + CREATE TABLE IF NOT EXISTS user_stream_orders ( + -- event_type VARCHAR(8), + timestamp_arrival BIGINT, + id VARCHAR(100), + owner VARCHAR(100), + market VARCHAR(100), + asset_id VARCHAR(100), + side VARCHAR(8), + order_owner VARCHAR(100), + original_size DOUBLE, + size_matched DOUBLE, + price DOUBLE, + associate_trades JSON NULL, + outcome VARCHAR(20), + type VARCHAR(20), + created_at BIGINT, + expiration VARCHAR(20), + order_type VARCHAR(8), + status VARCHAR(20), + maker_address VARCHAR(100), + timestamp BIGINT + ); + """)) + await CON.commit() + else: + raise ValueError('Only MySQL engine is implemented') + +async def insert_user_orders_table( + params: dict, + CON: AsyncContextManager, + engine: str = 'mysql', # mysql | duckdb + ) -> None: + if CON is None: + logging.info("NO DB CONNECTION, SKIPPING Insert Statements") + else: + if engine == 'mysql': + await CON.execute(text(""" + INSERT INTO user_stream_orders + ( + timestamp_arrival, + id, + owner, + market, + asset_id, + side, + order_owner, + original_size, + size_matched, + price, + associate_trades, + outcome, + type, + created_at, + expiration, + order_type, + status, + maker_address, + timestamp + ) + VALUES + ( + :timestamp_arrival, + :id, + :owner, + :market, + :asset_id, + :side, + :order_owner, + :original_size, + :size_matched, + :price, + :associate_trades, + :outcome, + :type, + :created_at, + :expiration, + :order_type, + :status, + :maker_address, + :timestamp + ) + """), + parameters=params + ) + await CON.commit() + else: + raise ValueError('Only MySQL engine is implemented') + +### Helpers ### +def live_orders_only(orders: list[dict]) -> list[dict]: + return [d for d in orders if d.get('status') in ['LIVE','MATCHED']] + +def upsert_list_of_dicts_by_id(list_of_dicts, new_dict): + for index, item in enumerate(list_of_dicts): + if item.get('id') == new_dict.get('id'): + list_of_dicts[index] = new_dict + return list_of_dicts + + list_of_dicts.append(new_dict) + return list_of_dicts + + +async def polymarket_stream(): + global TARGET_PX + global HIST_TRADES + global LOCAL_LIVE_ORDERS + global LOCAL_RECENT_TRADES + + POLY_API_KEY = API_CREDS.api_key + POLY_API_SECRET = API_CREDS.api_secret + POLY_API_PASS = API_CREDS.api_passphrase + + async for websocket in websockets.connect(WSS_URL): + print(f"Connected to {WSS_URL}") + + subscribe_msg = { + "auth": { + "apiKey": POLY_API_KEY, + "secret": POLY_API_SECRET, + "passphrase": POLY_API_PASS, + }, + "type": "user", + "markets": [] + } + + await websocket.send(json.dumps(subscribe_msg)) + print("Subscribed to User Data") + + try: + async for message in websocket: + ts_arrival = round(datetime.now().timestamp()*1000) + if isinstance(message, str): + data = json.loads(message) + if data == {}: # Handle empty server ping - return pong + await websocket.send(json.dumps({})) + print('SENT HEARTBEAT PING') + continue + + data['timestamp_arrival'] = ts_arrival + event_type = data.get('event_type', None) + match event_type: + case 'trade': + # logging.info(f'TRADE: {data}') + # trade_status = data.get('status') + # match trade_status: # Raise TELEGRAM ALERT ??? + # case 'MATCHED': + # pass + # case 'MINED': + # pass + # case 'CONFIRMED': + # pass + # case 'RETRYING': + # pass + # case 'FAILED': + # pass + + ### Convert Datatypes ### + data['size'] = float(data['size']) + data['price'] = float(data['price']) + data['fee_rate_bps'] = float(data['fee_rate_bps']) + data['matchtime'] = int(data['match_time']) + data['last_update'] = int(data['last_update']) + data['timestamp'] = int(data['timestamp']) + data['maker_orders'] = json.dumps(data['maker_orders']) if data['maker_orders'] else None + + LOCAL_RECENT_TRADES = upsert_list_of_dicts_by_id(LOCAL_RECENT_TRADES, data) + LOOKBACK_MIN_TS_MS = ts_arrival-LOCAL_RECENT_TRADES_LOOKBACK_SEC*1000 + LOCAL_RECENT_TRADES = [t for t in LOCAL_RECENT_TRADES if t.get('timestamp_arrival', 0) >= LOOKBACK_MIN_TS_MS] + + + VAL_KEY_OBJ = json.dumps(LOCAL_RECENT_TRADES) + # VAL_KEY.publish(VK_CHANNEL, VAL_KEY_OBJ) + logging.info("----------LOCAL_RECENT_TRADES-VK-----------") + logging.info(VAL_KEY_OBJ) + logging.info("-------------------------------------------") + VAL_KEY.set(VK_RECENT_TRADES, VAL_KEY_OBJ) + + # logging.info(f'User Trade Update: {data}') + + ### Insert into DB ### + await insert_user_trades_table( + params=data, + CON=CON + ) + case 'order': + logging.info(f'ORDER: {data}') + ### Convert Datatypes ### + data['original_size'] = float(data['original_size']) + data['size_matched'] = float(data['size_matched']) + data['price'] = float(data['price']) + data['associate_trades'] = json.dumps(data['associate_trades']) if data['associate_trades'] else None + data['created_at'] = int(data['created_at']) + data['timestamp'] = int(data['timestamp']) + + ### Match on Status - Pass Live orders to Valkey for Algo Engine ### + order_status = data.get('status') + match order_status: + case 'live': + LOCAL_LIVE_ORDERS = upsert_list_of_dicts_by_id(LOCAL_LIVE_ORDERS, data) + LOCAL_LIVE_ORDERS = live_orders_only(LOCAL_LIVE_ORDERS) + VAL_KEY_OBJ = json.dumps(LOCAL_LIVE_ORDERS) + # VAL_KEY.publish(VK_CHANNEL, VAL_KEY_OBJ) + VAL_KEY.set(VK_LIVE_ORDERS, VAL_KEY_OBJ) + logging.info(f'Order(s) RESTING: {data}') + case 'matched': + logging.info(f'Order(s) MATCHED: {data}') + case 'delayed': + raise ValueError(f'Order Status of "delayed" which is not expected for non-sports orders: {data}') + case 'unmatched': + raise ValueError(f'Order Status of "unmatched" which is not expected for non-sports orders: {data}') + + ### Insert into DB ### + await insert_user_orders_table( + params=data, + CON=CON, + ) + else: + raise ValueError(f'Type: {type(data)} not expected: {message}') + + except websockets.ConnectionClosed as e: + print(f"Connection closed by server. Exception: {e}") + +async def main(): + global VAL_KEY + global CON + global API_CREDS + + private_key = os.getenv("PRIVATE_KEY") + host = "https://clob.polymarket.com" + chain_id = 137 # Polygon mainnet + + temp_client = ClobClient(host, key=private_key, chain_id=chain_id) + API_CREDS = temp_client.create_or_derive_api_creds() + + if USE_VK: + VAL_KEY = valkey.Valkey(host='localhost', port=6379, db=0) + # published_count = VAL_KEY.publish(VK_CHANNEL,f"Hola, starting to publish to valkey: {VK_CHANNEL} @ {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + # logging.info(f"Valkey message published to {published_count} subscribers of {VK_CHANNEL}") + else: + VAL_KEY = None + logging.warning("VALKEY NOT BEING USED, NO DATA WILL BE PUBLISHED") + + if USE_DB: + engine = create_async_engine('mysql+asyncmy://root:pwd@localhost/polymarket') + async with engine.connect() as CON: + await create_user_trades_table(CON=CON) + await create_user_orders_table(CON=CON) + await polymarket_stream() + else: + CON = None + logging.warning("DATABASE NOT BEING USED, NO DATA WILL BE RECORDED") + await polymarket_stream() + + +if __name__ == '__main__': + START_TIME = round(datetime.now().timestamp()*1000) + + logging.info(f'Log FilePath: {LOG_FILEPATH}') + + logging.basicConfig( + force=True, + filename=LOG_FILEPATH, + level=logging.INFO, + format='%(asctime)s - %(levelname)s - %(message)s', + filemode='w' + ) + logging.info(f"STARTED: {START_TIME}") + + try: + asyncio.run(main()) + except KeyboardInterrupt as e: + print(f"Stream stopped: {e}") \ No newline at end of file diff --git a/ws_user/Dockerfile b/ws_user/Dockerfile new file mode 100644 index 0000000..0753635 --- /dev/null +++ b/ws_user/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3.13-slim + +RUN apt-get update && \ + apt-get install -y build-essential + +RUN gcc --version +RUN rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt . + +RUN pip install --no-cache-dir -r requirements.txt + +COPY . . + +# Finally, run gunicorn. +CMD [ "python", "ws_user.py"] +# CMD [ "gunicorn", "--workers=5", "--threads=1", "-b 0.0.0.0:8000", "app:server"] \ No newline at end of file