stevekeyharvey/Polymarket
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

saving

Browse Source
This commit is contained in:
stevekeyharvey
2026-04-01 17:37:19 +00:00
parent 8d7d99d749
commit 259ea93479
21 changed files with 2900 additions and 436 deletions
Download Patch File Download Diff File Expand all files Collapse all files

400
database.ipynb
View File

@@ -14,7 +14,7 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 11,
"id": "f5040527",
"metadata": {},
"outputs": [
@@ -36,195 +36,236 @@
" print(f\"Connection failed: {e}\") "
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "5c23110d",
"metadata": {},
"outputs": [],
"source": [
"query = '''\n",
"SELECT timestamp_msg,timestamp_value,`value` FROM poly_rtds_cl_btcusd;\n",
"'''"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "a866e9ca",
"metadata": {},
"outputs": [],
"source": [
"df = pd.read_sql(query, con=engine)"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "954a3c3c",
"id": "72059b3f",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Connection successful\n"
]
}
],
"source": [
"### MYSQL ###\n",
"engine_inter_storage = create_engine('mysql+pymysql://root:pwd@100.84.226.40/polymarket')\n",
"try:\n",
" with engine.connect() as conn:\n",
" print(\"Connection successful\")\n",
"except Exception as e:\n",
" print(f\"Connection failed: {e}\") "
]
},
{
"cell_type": "code",
"execution_count": 48,
"id": "b723a51f",
"metadata": {},
"outputs": [],
"source": [
"df['ts'] = pd.to_datetime(df['timestamp_value'], unit='ms')"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "f11fd680",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"1774752413939"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"round(datetime.now().timestamp()*1000)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "eadb0364",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"Timestamp('2026-03-29 03:22:03.145000')"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"pd.to_datetime(1774754523145, unit='ms')"
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "3fbf525c",
"metadata": {},
"outputs": [],
"source": [
"d = {'id': '1753563', 'question': 'Bitcoin Up or Down - March 29, 12:40AM-12:45AM ET', 'conditionId': '0x4cf6815a61939a9a0ee308772e727bbaf8e95803577aad7b9a9d3e028e37f13b', 'slug': 'btc-updown-5m-1774759200', 'resolutionSource': 'https://data.chain.link/streams/btc-usd', 'endDate': '2026-03-29T04:45:00Z', 'liquidity': '40412.2984', 'startDate': '2026-03-28T04:48:26.823506Z', 'image': 'https://polymarket-upload.s3.us-east-2.amazonaws.com/BTC+fullsize.png', 'icon': 'https://polymarket-upload.s3.us-east-2.amazonaws.com/BTC+fullsize.png', 'description': 'This market will resolve to \"Up\" if the Bitcoin price at the end of the time range specified in the title is greater than or equal to the price at the beginning of that range. Otherwise, it will resolve to \"Down\".\\nThe resolution source for this market is information from Chainlink, specifically the BTC/USD data stream available at https://data.chain.link/streams/btc-usd.\\nPlease note that this market is about the price according to Chainlink data stream BTC/USD, not according to other sources or spot markets.', 'outcomes': '[\"Up\", \"Down\"]', 'outcomePrices': '[\"0.505\", \"0.495\"]', 'volume': '1154.594883', 'active': True, 'closed': False, 'marketMakerAddress': '', 'createdAt': '2026-03-28T04:47:13.404208Z', 'updatedAt': '2026-03-29T04:38:45.64501Z', 'new': False, 'featured': False, 'archived': False, 'restricted': True, 'groupItemThreshold': '0', 'questionID': '0x7e86ce5e7ba0eb24758756db7c1443bf48041d08ac10364cf6771ef3f9c26733', 'enableOrderBook': True, 'orderPriceMinTickSize': 0.01, 'orderMinSize': 5, 'volumeNum': 1154.594883, 'liquidityNum': 40412.2984, 'endDateIso': '2026-03-29', 'startDateIso': '2026-03-28', 'hasReviewedDates': True, 'volume24hr': 1154.594883, 'volume1wk': 1154.594883, 'volume1mo': 1154.594883, 'volume1yr': 1154.594883, 'clobTokenIds': '[\"3132608671382208432230794800974499111421928258370863811882545679011841068490\", \"6273455409880805876304408793123549040642604317110995252288748628688958052125\"]', 'volume24hrClob': 1154.594883, 'volume1wkClob': 1154.594883, 'volume1moClob': 1154.594883, 'volume1yrClob': 1154.594883, 'volumeClob': 1154.594883, 'liquidityClob': 40412.2984, 'makerBaseFee': 1000, 'takerBaseFee': 1000, 'acceptingOrders': True, 'negRisk': False, 'ready': False, 'funded': False, 'acceptingOrdersTimestamp': '2026-03-28T04:47:21Z', 'cyom': False, 'competitive': 0.9999750006249843, 'pagerDutyNotificationEnabled': False, 'approved': True, 'rewardsMinSize': 50, 'rewardsMaxSpread': 4.5, 'spread': 0.01, 'lastTradePrice': 0.51, 'bestBid': 0.5, 'bestAsk': 0.51, 'automaticallyActive': True, 'clearBookOnStart': False, 'showGmpSeries': False, 'showGmpOutcome': False, 'manualActivation': False, 'negRiskOther': False, 'umaResolutionStatuses': '[]', 'pendingDeployment': False, 'deploying': False, 'rfqEnabled': False, 'eventStartTime': '2026-03-29T04:40:00Z', 'holdingRewardsEnabled': False, 'feesEnabled': True, 'requiresTranslation': False, 'makerRebatesFeeShareBps': 10000, 'feeType': 'crypto_fees', 'feeSchedule': {'exponent': 2, 'rate': 0.25, 'takerOnly': True, 'rebateRate': 0.2}}"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "699031ee",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"{'id': '1753563',\n",
" 'question': 'Bitcoin Up or Down - March 29, 12:40AM-12:45AM ET',\n",
" 'conditionId': '0x4cf6815a61939a9a0ee308772e727bbaf8e95803577aad7b9a9d3e028e37f13b',\n",
" 'slug': 'btc-updown-5m-1774759200',\n",
" 'resolutionSource': 'https://data.chain.link/streams/btc-usd',\n",
" 'endDate': '2026-03-29T04:45:00Z',\n",
" 'liquidity': '40412.2984',\n",
" 'startDate': '2026-03-28T04:48:26.823506Z',\n",
" 'image': 'https://polymarket-upload.s3.us-east-2.amazonaws.com/BTC+fullsize.png',\n",
" 'icon': 'https://polymarket-upload.s3.us-east-2.amazonaws.com/BTC+fullsize.png',\n",
" 'description': 'This market will resolve to \"Up\" if the Bitcoin price at the end of the time range specified in the title is greater than or equal to the price at the beginning of that range. Otherwise, it will resolve to \"Down\".\\nThe resolution source for this market is information from Chainlink, specifically the BTC/USD data stream available at https://data.chain.link/streams/btc-usd.\\nPlease note that this market is about the price according to Chainlink data stream BTC/USD, not according to other sources or spot markets.',\n",
" 'outcomes': '[\"Up\", \"Down\"]',\n",
" 'outcomePrices': '[\"0.505\", \"0.495\"]',\n",
" 'volume': '1154.594883',\n",
" 'active': True,\n",
" 'closed': False,\n",
" 'marketMakerAddress': '',\n",
" 'createdAt': '2026-03-28T04:47:13.404208Z',\n",
" 'updatedAt': '2026-03-29T04:38:45.64501Z',\n",
" 'new': False,\n",
" 'featured': False,\n",
" 'archived': False,\n",
" 'restricted': True,\n",
" 'groupItemThreshold': '0',\n",
" 'questionID': '0x7e86ce5e7ba0eb24758756db7c1443bf48041d08ac10364cf6771ef3f9c26733',\n",
" 'enableOrderBook': True,\n",
" 'orderPriceMinTickSize': 0.01,\n",
" 'orderMinSize': 5,\n",
" 'volumeNum': 1154.594883,\n",
" 'liquidityNum': 40412.2984,\n",
" 'endDateIso': '2026-03-29',\n",
" 'startDateIso': '2026-03-28',\n",
" 'hasReviewedDates': True,\n",
" 'volume24hr': 1154.594883,\n",
" 'volume1wk': 1154.594883,\n",
" 'volume1mo': 1154.594883,\n",
" 'volume1yr': 1154.594883,\n",
" 'clobTokenIds': '[\"3132608671382208432230794800974499111421928258370863811882545679011841068490\", \"6273455409880805876304408793123549040642604317110995252288748628688958052125\"]',\n",
" 'volume24hrClob': 1154.594883,\n",
" 'volume1wkClob': 1154.594883,\n",
" 'volume1moClob': 1154.594883,\n",
" 'volume1yrClob': 1154.594883,\n",
" 'volumeClob': 1154.594883,\n",
" 'liquidityClob': 40412.2984,\n",
" 'makerBaseFee': 1000,\n",
" 'takerBaseFee': 1000,\n",
" 'acceptingOrders': True,\n",
" 'negRisk': False,\n",
" 'ready': False,\n",
" 'funded': False,\n",
" 'acceptingOrdersTimestamp': '2026-03-28T04:47:21Z',\n",
" 'cyom': False,\n",
" 'competitive': 0.9999750006249843,\n",
" 'pagerDutyNotificationEnabled': False,\n",
" 'approved': True,\n",
" 'rewardsMinSize': 50,\n",
" 'rewardsMaxSpread': 4.5,\n",
" 'spread': 0.01,\n",
" 'lastTradePrice': 0.51,\n",
" 'bestBid': 0.5,\n",
" 'bestAsk': 0.51,\n",
" 'automaticallyActive': True,\n",
" 'clearBookOnStart': False,\n",
" 'showGmpSeries': False,\n",
" 'showGmpOutcome': False,\n",
" 'manualActivation': False,\n",
" 'negRiskOther': False,\n",
" 'umaResolutionStatuses': '[]',\n",
" 'pendingDeployment': False,\n",
" 'deploying': False,\n",
" 'rfqEnabled': False,\n",
" 'eventStartTime': '2026-03-29T04:40:00Z',\n",
" 'holdingRewardsEnabled': False,\n",
" 'feesEnabled': True,\n",
" 'requiresTranslation': False,\n",
" 'makerRebatesFeeShareBps': 10000,\n",
" 'feeType': 'crypto_fees',\n",
" 'feeSchedule': {'exponent': 2,\n",
" 'rate': 0.25,\n",
" 'takerOnly': True,\n",
" 'rebateRate': 0.2}}"
]
},
"execution_count": 15,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"d"
"# with engine.connect() as conn:\n",
"# print(\"Connection successful\")\n",
"# sql = text(\"TRUNCATE TABLE coinbase_btcusd_trades;\")\n",
"# conn.execute(sql)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a620fa17",
"id": "5c23110d",
"metadata": {},
"outputs": [],
"source": [
"q_binance = '''\n",
"SELECT * FROM binance_btcusd_trades;\n",
"'''\n",
"q_coinbase = '''\n",
"SELECT * FROM coinbase_btcusd_trades;\n",
"'''\n",
"q_rtds = '''\n",
"SELECT * FROM poly_rtds_cl_btcusd;\n",
"'''\n",
"q_clob = '''\n",
"SELECT * FROM poly_btcusd_trades;\n",
"'''"
]
},
{
"cell_type": "code",
"execution_count": 24,
"id": "a866e9ca",
"metadata": {},
"outputs": [],
"source": [
"# df_binance = pd.read_sql(q_binance, con=engine)\n",
"df_coinbase = pd.read_sql(q_coinbase, con=engine)\n",
"df_rtds = pd.read_sql(q_rtds, con=engine)\n",
"df_clob = pd.read_sql(q_clob, con=engine)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "954a3c3c",
"metadata": {},
"outputs": [],
"source": [
"# df_binance['timestamp_arrival'] = pd.to_datetime(df_binance['timestamp_arrival'], unit='ms')\n",
"df_coinbase['timestamp_arrival'] = pd.to_datetime(df_coinbase['timestamp_arrival'], unit='ms')\n",
"df_rtds['timestamp_arrival'] = pd.to_datetime(df_rtds['timestamp_arrival'], unit='ms')\n",
"df_clob['timestamp_arrival'] = pd.to_datetime(df_clob['timestamp_arrival'], unit='ms')"
]
},
{
"cell_type": "code",
"execution_count": 57,
"id": "50c6339f",
"metadata": {},
"outputs": [],
"source": [
"def copy_table_data_btw_servers(df, table_name, engine_destination) -> None:\n",
" rows_imported = df.to_sql(name=table_name, con=engine_destination, if_exists='append')\n",
" if rows_imported == len(df):\n",
" print(f'SUCCESS: COPIED {rows_imported} to table \"{table_name}\" on INTERSERVER_STORAGE')\n",
" else:\n",
" raise ValueError(f'FAILED: COPIED {rows_imported} rows to table {table_name} on INTERSERVER_STORAGE; EXPECTED {len(df)}')\n",
" \n",
"def truncate_table(engine, table):\n",
" with engine.connect() as conn:\n",
" sql = text(f\"TRUNCATE TABLE {table};\")\n",
" conn.execute(sql)\n",
" conn.commit()"
]
},
{
"cell_type": "code",
"execution_count": 61,
"id": "d0399a96",
"metadata": {},
"outputs": [],
"source": [
"def backup_all_tables(engine_origin, engine_destination, tables_to_copy):\n",
" for t in tables_to_copy:\n",
" q = f'''\n",
" SELECT * FROM {t};\n",
" '''\n",
" df = pd.read_sql(q, con=engine_origin)\n",
" print('-------------------------------------------------------------------------')\n",
" print(f'Loaded Data for Table: {t}...Attempting to Transfer to Destination Server')\n",
" copy_table_data_btw_servers(\n",
" df=df,\n",
" table_name=t,\n",
" engine_destination=engine_destination,\n",
" )\n",
" print(f'Attempting to Truncate Table: {t}...')\n",
" \n",
" ### FOR REALTIME - instead of truncate, need to delete rows using a conditon (e.g. delete all rows <= max timestamp arrival in the DF)\n",
" \n",
" truncate_table(\n",
" engine=engine_origin,\n",
" table=t,\n",
" )\n",
" print(f'...Successfully Truncated Table: {t}')\n",
" print(f'Done Transferring Data for Table: {t}')\n",
" \n"
]
},
{
"cell_type": "code",
"execution_count": 59,
"id": "0de1629a",
"metadata": {},
"outputs": [],
"source": [
"tables_to_copy = [\n",
" # 'binance_btcusd_trades',\n",
" # 'coinbase_btcusd_trades',\n",
" 'poly_btcusd_trades',\n",
" 'poly_rtds_cl_btcusd',\n",
" # 'user_stream_orders',\n",
" # 'user_stream_trades',\n",
"]"
]
},
{
"cell_type": "code",
"execution_count": 60,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"-------------------------------------------------------------------------\n",
"Loaded Data for Table: poly_btcusd_trades...Attempting to Transfer to Destination Server\n",
"SUCCESS: COPIED 720568 to table \"poly_btcusd_trades\" on INTERSERVER_STORAGE\n",
"Attempting to Truncate Table: poly_btcusd_trades...\n",
"...Successfully Truncated Table: poly_btcusd_trades\n",
"Done Transferring Data for Table: poly_btcusd_trades\n",
"-------------------------------------------------------------------------\n",
"Loaded Data for Table: poly_rtds_cl_btcusd...Attempting to Transfer to Destination Server\n",
"SUCCESS: COPIED 73771 to table \"poly_rtds_cl_btcusd\" on INTERSERVER_STORAGE\n",
"Attempting to Truncate Table: poly_rtds_cl_btcusd...\n",
"...Successfully Truncated Table: poly_rtds_cl_btcusd\n",
"Done Transferring Data for Table: poly_rtds_cl_btcusd\n"
]
}
],
"source": [
"backup_all_tables(\n",
" engine_origin=engine,\n",
" engine_destination=engine_inter_storage,\n",
" tables_to_copy=tables_to_copy\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "cd0b40d2",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"SUCCESS COPIED 326007 to binance_btcusd_trades to INTERSERVER_STORAGE\n"
]
}
],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "48b47799",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "ad030f88",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "cafc5060",
"metadata": {},
"outputs": [],
"source": []
@@ -232,7 +273,6 @@
{
"cell_type": "code",
"execution_count": null,
"id": "2071f014",
"metadata": {},
"outputs": [],
"source": []