stevekeyharvey/Funding_Rate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

added extend symbol change for ws

Browse Source
This commit is contained in:
stevekeyharvey
2026-04-30 04:32:49 +00:00
parent dc3409ac40
commit 1ac0909c21
20 changed files with 28960 additions and 1221 deletions
Download Patch File Download Diff File Expand all files Collapse all files

159
ws_aster_user.py
View File

@@ -1,20 +1,19 @@
import asyncio
import json
import logging
import os
import socket
import traceback
from datetime import datetime
from typing import AsyncContextManager
import numpy as np
import pandas as pd
import requests.packages.urllib3.util.connection as urllib3_cn # type: ignore
from sqlalchemy import text
import websockets
from sqlalchemy.ext.asyncio import create_async_engine
import valkey
import os
import websockets
from dotenv import load_dotenv
from sqlalchemy.ext.asyncio import create_async_engine
import modules.aster_auth as aster_auth
import modules.aster_db as aster_db
import modules.db as db
@@ -28,84 +27,84 @@ urllib3_cn.allowed_gai_family = allowed_gai_family
### Database ###
USE_DB: bool = True
USE_VK: bool = True
VK_ORDERS_TRADES = 'fr_aster_user_orders'
VK_MARGIN_CALLS = 'fr_aster_user_margin_calls'
VK_BALANCES = 'fr_aster_user_balances'
VK_POSITIONS = 'fr_aster_user_positions'
CON: AsyncContextManager | None = None
VAL_KEY = None
CON: AsyncContextManager
VAL_KEY: valkey.Valkey
VK_ORDERS_TRADES: str = 'fr_aster_user_orders'
VK_MARGIN_CALLS: str = 'fr_aster_user_margin_calls'
VK_BALANCES: str = 'fr_aster_user_balances'
VK_POSITIONS: str = 'fr_aster_user_positions'
### Logging ###
load_dotenv()
LOG_FILEPATH: str = os.getenv("LOGS_PATH") + '/Fund_Rate_Aster_User.log'
LOG_FILEPATH: str = f'{os.getenv(key="LOGS_PATH")}/Fund_Rate_Aster_User.log'
### CONSTANTS ###
WSS_URL = "wss://fstream.asterdex.com/ws/"
LOCAL_RECENT_UPDATES_LOOKBACK_SEC = 30
WSS_URL: str = "wss://fstream.asterdex.com/ws/"
LOCAL_RECENT_UPDATES_LOOKBACK_SEC: int = 30
### Globals ###
LISTEN_KEY: str | None = None
LISTEN_KEY_LAST_UPDATE_TS_S: int = 0
LISTEN_KEY_PUT_INTERVAL_SEC = 1800
Listen_Key: str
Listen_Key_Last_Update_TS_S: int = 0
Listen_Key_Put_Interval_Sec: int = 1800
LOCAL_RECENT_ORDERS: list = []
LOCAL_RECENT_MARGIN_CALLS: list = []
LOCAL_RECENT_BALANCES: list = []
LOCAL_RECENT_POSITIONS: list = []
Local_Recent_Orders: list[dict] = []
Local_Recent_Margin_Calls: list[dict] = []
Local_Recent_Balances: list[dict] = []
Local_Recent_Positions: list[dict] = []
async def get_new_listen_key() -> str:
global LISTEN_KEY_LAST_UPDATE_TS_S
global Listen_Key_Last_Update_TS_S
listen_key_request = {
listen_key_request: dict = {
"url": "/fapi/v3/listenKey",
"method": "POST",
"params": {}
}
r = await aster_auth.post_authenticated_url(listen_key_request)
listen_key = r.get('listenKey', None)
r: dict = await aster_auth.post_authenticated_url(listen_key_request) # ty:ignore[invalid-assignment]
listen_key: str = r.get('listenKey', '')
print(f'LISTEN KEY: {listen_key}')
if listen_key is not None:
LISTEN_KEY_LAST_UPDATE_TS_S = round(datetime.now().timestamp())
if listen_key:
Listen_Key_Last_Update_TS_S = round(number=datetime.now().timestamp())
return listen_key
else:
raise ValueError(f'Listen Key is None; Failed to Update. response: {r}')
raise ValueError(f'Listen Key is empty; Failed to Update. response: {r}')
async def listen_key_interval():
global LISTEN_KEY
global Listen_Key
while True:
await asyncio.sleep(LISTEN_KEY_PUT_INTERVAL_SEC)
LISTEN_KEY = await get_new_listen_key()
await asyncio.sleep(delay=Listen_Key_Put_Interval_Sec)
Listen_Key = await get_new_listen_key()
### Websocket ###
async def ws_stream():
global LISTEN_KEY
global LOCAL_RECENT_ORDERS
global LOCAL_RECENT_MARGIN_CALLS
global LOCAL_RECENT_BALANCES
global LOCAL_RECENT_POSITIONS
global Listen_Key
global Local_Recent_Orders
global Local_Recent_Margin_Calls
global Local_Recent_Balances
global Local_Recent_Positions
LISTEN_KEY = await get_new_listen_key()
Listen_Key = await get_new_listen_key()
async for websocket in websockets.connect(WSS_URL+LISTEN_KEY):
logging.info(f"Connected to {WSS_URL}")
asyncio.create_task(listen_key_interval())
async for websocket in websockets.connect(uri=WSS_URL+Listen_Key, ping_interval=5):
logging.info(msg=f"Connected to {WSS_URL}")
asyncio.create_task(coro=listen_key_interval())
try:
async for message in websocket:
ts_arrival = round(datetime.now().timestamp()*1000)
ts_arrival: int = round(number=datetime.now().timestamp()*1000)
if isinstance(message, str):
try:
data = json.loads(message)
channel = data.get('e', None)
if channel is not None:
LOOKBACK_MIN_TS_MS = ts_arrival - (LOCAL_RECENT_UPDATES_LOOKBACK_SEC*1000)
data: dict = json.loads(s=message)
channel: str = data.get('e', '')
if channel:
lookback_min_ts_ms: int = ts_arrival - (LOCAL_RECENT_UPDATES_LOOKBACK_SEC*1000)
match channel:
case 'ORDER_TRADE_UPDATE':
# logging.info(f'ORDER_TRADE_UPDATE: {data}')
new_order_update = {
new_order_update: dict = {
'timestamp_arrival': ts_arrival,
'timestamp_msg': data['E'],
'timestamp_transaction': data['T'],
@@ -141,11 +140,11 @@ async def ws_stream():
'callback_rate': float(data['o'].get("cr", 0)), # :"5.0", // Callback Rate, only puhed with TRAILING_STOP_MARKET order
'realized_profit': float(data['o']["rp"]), # :"0" // Realized Profit of the trade
}
LOCAL_RECENT_ORDERS = utils.upsert_list_of_dicts_by_id(LOCAL_RECENT_ORDERS, new_order_update, id='order_id', seq_check_field='timestamp_msg')
LOCAL_RECENT_ORDERS = [t for t in LOCAL_RECENT_ORDERS if t.get('timestamp_arrival', 0) >= LOOKBACK_MIN_TS_MS]
Local_Recent_Orders = utils.upsert_list_of_dicts_by_id(Local_Recent_Orders, new_order_update, id='order_id', seq_check_field='timestamp_msg')
Local_Recent_Orders = [t for t in Local_Recent_Orders if t.get('timestamp_arrival', 0) >= lookback_min_ts_ms]
VAL_KEY_OBJ = json.dumps(LOCAL_RECENT_ORDERS)
VAL_KEY.set(VK_ORDERS_TRADES, VAL_KEY_OBJ)
VAL_KEY_OBJ: str = json.dumps(obj=Local_Recent_Orders)
VAL_KEY.set(name=VK_ORDERS_TRADES, value=VAL_KEY_OBJ)
await db.insert_df_to_mysql(table_name='fr_aster_user_order_trade', params=new_order_update, CON=CON)
continue
@@ -153,7 +152,7 @@ async def ws_stream():
# logging.info(f'MARGIN_CALL: {data}')
list_for_df = []
for p in list(data['p']):
margin_call_update = {
margin_call_update: dict = {
'timestamp_arrival': ts_arrival,
'timestamp_msg': data['E'],
'cross_wallet_balance': float(data.get('cw', 0)),
@@ -168,11 +167,11 @@ async def ws_stream():
'maint_margin_required': float(p["mm"]), # :"1.614445" // Maintenance Margin Required
}
list_for_df.append(margin_call_update)
LOCAL_RECENT_MARGIN_CALLS = utils.upsert_list_of_dicts_by_id(LOCAL_RECENT_MARGIN_CALLS, margin_call_update, id='symbol', seq_check_field='timestamp_msg')
LOCAL_RECENT_MARGIN_CALLS = [t for t in LOCAL_RECENT_MARGIN_CALLS if t.get('timestamp_arrival', 0) >= LOOKBACK_MIN_TS_MS]
Local_Recent_Margin_Calls = utils.upsert_list_of_dicts_by_id(Local_Recent_Margin_Calls, margin_call_update, id='symbol', seq_check_field='timestamp_msg')
Local_Recent_Margin_Calls = [t for t in Local_Recent_Margin_Calls if t.get('timestamp_arrival', 0) >= lookback_min_ts_ms]
VAL_KEY_OBJ = json.dumps(LOCAL_RECENT_MARGIN_CALLS)
VAL_KEY.set(VK_MARGIN_CALLS, VAL_KEY_OBJ)
VAL_KEY_OBJ: str = json.dumps(obj=Local_Recent_Margin_Calls)
VAL_KEY.set(name=VK_MARGIN_CALLS, value=VAL_KEY_OBJ)
await db.insert_df_to_mysql(table_name='fr_aster_user_margin', params=list_for_df, CON=CON)
continue
@@ -183,7 +182,7 @@ async def ws_stream():
### Balance Updates ###
if len(list(data['a']['B'])) > 0:
for b in list(data['a']['B']):
balance_update = {
balance_update: dict = {
'timestamp_arrival': ts_arrival,
'timestamp_msg': data['E'],
'timestamp_transaction': data['T'],
@@ -196,13 +195,13 @@ async def ws_stream():
'balance_change_excl_pnl_comms': float(b['bc']),
}
list_for_df_bal.append(balance_update)
LOCAL_RECENT_BALANCES = utils.upsert_list_of_dicts_by_id(LOCAL_RECENT_BALANCES, balance_update, id='asset', seq_check_field='timestamp_msg')
LOCAL_RECENT_BALANCES = [t for t in LOCAL_RECENT_BALANCES if t.get('timestamp_arrival', 0) >= LOOKBACK_MIN_TS_MS]
VAL_KEY.set(VK_BALANCES, json.dumps(LOCAL_RECENT_BALANCES))
Local_Recent_Balances = utils.upsert_list_of_dicts_by_id(Local_Recent_Balances, balance_update, id='asset', seq_check_field='timestamp_msg')
Local_Recent_Balances = [t for t in Local_Recent_Balances if t.get('timestamp_arrival', 0) >= lookback_min_ts_ms]
VAL_KEY.set(name=VK_BALANCES, value=json.dumps(obj=Local_Recent_Balances))
### Position Updates ###
if len(list(data['a']['P'])) > 0:
for p in list(data['a']['P']):
position_update = {
position_update: dict = {
'timestamp_arrival': ts_arrival,
'timestamp_msg': data['E'],
'timestamp_transaction': data['T'],
@@ -219,33 +218,34 @@ async def ws_stream():
'position_side': p['ps'],
}
list_for_df_pos.append(position_update)
LOCAL_RECENT_POSITIONS = utils.upsert_list_of_dicts_by_id(LOCAL_RECENT_POSITIONS, position_update, id='symbol', seq_check_field='timestamp_msg')
LOCAL_RECENT_POSITIONS = [t for t in LOCAL_RECENT_POSITIONS if t.get('timestamp_arrival', 0) >= LOOKBACK_MIN_TS_MS]
VAL_KEY.set(VK_POSITIONS, json.dumps(LOCAL_RECENT_POSITIONS))
Local_Recent_Positions = utils.upsert_list_of_dicts_by_id(Local_Recent_Positions, position_update, id='symbol', seq_check_field='timestamp_msg')
Local_Recent_Positions = [t for t in Local_Recent_Positions if t.get('timestamp_arrival', 0) >= lookback_min_ts_ms]
VAL_KEY.set(name=VK_POSITIONS, value=json.dumps(obj=Local_Recent_Positions))
if list_for_df_bal:
await db.insert_df_to_mysql(table_name='fr_aster_user_account_bal', params=list_for_df_bal, CON=CON)
if list_for_df_pos:
await db.insert_df_to_mysql(table_name='fr_aster_user_account_pos', params=list_for_df_pos, CON=CON)
continue
case 'listenKeyExpired':
raise('Listen Key Has Expired; Failed to Update Properly. Restarting.')
raise ValueError('Listen Key Has Expired; Failed to Update Properly. Restarting.')
case _:
logging.warning(f'UNMATCHED OTHER MSG: {data}')
logging.warning(msg=f'UNMATCHED OTHER MSG: {data}')
else:
logging.info(f'Initial or unexpected data struct, skipping: {data}')
logging.info(msg=f'Initial or unexpected data struct, skipping: {data}')
continue
except (json.JSONDecodeError, ValueError):
logging.warning(f'Message not in JSON format, skipping: {message}')
logging.warning(msg=f'Message not in JSON format, skipping: {message}')
continue
else:
raise ValueError(f'Type: {type(data)} not expected: {message}')
except websockets.ConnectionClosed as e:
logging.error(f'Connection closed: {e}')
logging.error(traceback.format_exc())
continue
logging.error(msg=f'Connection closed: {e}')
logging.error(msg=traceback.format_exc())
utils.send_tg_alert(msg=f'WS_Aster_User - Failure: {e}')
except Exception as e:
logging.error(f'Connection closed: {e}')
logging.error(traceback.format_exc())
logging.error(msg=f'Connection closed: {e}')
logging.error(msg=traceback.format_exc())
utils.send_tg_alert(msg=f'WS_Aster_User - Failure: {e}')
async def main():
@@ -255,8 +255,8 @@ async def main():
if USE_VK:
VAL_KEY = valkey.Valkey(host='localhost', port=6379, db=0)
else:
VAL_KEY = None
logging.warning("VALKEY NOT BEING USED, NO DATA WILL BE PUBLISHED")
raise NotImplementedError('Cannot run without Valkey')
if USE_DB:
engine = create_async_engine('mysql+asyncmy://root:pwd@localhost/fund_rate')
@@ -267,15 +267,14 @@ async def main():
await aster_db.create_fr_aster_user_account_pos(CON=CON)
await ws_stream()
else:
CON = None
logging.warning("DATABASE NOT BEING USED, NO DATA WILL BE RECORDED")
await ws_stream()
raise NotImplementedError('Cannot run without DB')
if __name__ == '__main__':
START_TIME = round(datetime.now().timestamp()*1000)
START_TIME: int = round(number=datetime.now().timestamp()*1000)
logging.info(f'Log FilePath: {LOG_FILEPATH}')
logging.info(msg=f'Log FilePath: {LOG_FILEPATH}')
logging.basicConfig(
force=True,
@@ -284,9 +283,9 @@ if __name__ == '__main__':
format='%(asctime)s - %(levelname)s - %(message)s',
filemode='w'
)
logging.info(f"STARTED: {START_TIME}")
logging.info(msg=f"STARTED: {START_TIME}")
try:
asyncio.run(main())
except KeyboardInterrupt:
logging.info("Stream stopped")
logging.info(msg="Stream stopped")