refactor valkey into objects with health check

.dockerignore

@@ -0,0 +1 @@
+/rust/

.gitignore

@@ -1 +1,9 @@
+# General
 .env
+
+# Python
+*.pyc
+
+# Rust
+/rust/
+Cargo.lock

_On_Ice/algo_config_backup.json

@@ -0,0 +1,25 @@
+{
+    "Updated_Timestamp": 1777433705326,
+    "Config": {
+        "Loop_Sleep_Sec": 0.0,
+        "Max_Order_Over_Notional_Ratio": 1.05,
+        "Max_Target_Notional": 0.0,
+        "Min_Time_To_Funding_Minutes": 60,
+        "Min_Fund_Rate_Pct_To_Trade": 0.0,
+        "Price_Worsener_Aster": 0.0,
+        "Price_Worsener_Extend": -0.1,
+        "Switch_To_Taker_Seconds": 1,
+        "Target_Open_Cash_Position": 10
+    },
+    "Logging": {
+        "Log_Summary_Each_Loop": false,
+        "Print_Summary_Each_Loop": false
+    },
+    "Overrides": {
+        "Allow_Ordering_Aster": true,
+        "Allow_Ordering_Extend": true,
+        "Flatten_Open_Positions": false,
+        "Flip_Side_For_Testing": false
+    }
+    
+}

_On_Ice/ws_aster_fund_rate_all.py

@@ -0,0 +1,105 @@
+import asyncio
+import json
+import logging
+import socket
+import traceback
+from datetime import datetime
+from typing import AsyncContextManager
+
+# import numpy as np
+# import pandas as pd
+import requests.packages.urllib3.util.connection as urllib3_cn  # type: ignore
+# from sqlalchemy import text
+import websockets
+# from sqlalchemy.ext.asyncio import create_async_engine
+import valkey
+import os
+from dotenv import load_dotenv
+import modules.utils as utils
+
+### Allow only ipv4 ###
+def allowed_gai_family():
+    return socket.AF_INET
+urllib3_cn.allowed_gai_family = allowed_gai_family
+
+### Database ###
+USE_DB: bool = False
+VK_CHANNEL = 'fund_rate_aster_all'
+
+CON: AsyncContextManager
+VAL_KEY: valkey.Valkey
+
+### Logging ###
+load_dotenv()
+LOG_FILEPATH: str = f'{os.getenv(key="LOGS_PATH")}/Fund_Rate_Aster_FR_ALL.log'
+
+### Globals ###
+WSS_URL: str = "wss://fstream.asterdex.com/ws/!markPrice@arr"
+
+
+### Websocket ###
+async def ws_stream():
+    async for websocket in websockets.connect(WSS_URL):
+        logging.info(msg=f"Connected to {WSS_URL}")
+        try:
+            async for message in websocket:
+                if isinstance(message, str):
+                    try: 
+                        data: dict = json.loads(message)
+                        if data:
+                            
+                            VAL_KEY.set(name=VK_CHANNEL, value=json.dumps(obj=data))
+                            # print(f'VK_SAVED: {len(data)}')
+                            continue
+                        else:
+                            logging.info(f'Initial or unexpected data struct, skipping: {data}')
+                            continue
+                    except (json.JSONDecodeError, ValueError):
+                        logging.warning(f'Message not in JSON format, skipping: {message}')
+                        continue
+                else:
+                    raise ValueError(f'Type: {type(data)}  not expected: {message}')
+        except websockets.ConnectionClosed as e:
+            logging.error(msg=f'Connection closed: {e}')
+            logging.error(msg=traceback.format_exc())
+            utils.send_tg_alert(msg=f'WS: {VK_CHANNEL} - Failure: {e}')
+        except Exception as e:
+            logging.error(msg=f'Connection closed: {e}')
+            logging.error(msg=traceback.format_exc())
+            utils.send_tg_alert(msg=f'WS: {VK_CHANNEL} - Failure: {e}')
+
+### Startup ###
+async def main():
+    global VAL_KEY
+    global CON
+    
+    VAL_KEY = valkey.Valkey(host='localhost', port=6379, db=0)
+    
+    if USE_DB:
+        raise NotImplementedError('DB not implemented for this ws.')
+        # engine = create_async_engine('mysql+asyncmy://root:pwd@localhost/fund_rate')
+        # async with engine.connect() as CON:
+            # await ws_stream()
+    else:
+        logging.warning("DATABASE NOT BEING USED, NO DATA WILL BE RECORDED")
+        await ws_stream()
+
+
+if __name__ == '__main__':
+    START_TIME: int = round(number=datetime.now().timestamp()*1000)
+
+    logging.info(msg=f'Log FilePath: {LOG_FILEPATH}')
+
+    logging.basicConfig(
+        force=True,
+        filename=LOG_FILEPATH,
+        level=logging.INFO,
+        format='%(asctime)s - %(levelname)s - %(message)s',
+        filemode='w'
+    )
+    logging.info(msg=f"STARTED: {START_TIME}")
+    
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        logging.info(msg="Stream stopped")

_On_Ice/ws_aster_fund_rate_all/.dockerignore

@@ -0,0 +1 @@
+../rust/

_On_Ice/ws_aster_fund_rate_all/Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.13-slim
+
+RUN apt-get update && \
+    apt-get install -y build-essential
+
+RUN gcc --version
+RUN rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY requirements.txt .
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+# Finally, run gunicorn.
+CMD [ "python", "ws_aster_fund_rate_all.py"]
+# CMD [ "gunicorn", "--workers=5", "--threads=1", "-b 0.0.0.0:8000", "app:server"]

_On_Ice/ws_extended_fund_rate_all.py

@@ -0,0 +1,132 @@
+import asyncio
+import json
+import logging
+import socket
+import traceback
+from datetime import datetime, timezone
+from typing import AsyncContextManager
+import math
+
+import numpy as np
+import pandas as pd
+import requests.packages.urllib3.util.connection as urllib3_cn  # type: ignore
+from sqlalchemy import text
+import websockets
+from sqlalchemy.ext.asyncio import create_async_engine
+import valkey
+import os
+from dotenv import load_dotenv
+import modules.utils as utils
+
+### Allow only ipv4 ###
+def allowed_gai_family():
+    return socket.AF_INET
+urllib3_cn.allowed_gai_family = allowed_gai_family
+
+### Database ###
+USE_DB: bool = False
+USE_VK: bool = True
+# VK_FUND_RATE = 'fund_rate_extended'
+VK_FUND_RATE_ALL = 'fund_rate_extended_all'
+
+CON: AsyncContextManager
+VAL_KEY: valkey.Valkey
+
+### Logging ###
+load_dotenv()
+LOG_FILEPATH: str = f'{os.getenv("LOGS_PATH")}/Fund_Rate_Extended_FR_ALL.log'
+
+### Globals ###
+WSS_URL = "wss://api.starknet.extended.exchange/stream.extended.exchange/v1/funding/"
+LOCAL_FUNDING_RATES = []
+
+def time_round_down(dt, interval_mins=5) -> int: # returns timestamp in seconds
+    interval_secs = interval_mins * 60
+    seconds = dt.timestamp()
+    rounded_seconds = math.floor(seconds / interval_secs) * interval_secs
+    
+    return rounded_seconds
+
+### Websocket ###
+async def ws_stream():
+    global LOCAL_FUNDING_RATES
+
+    async for websocket in websockets.connect(WSS_URL):
+        logging.info(f"Connected to {WSS_URL}")
+        try:
+            async for message in websocket:
+                ts_arrival = round(datetime.now().timestamp()*1000)
+                if isinstance(message, str):
+                    try: 
+                        data = json.loads(message)
+                        if data.get('data', None) is not None:
+                            # print(f'FR: {data}')
+                            # fr_next_update_ts = (time_round_down(dt=datetime.now(timezone.utc), interval_mins=60)+(60*60))*1000
+                            fr_update = {
+                                'sequence_id': data['seq'],
+                                'timestamp_arrival': ts_arrival,
+                                'timestamp_msg': data['ts'],
+                                'symbol': data['data']['m'],
+                                'funding_rate': float(data['data']['f']),
+                                'funding_rate_updated_ts_ms': data['data']['T'],
+                                # 'next_funding_time_ts_ms': fr_next_update_ts,
+                            }
+                            LOCAL_FUNDING_RATES = utils.upsert_list_of_dicts_by_id(LOCAL_FUNDING_RATES, fr_update, id='symbol', seq_check_field=None)
+                            VAL_KEY.set(VK_FUND_RATE_ALL, json.dumps(LOCAL_FUNDING_RATES))
+                            # print(f'VK_SAVED: {data}')
+                            continue
+                        else:
+                            logging.info(f'Initial or unexpected data struct, skipping: {data}')
+                            continue
+                    except (json.JSONDecodeError, ValueError):
+                        logging.warning(f'Message not in JSON format, skipping: {message}')
+                        continue
+                else:
+                    raise ValueError(f'Type: {type(data)}  not expected: {message}')
+        except websockets.ConnectionClosed as e:
+            logging.error(f'Connection closed: {e}')
+            logging.error(traceback.format_exc())
+            continue
+        except Exception as e:
+            logging.error(f'Connection closed: {e}')
+            logging.error(traceback.format_exc())
+
+### Startup ###
+async def main():
+    global VAL_KEY
+    global CON
+    
+    if USE_VK:
+        VAL_KEY = valkey.Valkey(host='localhost', port=6379, db=0)
+    else:
+        logging.warning("VALKEY NOT BEING USED, NO DATA WILL BE PUBLISHED")
+        raise NotImplementedError('Cannot run without VK')
+
+    if USE_DB:
+        engine = create_async_engine('mysql+asyncmy://root:pwd@localhost/fund_rate')
+        async with engine.connect() as CON:
+            # await create_rtds_btcusd_table(CON=CON)
+            await ws_stream()
+    else:
+        logging.warning("DATABASE NOT BEING USED, NO DATA WILL BE RECORDED")
+        raise NotImplementedError('DB not implemented')
+
+
+if __name__ == '__main__':
+    START_TIME = round(datetime.now().timestamp()*1000)
+
+    logging.info(f'Log FilePath: {LOG_FILEPATH}')
+
+    logging.basicConfig(
+        force=True,
+        filename=LOG_FILEPATH,
+        level=logging.INFO,
+        format='%(asctime)s - %(levelname)s - %(message)s',
+        filemode='w'
+    )
+    logging.info(f"STARTED: {START_TIME}")
+    
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        logging.info("Stream stopped")

_On_Ice/ws_extended_fund_rate_all/.dockerignore

@@ -0,0 +1 @@
+../rust/

_On_Ice/ws_extended_fund_rate_all/Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.13-slim
+
+RUN apt-get update && \
+    apt-get install -y build-essential
+
+RUN gcc --version
+RUN rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY requirements.txt .
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+# Finally, run gunicorn.
+CMD [ "python", "ws_extended_fund_rate_all.py"]
+# CMD [ "gunicorn", "--workers=5", "--threads=1", "-b 0.0.0.0:8000", "app:server"]

algo/.dockerignore

@@ -0,0 +1 @@
+../rust/

algo/Dockerfile

@@ -15,5 +15,5 @@ RUN pip install --no-cache-dir -r requirements.txt
 COPY . .
 
 # Finally, run gunicorn.
-CMD [ "python", "-u" ,"main.py"]
+CMD [ "python", "main.py"]
 # CMD [ "gunicorn", "--workers=5", "--threads=1", "-b 0.0.0.0:8000", "app:server"]

algo_config.json

@@ -1,12 +1,26 @@
 {
-    "Config_Updated_Timestamp": 1777098091913,
+    "Updated_Timestamp": 1778798867547,
-    "Allow_Ordering_Aster": true,
+    "Config": {
-    "Allow_Ordering_Extend": true,
+        "Loop_Sleep_Sec": 0.0,
-    "Loop_Sleep_Sec": 1,
+        "Max_Order_Over_Notional_Ratio": 1.5,
-    "Max_Target_Notional": 0.00,
+        "Max_Target_Notional": 0.0,
-    "Min_Time_To_Funding_Minutes": 60,
+        "Min_Time_To_Funding_Minutes": 57,
-    "Price_Worsener_Aster": 0.0,
+        "Min_Fund_Rate_Pct_To_Trade": 0.0,
-    "Price_Worsener_Extend": 0.0,
+        "Price_Worsener_Aster": 0,
-    "Target_Open_Cash_Position": 10,
+        "Price_Worsener_Extend": -1,
-    "Print_Summary_Each_Loop" : false
+        "Switch_To_Taker_Seconds": 3,
+        "Target_Open_Cash_Position": 10
+    },
+    "Logging": {
+        "Log_Summary_Each_Loop": false,
+        "Print_Summary_Each_Loop": false
+    },
+    "Overrides": {
+        "Allow_Ordering_Aster": true,
+        "Allow_Ordering_Extend": true,
+        "Allow_Symbol_Change": true,
+        "Flatten_Open_Positions": false,
+        "Flatten_Open_Positions_Opportunistic": false,
+        "Flip_Side_For_Testing": false
+    }
 }

algo_orchestrator.py

@@ -4,11 +4,13 @@ import logging
 import os
 import traceback
 from datetime import datetime
-from typing import AsyncContextManager
 
 import valkey
 from dotenv import load_dotenv
-# from sqlalchemy.ext.asyncio import create_async_engine
+import modules.utils as utils
+import modules.structs as structs
+from pathlib import Path
+
 
 '''
 TO DO:
@@ -16,72 +18,76 @@ TO DO:
 '''
 
 ### Database ###
-CON: AsyncContextManager | None = None
+VK_IN: str = 'fr_orchestrator_input'
-VAL_KEY = None
+VK_OUT: str = 'fr_orchestrator_output'
-VK_IN = 'fr_orchestrator_input'
-VK_OUT = 'fr_orchestrator_output'
 
 ### Logging ###
 load_dotenv()
-LOG_FILEPATH: str = os.getenv("LOGS_PATH") + '/Fund_Rate_Algo_Orchestrator.log'
+LOG_FILEPATH: str = f'{os.getenv("LOGS_PATH")}/Fund_Rate_Algo_Orchestrator.log'
 
-ALGO_CONFIG: None | dict
+async def main() -> None:
-# ALGO_CONFIG: None | Algo_Config = None
+    VAL_KEY: valkey.Valkey = valkey.Valkey(host='localhost', port=6379, db=0, decode_responses=True)
+    CONFIG_FILEPATH: str = '/algo_local_drive/algo_config.json'
+    if not Path(CONFIG_FILEPATH).exists():
+        CONFIG_FILEPATH: str = 'algo_config.json'
 
-async def orchestrator() -> None:
+    # Init Load Config File
-    global ALGO_CONFIG
+    with open(file=CONFIG_FILEPATH, mode='r', encoding='utf-8') as f: 
+        Algo_Config: dict = json.load(fp=f)
+        Algo_Config['Updated_Timestamp'] = round(number=datetime.now().timestamp()*1000)
+        
+        # vk = structs.VK_Obj(vk_name = 'fr_orchestrator_output', data=Algo_Config)
+        vk = structs.VK_Orchestrator_Out()
+        await vk.set(VK_CON=VAL_KEY)
     
     try:
-        VK_PUBSUB = VAL_KEY.pubsub()
+        VK_PUBSUB: valkey.client.PubSub = VAL_KEY.pubsub()
         VK_PUBSUB.subscribe(VK_IN)
         
-        print(f"Subscribed to '{VK_IN}'. Waiting for messages...")
+        logging.info(msg=f"Subscribed to '{VK_IN}'. Waiting for messages...")
 
         for message in VK_PUBSUB.listen():
             if message['type'] == 'message':
-                timestamp = round(datetime.now().timestamp()*1000)
+                timestamp: int = round(number=datetime.now().timestamp()*1000)
-                data = json.loads(message['data'])
-                # channel = message['channel']
                 
-                for k, v in data.items():
+                # Receive Update Msg from PubSub
-                    if ALGO_CONFIG.get(k, None) is not None:
+                data: dict = json.loads(s=message['data']) 
-                        ALGO_CONFIG[k] = v
 
-                ALGO_CONFIG['Config_Updated_Timestamp'] = timestamp
+                # Load Config File
-                VAL_KEY.set(VK_OUT, json.dumps(ALGO_CONFIG))
+                with open(file=CONFIG_FILEPATH, mode='r', encoding='utf-8') as f: 
-                with open('algo_config.json', 'w', encoding='utf-8') as f:
+                    Algo_Config: dict = json.load(fp=f)
-                    json.dump(ALGO_CONFIG, f, indent=4)
+                    Algo_Config['Updated_Timestamp'] = timestamp
-                print(f"Algo Config Updated @ {timestamp}; {data}")
+
+                if not Algo_Config:
+                    raise ValueError(f'Algo Orchestrator, config is none: {Algo_Config}')
+
+                # Update Config w Update Data
+                Algo_Config: dict = utils.rec_set_dict(orig_dict=Algo_Config, new_dict=data)
+                # Set VK KV w Updated Config
+                
+                # vk = structs.VK_Obj(vk_name = 'fr_orchestrator_output', data=Algo_Config)
+                vk = structs.VK_Orchestrator_Out()
+                await vk.set(VK_CON=VAL_KEY)
+
+                # Save Updated Config to File
+                with open(file=CONFIG_FILEPATH, mode='w', encoding='utf-8') as f: 
+                    json.dump(obj=Algo_Config, fp=f, indent=4)
+                
+                logging.info(msg=f"Algo Config Updated @ {timestamp}; {data}")
 
     except valkey.exceptions.ConnectionError as e:
-        print(f"Could not connect to Valkey. Please check the publish server is up; {e}")
+        logging.info(msg=f"Could not connect to Valkey. Please check the publish server is up; {e}")
     except KeyboardInterrupt:
-        logging.info('ORCHESTRATOR SHUTTING DOWN...')
+        logging.info(msg='ORCHESTRATOR SHUTTING DOWN...')
     except Exception as e:
-        logging.error(traceback.format_exc())
+        logging.error(msg=traceback.format_exc())
-        logging.critical(f'*** ORCHESTRATOR CRASHED: {e}')
+        logging.critical(msg=f'*** ORCHESTRATOR CRASHED: {e}')
 
-### MAIN STARTUP ###
-async def main() -> None:
-    global VAL_KEY
-    global CON
-    global ALGO_CONFIG
-
-    VAL_KEY = valkey.Valkey(host='localhost', port=6379, db=0, decode_responses=True)
-    # engine = create_async_engine('mysql+asyncmy://root:pwd@localhost/fund_rate')
-
-    with open('algo_config.json', 'r', encoding='utf-8') as f:
-        # ALGO_CONFIG = json.load(f, object_hook=lambda d: Algo_Config(**d))
-        ALGO_CONFIG = json.load(f)
-        ALGO_CONFIG['Config_Updated_Timestamp'] = round(datetime.now().timestamp()*1000)
-
-    # async with engine.connect() as CON:
-    await orchestrator()
 
 if __name__ == '__main__':
-    START_TIME = round(datetime.now().timestamp()*1000)
+    START_TIME: int = round(number=datetime.now().timestamp()*1000)
 
-    logging.info(f'Log FilePath: {LOG_FILEPATH}')
+    logging.info(msg=f'Log FilePath: {LOG_FILEPATH}')
 
     logging.basicConfig(
         force=True,
@@ -90,6 +96,6 @@ if __name__ == '__main__':
         format='%(asctime)s - %(levelname)s - %(message)s',
         filemode='w'
     )
-    logging.info(f"STARTED: {START_TIME}")
+    logging.info(msg=f"STARTED: {START_TIME}")
 
     asyncio.run(main())

algo_orchestrator/.dockerignore

@@ -0,0 +1 @@
+../rust/

algo_orchestrator/Dockerfile

@@ -15,5 +15,5 @@ RUN pip install --no-cache-dir -r requirements.txt
 COPY . .
 
 # Finally, run gunicorn.
-CMD [ "python", "-u" ,"algo_orchestrator.py"]
+CMD [ "python", "algo_orchestrator.py"]
 # CMD [ "gunicorn", "--workers=5", "--threads=1", "-b 0.0.0.0:8000", "app:server"]

docker-compose-algo.yml

@@ -0,0 +1,15 @@
+# tail -f Fund_Rate_Algo.log
+# docker compose -f docker-compose-algo.yml up --build
+
+services:
+  algo:
+    container_name: algo
+    restart: "no"
+    build:
+      context: ./
+      dockerfile: ./algo/Dockerfile
+    volumes:
+      - /root/data:/root/data:rw # Read-write access to data
+      - /root/logs:/root/logs:rw # Read-write access to data
+      - ./:/algo_local_drive:rw # Read-write access to data
+    network_mode: "host"

docker-compose.yml

@@ -1,92 +1,114 @@
-# tail -f Fund_Rate_Algo.log Fund_Rate_Aster_User.log Fund_Rate_Aster.log Fund_Rate_Extended_FR.log Fund_Rate_Extended_OB.log Fund_Rate_Extended_User.log
+# tail -f Fund_Rate_Algo.log Fund_Rate_Engine_BFR.log Fund_Rate_Algo_Orchestrator.log Fund_Rate_Aster_User.log Fund_Rate_Aster.log Fund_Rate_Extended_FR.log Fund_Rate_Extended_OB.log Fund_Rate_Extended_User.log
 
 services:
-  algo:
+  # algo:
-    container_name: algo
+  #   container_name: algo
-    restart: "no"
+  #   restart: "no"
-    build:
+  #   build:
-      context: ./
+  #     context: ./
-      dockerfile: ./algo/Dockerfile
+  #     dockerfile: ./algo/Dockerfile
-    depends_on:
+  #   depends_on:
-      - algo_orchestrator
+  #     - algo_orchestrator
-      - ws_aster
+  #     - engine_best_funding_rate
-      - ws_aster_user
+  #     - ws_aster
-      - ws_extended_fund_rate
+  #     - ws_aster_user
-      - ws_extended_orderbook
+  #     - ws_extended_fund_rate
-      - ws_extended_user
+  #     - ws_extended_orderbook
-    volumes:
+  #     - ws_extended_user
-      - /home/ubuntu/data:/home/ubuntu/data:rw # Read-write access to data
+  #   volumes:
-      - /home/ubuntu/logs:/home/ubuntu/logs:rw # Read-write access to data
+  #     - /root/data:/root/data:rw # Read-write access to data
-    network_mode: "host"
+  #     - /root/logs:/root/logs:rw # Read-write access to data
+  #   network_mode: "host"
   algo_orchestrator:
     container_name: algo_orchestrator
-    restart: "unless-stopped"
+    restart: "no"
     build:
       context: ./
       dockerfile: ./algo_orchestrator/Dockerfile
     volumes:
-      - /home/ubuntu/data:/home/ubuntu/data:rw # Read-write access to data
+      - /root/data:/root/data:rw # Read-write access to data
-      - /home/ubuntu/logs:/home/ubuntu/logs:rw # Read-write access to data
+      - /root/logs:/root/logs:rw # Read-write access to data
+      - ./:/algo_local_drive:rw # Read-write access to data
+    network_mode: "host"
+  engine_best_funding_rate:
+    container_name: engine_best_funding_rate
+    restart: "no"
+    build:
+      context: ./
+      dockerfile: ./engine_best_funding_rate/Dockerfile
+    volumes:
+      - /root/data:/root/data:rw # Read-write access to data
+      - /root/logs:/root/logs:rw # Read-write access to data
     network_mode: "host"
   ws_aster:
     container_name: ws_aster
-    restart: "unless-stopped"
+    restart: "no"
     build:
       context: ./
       dockerfile: ./ws_aster/Dockerfile
     volumes:
-      - /home/ubuntu/data:/home/ubuntu/data:rw # Read-write access to data
+      - /root/data:/root/data:rw # Read-write access to data
-      - /home/ubuntu/logs:/home/ubuntu/logs:rw # Read-write access to data
+      - /root/logs:/root/logs:rw # Read-write access to data
     network_mode: "host"
   ws_aster_user:
     container_name: ws_aster_user
-    restart: "unless-stopped"
+    restart: "no"
     build:
       context: ./
       dockerfile: ./ws_aster_user/Dockerfile
     volumes:
-      - /home/ubuntu/data:/home/ubuntu/data:rw # Read-write access to data
+      - /root/data:/root/data:rw # Read-write access to data
-      - /home/ubuntu/logs:/home/ubuntu/logs:rw # Read-write access to data
+      - /root/logs:/root/logs:rw # Read-write access to data
     network_mode: "host"
   ws_extended_fund_rate:
     container_name: ws_extended_fund_rate
-    restart: "unless-stopped"
+    restart: "no"
     build:
       context: ./
       dockerfile: ./ws_extended_fund_rate/Dockerfile
     volumes:
-      - /home/ubuntu/data:/home/ubuntu/data:rw # Read-write access to data
+      - /root/data:/root/data:rw # Read-write access to data
-      - /home/ubuntu/logs:/home/ubuntu/logs:rw # Read-write access to data
+      - /root/logs:/root/logs:rw # Read-write access to data
     network_mode: "host"
   ws_extended_orderbook:
     container_name: ws_extended_orderbook
-    restart: "unless-stopped"
+    restart: "no"
     build:
       context: ./ 
       dockerfile: ./ws_extended_orderbook/Dockerfile
     volumes:
-      - /home/ubuntu/data:/home/ubuntu/data:rw # Read-write access to dataw
+      - /root/data:/root/data:rw # Read-write access to dataw
-      - /home/ubuntu/logs:/home/ubuntu/logs:rw # Read-write access to data
+      - /root/logs:/root/logs:rw # Read-write access to data
     network_mode: "host"
   ws_extended_user:
     container_name: ws_extended_user
-    restart: "unless-stopped"
+    restart: "no"
     build:
       context: ./
       dockerfile: ./ws_extended_user/Dockerfile
     volumes:
-      - /home/ubuntu/data:/home/ubuntu/data:rw # Read-write access to data
+      - /root/data:/root/data:rw # Read-write access to data
-      - /home/ubuntu/logs:/home/ubuntu/logs:rw # Read-write access to data
+      - /root/logs:/root/logs:rw # Read-write access to data
     network_mode: "host"
+  # ws_extended_trades:
+  #   container_name: ws_extended_trades
+  #   restart: "no"
+  #   build:
+  #     context: ./ 
+  #     dockerfile: ./ws_extended_trades/Dockerfile
+  #   volumes:
+  #     - /root/data:/root/data:rw # Read-write access to dataw
+  #     - /root/logs:/root/logs:rw # Read-write access to data
+  #   network_mode: "host"
   
 
   # ng:
   #   container_name: ng
-  #   restart: "unless-stopped"
+  #   restart: "no"
   #   build:
   #     context: ./
   #     dockerfile: ./ng/Dockerfile
   #   volumes:
-  #     - /home/ubuntu/data:/home/ubuntu/data:rw # Read-write access to data
+  #     - /root/data:/root/data:rw # Read-write access to data
-  #     - /home/ubuntu/logs:/home/ubuntu/logs:rw # Read-write access to data
+  #     - /root/logs:/root/logs:rw # Read-write access to data
   #   network_mode: "host"

engine_best_funding_rate.py

@@ -0,0 +1,320 @@
+import asyncio
+import json
+import logging
+import os
+import time
+import traceback
+from dataclasses import asdict
+from datetime import datetime
+from typing import AsyncContextManager
+import modules.structs as structs
+import pandas as pd
+import requests
+import valkey
+from dotenv import load_dotenv
+import modules.manual_leverage as leverage
+import modules.aster_auth as aster_auth
+import modules.utils as utils
+
+### MANUAL LEVERAGE DATA ###
+df_leverage_by_exch = pd.DataFrame(data=leverage.LEVERAGE_BY_EXCH)
+
+### Database ###
+# CON: AsyncContextManager | None = None
+VAL_KEY: valkey.Valkey
+
+### Logging ###
+load_dotenv()
+LOG_FILEPATH: str = f'{os.getenv(key="LOGS_PATH")}/Fund_Rate_Engine_BFR.log'
+
+### CONSTANTS ###
+LOOP_SLEEP_SEC: int = 5
+REFRESH_MKT_INFO_EVERY_SEC: int = 90
+REFRESH_MKT_VOLUME_EVERY_SEC: int = 30
+
+MINUTES_LOOKBACK: int = 60
+
+### GLOBALS ###
+Mkt_Info_Last_Refresh_TS_ms: int = 0 
+Mkt_Volume_Last_Refresh_TS_ms: int = 0
+
+### TODO: score by volume, how long since last trade?, volatility, volume by time of day (active or dormant period?), funding rate consistency (% one side last 24hrs and from active close to active open periods). trade cost estimate?, max tradeable notional.
+### TODO: figure out what is max percent of volume i can trade - TCA kinda? what is ideal slice size?
+### TODO: Redesign so Algo allocates across the best markets with a waterfall method until at target collateral usage. order waterfall by score above^^
+### TODO: NG display grid of markets sorted by above score. top left is control panel, top right is graph (goes to mkt you click on from table) (maybe tabs for different graph views/groups, e.g. PnL total or all mkts percent to liquidate, pov by market etc.) middle bottom is markets table (tabs for open orders, open positions, pnl)
+
+### Funcs - Load Data ###
+async def get_extended_markets_info() -> pd.DataFrame:
+    r: dict = json.loads(s=requests.get(url='https://api.starknet.extended.exchange/api/v1/info/markets').text)
+    
+    df: pd.DataFrame = pd.DataFrame(data=r['data'])
+    df['funding_rate'] = df['marketStats'].apply(lambda x: x.get('fundingRate',{}))
+    df['funding_rate_ts'] = df['marketStats'].apply(lambda x: x.get('nextFundingRate',{}))
+    df['daily_volume'] = df['marketStats'].apply(lambda x: x.get('dailyVolume',{})).astype(float)
+    df['min_order_size'] = df['tradingConfig'].apply(lambda x: x.get('minOrderSize',{}))
+    df['min_price'] = df['tradingConfig'].apply(lambda x: x.get('minPriceChange',{}))
+    df['min_notional'] = 0
+    df['min_lot_size'] = df['tradingConfig'].apply(lambda x: x.get('minOrderSizeChange',{}))
+    df['max_leverage'] = df['tradingConfig'].apply(lambda x: x.get('maxLeverage',{}))
+
+    print('Extend markets info refreshed successfully')
+
+    return df
+
+async def get_aster_exch_info() -> pd.DataFrame:
+    ### ASTER EXCHANGE INFO ###
+    fut_acct_exchangeInfo: dict = {
+        "url": "/fapi/v3/exchangeInfo",
+        "method": "GET",
+        "params": {}
+    }
+    r: dict = await aster_auth.post_authenticated_url(fut_acct_exchangeInfo)  # ty:ignore[invalid-assignment]
+    df = pd.DataFrame(r['symbols'])
+    df['min_order_size'] = df['filters'].apply(lambda x: [f for f in x if f.get('filterType', None) == 'LOT_SIZE'][0]['minQty'] )
+    df['min_price'] = df['filters'].apply(lambda x: [f for f in x if f.get('filterType', None) == 'PRICE_FILTER'][0]['minPrice'] )
+    df['min_notional'] = df['filters'].apply(lambda x: [f for f in x if f.get('filterType', None) == 'MIN_NOTIONAL'][0]['notional'] )
+    df['min_lot_size'] = df['filters'].apply(lambda x: [f for f in x if f.get('filterType', None) == 'LOT_SIZE'][0]['stepSize'] )
+
+    fut_acct_ticker_stats: dict = {
+        "url": "/fapi/v3/ticker/24hr",
+        "method": "GET",
+        "params": {}
+    }
+    r: dict = await aster_auth.post_authenticated_url(fut_acct_ticker_stats)  # ty:ignore[invalid-assignment]
+    df_stats = pd.DataFrame(r)
+    df_stats['last_trade_ts_ast'] = df_stats['closeTime']
+
+    df = df.merge(df_stats[['symbol','quoteVolume','last_trade_ts_ast']].rename({'quoteVolume':'daily_volume'}, axis=1), on='symbol', how='left')
+    df['daily_volume'] = df['daily_volume'].astype(float)
+
+
+    print('Aster markets info refreshed successfully')
+
+    return df
+
+def load_aster_current_fr(df_aster_exch_info: pd.DataFrame) -> pd.DataFrame:
+    vk_get: str = VAL_KEY.get(name='fund_rate_aster_all')  # ty:ignore[invalid-assignment]
+    if not vk_get:
+        raise ValueError(f'fund_rate_aster_all is empty: {vk_get}')
+    df = pd.DataFrame(data=json.loads(vk_get))
+    
+    df: pd.DataFrame = df[['s','E','r','T']].rename({'s':'symbol','E':'funding_rate_updated_ts_ms','r':'funding_rate','T':'next_funding_ts'}, axis=1)
+    df['funding_rate_updated_dt'] = pd.to_datetime(df['funding_rate_updated_ts_ms'], unit='ms')
+    df['funding_rate'] = df['funding_rate'].astype(float)
+    df['time_delta_to_next_funding'] = pd.to_datetime(df['next_funding_ts'], unit='ms') - pd.Timestamp.now()
+    df = df.merge(df_aster_exch_info[['symbol','daily_volume','min_order_size','min_price','min_lot_size','min_notional', 'last_trade_ts_ast']], on='symbol', how='left')
+    
+    return df
+
+def load_extend_current_fr(df_mkt_stats: pd.DataFrame) -> pd.DataFrame:
+    df = pd.DataFrame(data=json.loads(s=VAL_KEY.get(name='fund_rate_extended_all')))  # ty:ignore[invalid-argument-type]
+    
+    df: pd.DataFrame = df[['symbol','funding_rate_updated_ts_ms','funding_rate']]
+    df['funding_rate_updated_dt'] = pd.to_datetime(df['funding_rate_updated_ts_ms'], unit='ms')
+    df['funding_rate'] = df['funding_rate'].astype(float)
+
+    df = df.merge(df_mkt_stats[['name','assetName','status','funding_rate_ts','min_order_size','min_price','min_lot_size','min_notional','daily_volume']].rename({'name':'symbol','funding_rate_ts':'next_funding_ts'}, axis=1), on='symbol', how='left')
+    df: pd.DataFrame = df.loc[df['status']=='ACTIVE',:]
+    df['USDT_Symbol'] = df['assetName'] + 'USDT'
+    
+    df['time_delta_to_next_funding'] = pd.to_datetime(arg=df['next_funding_ts'], unit='ms') - pd.Timestamp.now()
+
+    return df
+
+async def get_candles(symbol: str, limit: int = MINUTES_LOOKBACK) -> pd.DataFrame:
+    ### Candles for Midpoint Dispersion ###
+    # Aster
+    symbol_ast = utils.symbol_to_aster_fmt(symbol)
+    aster_candles = {
+        "url": "/fapi/v3/klines",
+        "method": "GET",
+        "params": {
+            'symbol': symbol_ast,
+            'interval': '1m',
+            'limit':str(limit)
+        }
+    }
+    j = await aster_auth.post_authenticated_url(aster_candles)
+    df_candles_aster = pd.DataFrame(j, columns=['open_ts','open_px','high_px','low_px','close_px','volume','close_ts','quote_asset_volume','count_trades','taker_buy_base_asset_volume','taker_buy_quote_asset_volume','_drop'])
+    df_candles_aster = df_candles_aster[['open_px', 'low_px', 'high_px', 'close_px', 'volume', 'open_ts']]
+    df_candles_aster[['open_px', 'low_px', 'high_px', 'close_px', 'volume']] = df_candles_aster[['open_px', 'low_px', 'high_px', 'close_px', 'volume']].astype(float)
+
+    df_candles_aster['med_px'] = ( df_candles_aster['high_px'] + df_candles_aster['low_px'] ) / 2
+    df_candles_aster['typical_px'] = ( df_candles_aster['open_px'] + df_candles_aster['high_px'] + df_candles_aster['low_px'] + df_candles_aster['close_px'] ) / 4
+
+    # Extend
+    symbol_ext = utils.symbol_to_extend_fmt(symbol)
+    ext_params = {
+        'interval':'1m',
+        'limit': limit,
+    }
+    r = json.loads(requests.get(f'https://api.starknet.extended.exchange/api/v1/info/candles/{symbol_ext}/trades', params=ext_params).text)
+    df_candles_extended = pd.DataFrame(r['data'])
+    df_candles_extended = df_candles_extended.rename({'o':'open_px','l':'low_px','h':'high_px','c':'close_px','v':'volume','T':'open_ts'}, axis=1)
+    df_candles_extended[['open_px', 'low_px', 'high_px', 'close_px', 'volume']] = df_candles_extended[['open_px', 'low_px', 'high_px', 'close_px', 'volume']].astype(float)
+    df_candles_extended['med_px'] = ( df_candles_extended['high_px'] + df_candles_extended['low_px'] ) / 2
+    df_candles_extended['typical_px'] = ( df_candles_extended['open_px'] + df_candles_extended['high_px'] + df_candles_extended['low_px'] + df_candles_extended['close_px'] ) / 4
+
+    df_candles_comb = df_candles_aster.merge(df_candles_extended, on='open_ts', how='inner', suffixes=('_ast','_ext'))
+    df_candles_comb['open_dt'] = pd.to_datetime(df_candles_comb['open_ts'], unit='ms')
+    df_candles_comb['med_ratio_aster_over_extend'] = ( df_candles_comb['med_px_ast'] / df_candles_comb['med_px_ext'] ) - 1
+
+    return df_candles_comb
+
+
+async def loop() -> None:
+    global Mkt_Info_Last_Refresh_TS_ms
+    try:
+        while True:
+            ts_arrival = round(datetime.now().timestamp() * 1000)
+            if ( ts_arrival - Mkt_Info_Last_Refresh_TS_ms ) > ( REFRESH_MKT_INFO_EVERY_SEC * 1000 ):
+                df_extend_mkt_stats = await get_extended_markets_info()
+                df_aster_exch_info = await get_aster_exch_info()
+                Mkt_Info_Last_Refresh_TS_ms = round(datetime.now().timestamp() * 1000)
+
+            df_aster_fr = load_aster_current_fr(df_aster_exch_info=df_aster_exch_info)
+            df_extend_fr = load_extend_current_fr(df_mkt_stats=df_extend_mkt_stats)
+            
+            df_comb_fr = df_extend_fr.merge(df_aster_fr, left_on='USDT_Symbol', right_on='symbol', how='inner', suffixes=('_ext', '_ast'))
+            df_comb_fr['next_funding_at_same_time'] = (abs(df_comb_fr['time_delta_to_next_funding_ext'].dt.total_seconds() - df_comb_fr['time_delta_to_next_funding_ast'].dt.total_seconds()) / 60) < 1
+            df_comb_fr['net_funding_rate'] = (df_comb_fr[['funding_rate_ext', 'funding_rate_ast']].max(axis=1) - df_comb_fr[['funding_rate_ext', 'funding_rate_ast']].min(axis=1)).where(df_comb_fr['next_funding_at_same_time'], df_comb_fr['funding_rate_ext'])
+            df_comb_fr['net_funding_rate_abs'] = df_comb_fr['net_funding_rate'].abs()
+            
+            ### NET MULT ###
+            df_comb_fr = df_comb_fr.merge(right=df_leverage_by_exch.loc[df_leverage_by_exch['exchange']=='EXTEND'], left_on='assetName', right_on='lh_asset').merge(df_leverage_by_exch.loc[df_leverage_by_exch['exchange']=='ASTER'], left_on='assetName', right_on='lh_asset', suffixes=('_ext', '_ast'))
+            df_comb_fr['net_mult'] = 1 / ( ( 0.5 / df_comb_fr['max_leverage_ext'] ) + ( 0.5 / df_comb_fr['max_leverage_ast'] ) )
+            df_comb_fr['net_mult'] = df_comb_fr['net_mult'].round(2)
+            df_comb_fr['net_mult_x_net_fr_abs'] = df_comb_fr['net_funding_rate_abs'] * df_comb_fr['net_mult']
+
+            df_best_fr_rate = df_comb_fr[
+                ['symbol_ext','symbol_ast','daily_volume_ext','daily_volume_ast','min_price_ext','min_price_ast','min_order_size_ext',
+                'min_order_size_ast','min_lot_size_ext','min_lot_size_ast','min_notional_ext','min_notional_ast','funding_rate_ext',
+                'funding_rate_ast','max_leverage_ext','max_leverage_ast','lh_asset_ext','lh_asset_ast','rh_asset_ext','rh_asset_ast',
+                'net_mult_x_net_fr_abs','net_funding_rate_abs','net_funding_rate','next_funding_at_same_time','last_trade_ts_ast']
+                ].sort_values(by='net_mult_x_net_fr_abs', ascending=False).reset_index(drop=True)
+
+            last_trade_max_ts = []
+            for index, row in df_best_fr_rate.iterrows():
+                r = json.loads(requests.get(f'https://api.starknet.extended.exchange/api/v1/info/markets/{row['symbol_ext']}/trades').text)
+                max_ts = max([t['T'] for t in r['data']])
+                last_trade_max_ts.append({'symbol_ext':row['symbol_ext'],'last_trade_ts_ext': max_ts})
+                time.sleep(0.01)
+
+            df_best_fr_rate = df_best_fr_rate.merge(pd.DataFrame(last_trade_max_ts), on='symbol_ext', how='left')
+            
+            df_best_fr_rate['last_trade_ts_dt_ast'] = pd.to_datetime(df_best_fr_rate['last_trade_ts_ast'], unit='ms')
+            df_best_fr_rate['last_trade_ts_dt_ext'] = pd.to_datetime(df_best_fr_rate['last_trade_ts_ext'], unit='ms')
+            
+
+            candles_ratios = []
+
+            for index, row in df_best_fr_rate.iterrows():
+                try:
+                    df = await get_candles(symbol=row['symbol_ext'])
+                except Exception as e:
+                    logging.warning(f'BFR failed to get candles...sleeping and retrying: {e}')
+                    time.sleep(5)
+                    df = await get_candles(symbol=row['symbol_ext'])
+
+                buy_ratio_ext = float(df['med_ratio_aster_over_extend'].median())
+                buy_ratio_std = float(df['med_ratio_aster_over_extend'].std())
+                candles_ratios.append({'symbol_ext':row['symbol_ext'], 'buy_ratio_std': buy_ratio_std, 'buy_ratio_ext':buy_ratio_ext,'buy_ratio_ast':buy_ratio_ext*-1})
+
+            df_best_fr_rate = df_best_fr_rate.merge(pd.DataFrame(candles_ratios), on='symbol_ext', how='left')
+            
+            ### Set Unfiltered Master Data ###
+            master_data = df_best_fr_rate[
+                ['symbol_ast','max_leverage_ast','lh_asset_ast','rh_asset_ast','funding_rate_ast','min_price_ast','min_order_size_ast','min_lot_size_ast','min_notional_ast','buy_ratio_ast',
+                 'symbol_ext','max_leverage_ext','lh_asset_ext','rh_asset_ext','funding_rate_ext','min_price_ext','min_order_size_ext','min_lot_size_ext','min_notional_ext','buy_ratio_ext', 'buy_ratio_std','next_funding_at_same_time']
+                ].to_json(orient='records')
+            VAL_KEY.set(name='fr_engine_best_fund_rate_master', value=str(master_data))
+            
+            
+            ### Filter BFR Data ###
+            df_best_fr_rate = df_best_fr_rate.loc[( (datetime.now().timestamp()*1000 )-df_best_fr_rate['last_trade_ts_ast']) < (5*60*1000) ] # Last traded in 3min
+            df_best_fr_rate = df_best_fr_rate.loc[( (datetime.now().timestamp()*1000 )-df_best_fr_rate['last_trade_ts_ext']) < (15*60*1000) ] # Last traded in 15min
+
+            min_daily_volume = 100_000
+            df_best_fr_rate = df_best_fr_rate.loc[ (df_best_fr_rate['daily_volume_ast']>=min_daily_volume) & (df_best_fr_rate['daily_volume_ext']>min_daily_volume) ,:].reset_index(drop=True)
+            
+            # print(df_best_fr_rate.columns)
+            # print(df_best_fr_rate.iloc[0])
+
+            if len(df_best_fr_rate) < 1:
+                raise ValueError(f'NO BFR RATE: {df_best_fr_rate}')
+
+            try:
+                ASTER = structs.Perpetual_Exchange(
+                    mult = int(df_best_fr_rate['max_leverage_ast'].iloc[0]),
+                    lh_asset = df_best_fr_rate['lh_asset_ast'].iloc[0],
+                    rh_asset = df_best_fr_rate['rh_asset_ast'].iloc[0],
+                    symbol_asset_separator = '',
+                    initial_funding_rate=float(df_best_fr_rate['funding_rate_ast'].iloc[0]),
+                    fund_rate_at_same_time=bool(df_best_fr_rate['next_funding_at_same_time'].iloc[0]),
+                    min_price=float(df_best_fr_rate['min_price_ast'].iloc[0]),
+                    min_order_size=float(df_best_fr_rate['min_order_size_ast'].iloc[0]),
+                    min_lot_size=float(df_best_fr_rate['min_lot_size_ast'].iloc[0]),
+                    min_notional=float(df_best_fr_rate['min_notional_ast'].iloc[0]),
+                    buy_ratio=float(df_best_fr_rate['buy_ratio_ast'].iloc[0]),
+                    buy_ratio_std=float(df_best_fr_rate['buy_ratio_std'].iloc[0]),
+                )          
+                EXTEND = structs.Perpetual_Exchange(
+                    mult = int(df_best_fr_rate['max_leverage_ext'].iloc[0]),
+                    lh_asset = df_best_fr_rate['lh_asset_ext'].iloc[0],
+                    rh_asset = df_best_fr_rate['rh_asset_ext'].iloc[0],
+                    symbol_asset_separator = '-',
+                    initial_funding_rate=float(df_best_fr_rate['funding_rate_ext'].iloc[0]),
+                    fund_rate_at_same_time=bool(df_best_fr_rate['next_funding_at_same_time'].iloc[0]),
+                    min_price=float(df_best_fr_rate['min_price_ext'].iloc[0]),
+                    min_order_size=float(df_best_fr_rate['min_order_size_ext'].iloc[0]),
+                    min_lot_size=float(df_best_fr_rate['min_lot_size_ext'].iloc[0]),
+                    min_notional=float(df_best_fr_rate['min_notional_ext'].iloc[0]),
+                    buy_ratio=float(df_best_fr_rate['buy_ratio_ext'].iloc[0]),
+                    buy_ratio_std=float(df_best_fr_rate['buy_ratio_std'].iloc[0]),
+                )
+            except Exception as e:
+                logging.critical(f'Failed to build ASTER/EXTEND objs err: {e}; df cols: {df_best_fr_rate.columns}')
+                logging.error(traceback.format_exc())
+                continue
+            best_next_funding_pair: dict[str, dict] = {'ASTER': asdict(obj=ASTER), 'EXTEND': asdict(obj=EXTEND)}
+            VAL_KEY.set(name='fr_engine_best_fund_rate_output', value=json.dumps(obj=best_next_funding_pair))
+            
+
+            print(df_best_fr_rate[['symbol_ext','max_leverage_ext','buy_ratio_ext','net_funding_rate','daily_volume_ast','buy_ratio_ast']].head(10))
+            logging.info(f'BFR REFRESHED @ {datetime.now()}')
+            time.sleep(LOOP_SLEEP_SEC)
+            continue
+    except valkey.exceptions.ConnectionError as e:
+        logging.info(f"Could not connect to Valkey. Please check the publish server is up; {e}")
+    except KeyboardInterrupt:
+        logging.info('SHUTTING DOWN...')
+    except Exception as e:
+        logging.error(traceback.format_exc())
+        logging.critical(f'*** CRASHED: {e}')
+
+
+### STARTUP ###
+async def main() -> None:
+    global VAL_KEY
+    # global CON
+
+    VAL_KEY = valkey.Valkey(host='localhost', port=6379, db=0, decode_responses=True)
+    # engine = create_async_engine('mysql+asyncmy://root:pwd@localhost/fund_rate')
+
+    await loop()
+
+if __name__ == '__main__':
+    START_TIME = round(number=datetime.now().timestamp()*1000)
+
+    logging.info(msg=f'Log FilePath: {LOG_FILEPATH}')
+
+    logging.basicConfig(
+        force=True,
+        filename=LOG_FILEPATH,
+        level=logging.INFO,
+        format='%(asctime)s - %(levelname)s - %(message)s',
+        filemode='w'
+    )
+    logging.info(msg=f"STARTED: {START_TIME}")
+
+    asyncio.run(main())

engine_best_funding_rate/.dockerignore

@@ -0,0 +1 @@
+../rust/

engine_best_funding_rate/Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.13-slim
+
+RUN apt-get update && \
+    apt-get install -y build-essential
+
+RUN gcc --version
+RUN rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY requirements.txt .
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+# Finally, run gunicorn.
+CMD [ "python", "engine_best_funding_rate.py"]
+# CMD [ "gunicorn", "--workers=5", "--threads=1", "-b 0.0.0.0:8000", "app:server"]

engine_health.py

@@ -0,0 +1,129 @@
+import asyncio
+import json
+import logging
+import os
+import traceback
+from datetime import datetime
+import time
+import valkey
+from dotenv import load_dotenv
+import modules.utils as utils
+import modules.structs as structs
+from pydantic import BaseModel
+import docker
+
+### Database ###
+VAL_KEY: valkey.Valkey = valkey.Valkey(host='localhost', port=6379, db=0, decode_responses=True)
+DOCKER = docker.from_env()
+
+### Logging ###
+load_dotenv()
+LOG_FILEPATH: str = f'{os.getenv("LOGS_PATH")}/Fund_Rate_Engine_Health.log'
+
+### CONSTANTS ###
+MAX_TIME_SINCE_LAST_UPDATE_MS: int = 1000 * 60 * 3 # 1000 x 60 sec x [minutes]
+LOOP_SLEEP_SEC: int = 5
+
+### Globals ###
+
+### Structs ###
+class Health_Status(BaseModel):
+    status: str # ENUM: 'HEALTHY' | 'UNHEALTHY' | 'DEAD'
+    timestamp: int
+    vk_objs: list[structs.VK_Obj]
+
+async def get_algo_working_symbol() -> str:
+    vk_get: str = VAL_KEY.get(name='fr_algo_working_symbol')  # ty:ignore[invalid-assignment]
+    d = json.loads(vk_get)
+    algo_symbol: str = d.get('EXTEND', {}).get('symbol', '')
+    
+    return algo_symbol
+
+async def main() -> None:
+    vk_objs = [
+        structs.VK_Orchestrator_Output(),
+        structs.VK_Working_Symbol(),
+        structs.VK_User_Orders_Extend(),
+        structs.VK_User_Trades_Extend(),
+        structs.VK_User_Balances_Aster(),
+        structs.VK_User_Balances_Extend(),
+        structs.VK_User_Positions_Aster(),
+        structs.VK_User_Positions_Extend(),
+        structs.VK_FR_Aster(),
+        structs.VK_FR_All_Aster(),
+        structs.VK_FR_Extend(),
+        structs.VK_FR_All_Extend(),
+        structs.VK_Ticker_Aster(),
+        structs.VK_Ticker_Extend(),
+        structs.VK_Trade_Aster(),
+        structs.VK_Trade_Extend(),
+    ]
+
+    health_status = Health_Status(
+        status = 'HEALTHY',
+        timestamp = round(number=datetime.now().timestamp()*1000),
+        vk_objs = vk_objs,  # ty:ignore[invalid-argument-type]
+    )
+
+    try:
+        while True:
+            algo_symbol = await get_algo_working_symbol()
+            health_status.timestamp = round(number=datetime.now().timestamp()*1000)
+
+            for o in health_status.vk_objs:
+                vk_symbol = o.data.get('symbol') if isinstance(o.data, dict) else None
+                await o.checks.run_checks(args={
+                    'timestamp': health_status.timestamp,
+                    'algo_symbol': algo_symbol,
+                    'vk_symbol': vk_symbol,
+                })
+
+            vk_statuses = [o.status for o in health_status.vk_objs]
+            if 'DEAD' in vk_statuses:
+                health_status.status = 'DEAD'
+            elif 'UNHEALTHY' in vk_statuses:
+                health_status.status = 'UNHEALTHY'
+            else:
+                health_status.status = 'HEALTHY'
+
+            if health_status.status != 'HEALTHY':
+                all_containers = DOCKER.containers.list(all=True)
+
+                for c in all_containers:
+                    if c.status == 'running':
+                        logging.warning(f"stopping: ID: {c.id}, Name: {c.name}, Status: {c.status}")
+                        container = DOCKER.containers.get(c.id)
+                        container.stop(timeout=10)
+                
+                logging.info('Stopped all containers')
+            
+            # VAL_KEY.set(name='health_status', value=json.dumps(obj=(health_status)))
+            
+            logging.info(vk_statuses)
+            
+            if LOOP_SLEEP_SEC > 0:
+                time.sleep(LOOP_SLEEP_SEC)
+            
+            continue
+    except KeyboardInterrupt:
+        logging.info(msg='ORCHESTRATOR SHUTTING DOWN...')
+    except Exception as e:
+        logging.error(msg=traceback.format_exc())
+        logging.critical(msg=f'*** ORCHESTRATOR CRASHED: {e}')
+
+
+if __name__ == '__main__':
+    START_TIME: int = round(number=datetime.now().timestamp()*1000)
+
+    logging.info(msg=f'Log FilePath: {LOG_FILEPATH}')
+
+    logging.basicConfig(
+        force=True,
+        filename=LOG_FILEPATH,
+        level=logging.INFO,
+        format='%(asctime)s - %(levelname)s - %(message)s',
+        filemode='w'
+    )
+    logging.info(msg=f"STARTED: {START_TIME}")
+
+    asyncio.run(main())

engine_orders.py

@@ -0,0 +1,159 @@
+import asyncio
+import json
+import logging
+import os
+import traceback
+from datetime import datetime
+import time
+import valkey
+from dotenv import load_dotenv
+import modules.utils as utils
+import multiprocessing as mp
+import threading
+from typing import Any
+
+
+
+'''
+TO DO:
+- Insert config changes into database for analysis later / general tracking
+'''
+
+LOCAL_ORDERS: list = []
+
+### Database ###
+VK_IN: str = 'fr_engine_orders_input'
+VK_OUT: str = 'fr_engine_orders_output'
+VAL_KEY: valkey.Valkey = valkey.Valkey(host='localhost', port=6379, db=0, decode_responses=True)
+
+### Logging ###
+load_dotenv()
+LOG_FILEPATH: str = f'{os.getenv("LOGS_PATH")}/Fund_Rate_Engine_Orders.log'
+
+### Main Listener for Order Requests (e.g. from Algo Engine) ###
+def receive_orders():
+    global LOCAL_ORDERS
+
+    VK_PUBSUB: valkey.client.PubSub = VAL_KEY.pubsub()
+    VK_PUBSUB.subscribe(VK_IN)
+    for message in VK_PUBSUB.listen():
+        loop_start = time.time()
+        if message['type'] == 'message':
+            ts_arrival: int = round(number=datetime.now().timestamp()*1000)
+            
+            # Receive Update Msg from PubSub
+            data: dict = json.loads(s=message['data']) 
+            print(data)
+            LOCAL_ORDERS.append(data)
+        
+        print(f'__ Rec Orders: Loop End __ - Algo Engine ms: {(time.time() - loop_start)*1000:.2f}')
+
+### Listeners - Aster ###
+def receive_position_updates_aster():
+    VK_PUBSUB: valkey.client.PubSub = VAL_KEY.pubsub()
+    VK_PUBSUB.subscribe('fr_aster_user_positions')
+    for message in VK_PUBSUB.listen():
+        loop_start = time.time()
+        if message['type'] == 'message':
+            ts_arrival: int = round(number=datetime.now().timestamp()*1000)
+            
+            # Receive Update Msg from PubSub
+            data: dict = json.loads(s=message['data']) 
+            print(data)
+
+        print(f'__ Aster Notional: Loop End __ - Algo Engine ms: {(time.time() - loop_start)*1000:.2f}')
+
+def receive_order_updates_aster():
+    VK_PUBSUB: valkey.client.PubSub = VAL_KEY.pubsub()
+    VK_PUBSUB.subscribe('fr_aster_user_orders')
+    for message in VK_PUBSUB.listen():
+        loop_start = time.time()
+        if message['type'] == 'message':
+            # ts_arrival: int = round(number=datetime.now().timestamp()*1000)
+            
+            # Receive Update Msg from PubSub
+            data: dict = json.loads(s=message['data']) 
+            print(data)
+
+        print(f'__ Aster Orders: Loop End __ - Algo Engine ms: {(time.time() - loop_start)*1000:.2f}')
+
+### Listeners - Extend ###
+def receive_position_updates_extend():
+    VK_PUBSUB: valkey.client.PubSub = VAL_KEY.pubsub()
+    VK_PUBSUB.subscribe('fr_extended_user_positions')
+    for message in VK_PUBSUB.listen():
+        loop_start = time.time()
+        if message['type'] == 'message':
+            ts_arrival: int = round(number=datetime.now().timestamp()*1000)
+            
+            # Receive Update Msg from PubSub
+            data: dict = json.loads(s=message['data']) 
+            print(data)
+
+        print(f'__ Aster Notional: Loop End __ - Algo Engine ms: {(time.time() - loop_start)*1000:.2f}')
+
+def receive_order_updates_extend():
+    VK_PUBSUB: valkey.client.PubSub = VAL_KEY.pubsub()
+    VK_PUBSUB.subscribe('fr_extended_user_orders')
+    for message in VK_PUBSUB.listen():
+        loop_start = time.time()
+        if message['type'] == 'message':
+            # ts_arrival: int = round(number=datetime.now().timestamp()*1000)
+            
+            # Receive Update Msg from PubSub
+            data: dict = json.loads(s=message['data']) 
+            print(data)
+
+        print(f'__ Aster Orders: Loop End __ - Algo Engine ms: {(time.time() - loop_start)*1000:.2f}')
+
+async def main() -> None:
+    global LOCAL_ORDERS
+
+    try:
+        t_rec_orders = threading.Thread(target=receive_orders)
+        t_rec_orders.daemon = True
+        t_rec_orders.start()
+
+
+        # while True:
+        #     print(f"Subscribed to '{VK_IN}'. Waiting for messages...")
+        #     print(f'LOCAL_ORDERS: {LOCAL_ORDERS}')
+            
+            
+        #     aster_position_updates: Any = VAL_KEY.get(name=VK_POS_ASTER)
+        #     aster_position_updates: list = json.loads(s=aster_position_updates) if aster_position_updates is not None else []
+            
+        #     print(f'Aster Pos Updates: {aster_position_updates}')
+            
+            
+        #     time.sleep(5)
+
+
+            
+                
+
+
+    except valkey.exceptions.ConnectionError as e:
+        logging.info(msg=f"Could not connect to Valkey. Please check the publish server is up; {e}")
+    except KeyboardInterrupt:
+        logging.info(msg='ORCHESTRATOR SHUTTING DOWN...')
+    except Exception as e:
+        logging.error(msg=traceback.format_exc())
+        logging.critical(msg=f'*** ORCHESTRATOR CRASHED: {e}')
+
+
+if __name__ == '__main__':
+    START_TIME: int = round(number=datetime.now().timestamp()*1000)
+
+    logging.info(msg=f'Log FilePath: {LOG_FILEPATH}')
+
+    logging.basicConfig(
+        force=True,
+        filename=LOG_FILEPATH,
+        level=logging.INFO,
+        format='%(asctime)s - %(levelname)s - %(message)s',
+        filemode='w'
+    )
+    logging.info(msg=f"STARTED: {START_TIME}")
+
+    asyncio.run(main())

modules/aster_auth.py

@@ -2,51 +2,53 @@ import requests
 from dotenv import load_dotenv
 import os
 import time
+import logging
 import threading
-import urllib
+from urllib import parse
 
 from eth_account.messages import encode_typed_data
 from eth_account import Account
+from eth_account.datastructures import SignedMessage
 
 load_dotenv()
 
-user = os.getenv("RABBY_WALLET")
+USER: str  = os.getenv(key="RABBY_WALLET")  # ty:ignore[invalid-assignment]
-signer = os.getenv("ASTER_API_WALLET_ADDRESS")
+SIGNER: str = os.getenv(key="ASTER_API_WALLET_ADDRESS") # ty:ignore[invalid-assignment]
-private_key = os.getenv("ASTER_API_PRIVATE_KEY")
+PRIVATE_KEY: str = os.getenv(key="ASTER_API_PRIVATE_KEY") # ty:ignore[invalid-assignment]
 
 _last_ms = 0
 _i = 0
 
-async def post_authenticated_url(req: dict) -> dict:
+async def post_authenticated_url(req: dict) -> list | dict:
-    typed_data = {
+    typed_data: dict = {
-    "types": {
+        "types": {
-        "EIP712Domain": [
+            "EIP712Domain": [
-        {"name": "name", "type": "string"},
+                {"name": "name", "type": "string"},
-        {"name": "version", "type": "string"},
+                {"name": "version", "type": "string"},
-        {"name": "chainId", "type": "uint256"},
+                {"name": "chainId", "type": "uint256"},
-        {"name": "verifyingContract", "type": "address"}
+                {"name": "verifyingContract", "type": "address"}
-        ],
+            ],
-        "Message": [
+            "Message": [
-        { "name": "msg", "type": "string" }
+                { "name": "msg", "type": "string" },
-        ]
+            ]
-    },
+        },
-    "primaryType": "Message",
+        "primaryType": "Message",
-    "domain": {
+        "domain": {
-        "name": "AsterSignTransaction",
+            "name": "AsterSignTransaction",
-        "version": "1",
+            "version": "1",
-        "chainId": 1666,
+            "chainId": 1666,
-        "verifyingContract": "0x0000000000000000000000000000000000000000"
+            "verifyingContract": "0x0000000000000000000000000000000000000000"
-    },
+        },
-    "message": {
+        "message": {
-        "msg": "$msg"
+            "msg": "$msg"
+        }
     }
-    }
+    
-    headers = {
+    headers: dict[str, str] = {
         'Content-Type': 'application/x-www-form-urlencoded',
         'User-Agent': 'PythonApp/1.0'
     }
-    host = 'https://fapi.asterdex.com'
+    host: str = 'https://fapi.asterdex.com'
-
 
     def get_nonce():
         _nonce_lock = threading.Lock()
@@ -71,38 +73,34 @@ async def post_authenticated_url(req: dict) -> dict:
         )
         return Account.sign_message(message, private_key=private_key)
 
-    async def send_by_url(req):
+    async def send_by_url(req) -> list | dict:  # ty:ignore[invalid-return-type]
         my_dict = req['params'].copy()
         url = host + req['url']
         method = req['method']
 
-        my_dict['nonce'] = str(get_nonce())
+        my_dict['nonce'] = str(object=get_nonce())
-        my_dict['user'] = user
+        my_dict['user'] = USER
-        my_dict['signer'] = signer
+        my_dict['signer'] = SIGNER
 
-        param = urllib.parse.urlencode(my_dict)
+        param: str = parse.urlencode(query=my_dict)
 
         typed_data['message']['msg'] = param
-        signed = sign_typed_data(typed_data, private_key)
+        signed: SignedMessage = sign_typed_data(data=typed_data, private_key=PRIVATE_KEY)
 
-        full_url = url + '?' + param + '&signature=' + signed.signature.hex()
+        full_url: str = url + '?' + param + '&signature=' + signed.signature.hex()
-        # print(full_url)
 
         if method == 'GET':
-            res = requests.get(full_url, headers=headers)
+            res: requests.Response = requests.get(url=full_url, headers=headers)
-            # print(res.status_code, res.text)
+            # logging.warning(res.status_code, res.text)
             return res.json()
         elif method == 'POST':
-            res = requests.post(full_url, headers=headers)
+            res: requests.Response = requests.post(url=full_url, headers=headers)
-            # print(res.status_code, res.text)
             return res.json()
         elif method == 'PUT':
-            res = requests.put(full_url, headers=headers)
+            res: requests.Response = requests.put(url=full_url, headers=headers)
-            # print(res.status_code, res.text)
             return res.json()
         elif method == 'DELETE':
-            res = requests.delete(full_url, headers=headers)
+            res: requests.Response = requests.delete(url=full_url, headers=headers)
-            # print(res.status_code, res.text)
             return res.json()
 
     return await send_by_url(req=req)

modules/aster_db.py

@@ -139,9 +139,30 @@ async def create_fr_aster_user_account_pos(
         else:
             raise ValueError('Only MySQL engine is implemented')
 
-
+### Mkt Trades Table ####
-
+async def create_fr_aster_mkt_trades(
-
+        CON: AsyncContextManager,
-
+        engine: str = 'mysql', # mysql | duckdb
-
+    ) -> None:
-
+    if CON is None:
+        logging.info("NO DB CONNECTION, SKIPPING Create Statements")
+    else:
+        if engine == 'mysql':
+            logging.info('Creating Table if Does Not Exist: fr_aster_mkt_trades')
+            await CON.execute(text("""
+            CREATE TABLE IF NOT EXISTS fr_aster_mkt_trades (
+                timestamp_arrival BIGINT,
+                timestamp_msg BIGINT,
+                timestamp_trade BIGINT,
+                symbol VARCHAR(20),
+                aggregate_trade_id VARCHAR(100),
+                price DOUBLE,
+                qty DOUBLE,
+                first_trade_id VARCHAR(100),
+                last_trade_id VARCHAR(100),
+                is_buyer_mkt_maker BOOL
+            );
+            """))
+            await CON.commit()
+        else:
+            raise ValueError('Only MySQL engine is implemented')

modules/db.py

@@ -20,7 +20,6 @@ async def insert_df_to_mysql(
                 df = pd.DataFrame(params)
             else:
                 df = params
-            print(f'DB INSERT: table: {table_name}; CON: {CON}; params: {params}')
             await CON.run_sync(
                 lambda sync_conn: df.to_sql(name=table_name, con=sync_conn, if_exists='append', index=False)
             )

modules/extended_db.py

@@ -150,3 +150,32 @@ async def create_fr_extended_user_position(
         else:
             raise ValueError('Only MySQL engine is implemented')
 
+### Market Trades Table ####
+async def create_fr_extended_mkt_trades(
+        CON: AsyncContextManager,
+        engine: str = 'mysql', # mysql | duckdb
+    ) -> None:
+    if CON is None:
+        logging.info("NO DB CONNECTION, SKIPPING Create Statements")
+    else:
+        if engine == 'mysql':
+            logging.info('Creating Table if Does Not Exist: fr_extended_mkt_trades')
+            await CON.execute(text("""
+            CREATE TABLE IF NOT EXISTS fr_extended_mkt_trades (
+                sequence_id INT,
+                timestamp_arrival BIGINT,
+                timestamp_msg BIGINT,
+                timestamp_trade BIGINT,
+                symbol VARCHAR(20),
+                side_taker VARCHAR(20),
+                trade_type VARCHAR(20),
+                price DOUBLE,
+                qty DOUBLE,
+                trade_id VARCHAR(100),
+                is_buyer_mkt_maker BOOL
+            );
+            """))
+            await CON.commit()
+        else:
+            raise ValueError('Only MySQL engine is implemented')
+

modules/manual_leverage.py

@@ -0,0 +1,38 @@
+from dataclasses import dataclass
+
+@dataclass(kw_only=False)
+class Asset_Leverage:
+    exchange: str
+    lh_asset: str
+    rh_asset: str
+    max_leverage: int
+    max_notional: float
+    # max_leverage_notional: list = field(default_factory=list)
+
+### MANUAL LEVERAGE DATA ###
+LEVERAGE_BY_EXCH: list[Asset_Leverage] = [
+    Asset_Leverage('ASTER', 'ASTER', 'USDT', 75 , 20_000 ), Asset_Leverage('EXTEND', 'ASTER', 'USD', 25, 400_000  ),
+    Asset_Leverage('ASTER', 'AAVE' , 'USDT', 10 , 115_290), Asset_Leverage('EXTEND', 'AAVE' , 'USD', 50, 500_000  ),
+    Asset_Leverage('ASTER', '4'    , 'USDT', 50 , 5_000  ), Asset_Leverage('EXTEND', '4'    , 'USD', 5 , 100_000  ),
+    Asset_Leverage('ASTER', 'BNB'  , 'USDT', 100, 10_000 ), Asset_Leverage('EXTEND', 'BNB'  , 'USD', 50, 500_000  ),
+    Asset_Leverage('ASTER', 'BTC'  , 'USDT', 150, 300_000), Asset_Leverage('EXTEND', 'BTC'  , 'USD', 50, 4_000_000),
+    Asset_Leverage('ASTER', 'CHIP' , 'USDT', 50 , 5_000  ), Asset_Leverage('EXTEND', 'CHIP' , 'USD', 5 , 100_000  ),
+    Asset_Leverage('ASTER', 'CLU'  , 'USDT', 50 , 10_000 ), Asset_Leverage('EXTEND', 'WTI'  , 'USD', 5 , 1_000_000),
+    Asset_Leverage('ASTER', 'DOGE' , 'USDT', 75 , 80_000 ), Asset_Leverage('EXTEND', 'DOGE' , 'USD', 50, 500_000  ),
+    Asset_Leverage('ASTER', 'ENA'  , 'USDT', 25 , 30_473 ), Asset_Leverage('EXTEND', 'ENA'  , 'USD', 50, 500_000  ),
+    Asset_Leverage('ASTER', 'ETH'  , 'USDT', 150, 300_000), Asset_Leverage('EXTEND', 'ETH'  , 'USD', 50, 4_000_000),
+    Asset_Leverage('ASTER', 'HYPE' , 'USDT', 300, 1_000  ), Asset_Leverage('EXTEND', 'HYPE' , 'USD', 50, 1_000_000),
+    Asset_Leverage('ASTER', 'INIT' , 'USDT', 50 , 5_000  ), Asset_Leverage('EXTEND', 'INIT' , 'USD', 5 , 100_000  ),
+    Asset_Leverage('ASTER', 'LIT'  , 'USDT', 50 , 2_500  ), Asset_Leverage('EXTEND', 'LIT'  , 'USD', 25, 400_000  ),
+    Asset_Leverage('ASTER', 'SOL'  , 'USDT', 100, 50_000 ), Asset_Leverage('EXTEND', 'SOL'  , 'USD', 50, 1_000_000),
+    Asset_Leverage('ASTER', 'SUI'  , 'USDT', 75 , 5_416  ), Asset_Leverage('EXTEND', 'SUI'  , 'USD', 50, 500_000  ),
+    Asset_Leverage('ASTER', 'TRUMP', 'USDT', 10 , 60_000 ), Asset_Leverage('EXTEND', 'TRUMP', 'USD', 25, 400_000  ),
+    Asset_Leverage('ASTER', 'WLFI' , 'USDT', 25 , 104_869), Asset_Leverage('EXTEND', 'WLFI' , 'USD', 10, 250_000  ),
+    # Asset_Leverage('ASTER', 'XAG'  , 'USDT', 100, 50_000 ), Asset_Leverage('EXTEND', 'XAG'  , 'USD', 10, 1_000_000),
+    # Asset_Leverage('ASTER', 'XAU'  , 'USDT', 75 , 2_500  ), Asset_Leverage('EXTEND', 'XAU'  , 'USD', 25, 2_000_000),
+    Asset_Leverage('ASTER', 'XMR'  , 'USDT', 50 , 10_000 ), Asset_Leverage('EXTEND', 'XMR'  , 'USD', 25, 400_000  ),
+    Asset_Leverage('ASTER', 'XPT'  , 'USDT', 3  , 30_000 ), Asset_Leverage('EXTEND', 'XPT'  , 'USD', 5 , 1_000_000),
+    Asset_Leverage('ASTER', 'XRP'  , 'USDT', 100, 40_000 ), Asset_Leverage('EXTEND', 'XRP'  , 'USD', 50, 500_000  ),
+    Asset_Leverage('ASTER', 'ZEC'  , 'USDT', 75 , 6_250  ), Asset_Leverage('EXTEND', 'ZEC'  , 'USD', 10, 250_000  ),
+    Asset_Leverage('ASTER', 'ZORA' , 'USDT', 5  , 100_000), Asset_Leverage('EXTEND', 'ZORA' , 'USD', 5 , 100_000  ),
+]

modules/structs.py

@@ -3,21 +3,337 @@ from dataclasses import dataclass, field
 from typing import Any
 
 import valkey
+from pydantic import BaseModel
+from datetime import datetime
+from sqlalchemy.util.typing import Self
+from collections.abc import Sequence, Callable
+import modules.utils as utils
+
+def ret_true():
+    return True
 
 
-@dataclass(kw_only=True)
+class Locked_Value(Sequence):
-class Algo_Config:
+    def __init__(self, initial_value: Any = None, unlock_func: Callable=ret_true):
-    Config_Updated_Timestamp: int
+        self._value: Any = initial_value
+        self._unlock_func: Callable = unlock_func
+        self._is_locked: bool = True
+
+    def __repr__(self):
+        return str((self._value, self._is_locked, self._unlock_func))
+
+    def __len__(self):
+        return len((self._value, self._is_locked, self._unlock_func))
+
+    def __getitem__(self, index):
+        return (self._value, self._is_locked, self._unlock_func)[index]
+
+    def __str__(self):
+        return str((self._value))
+
+    def unlock(self) -> Self:
+        if self._unlock_func():
+            self._is_locked = False
+        return self
+
+    def lock(self) -> Self:
+        self._is_locked = True
+        return self
+
+    @property
+    def is_locked(self):
+        return self._is_locked
+
+    @property
+    def is_unlocked(self):
+        return not(self._is_locked)
+
+    @property
+    def value(self):
+        return self._value
+
+    @value.setter
+    def value(self, v):
+        if not(self._is_locked):
+            self._value = v
+        else:
+            raise ValueError(f'Failed to set value, item is locked: {str(self.__repr__)}')
+
+
+class Current_Previous_Value:
+    def __init__(self, value: Any = None, previous_value: Any = None):
+        self._value: Any = value
+        self._previous_value: Any = previous_value
+
+    def __repr__(self):
+        return str((self._value, self._previous_value))
+
+    def __len__(self):
+        return len((self._value, self._previous_value))
+
+    def __getitem__(self, index):
+        return (self._value, self._previous_value)[index]
+
+    def __str__(self):
+        return str(self._value)
+
+    @property
+    def value(self):
+        return self._value
+
+    @property
+    def previous_value(self):
+        return self._previous_value
+
+    @value.setter
+    def value(self, v):
+        self._previous_value = self._value
+        self._value = v
+
+### Valkey Objects ###
+class VK_Check(BaseModel):
+    status: str = 'HEALTHY' # HEALTHY | UNHEALTHY | DEAD
+    method: str | None
+
+
+class VK_Checks(BaseModel):
+    status: VK_Check = VK_Check(method='check_status')
+    timestamp: VK_Check = VK_Check(method='check_timestamp')
+    symbol: VK_Check = VK_Check(method='check_symbol')
+
+    async def run_checks(self, args: dict | None = None):
+        for f in self.model_dump():
+            method = getattr(getattr(self, f), 'method')
+            if method is not None:
+                await getattr(self, method)(args = args)
+
+    async def check_status(self, args: dict | None = None) -> None:
+        # print('checking status')
+        if self.status.status == '':
+            self.status.status = 'UNHEALTHY'
+        else:
+            self.status.status = self.status.status
+
+    async def check_status_nested(self, args: dict | None = None) -> None:
+        # print('checking status')
+        if self.status.status == '':
+            self.status.status = 'UNHEALTHY'
+        else:
+            self.status.status = self.status.status
+
+    async def check_timestamp(self, args: dict | None = None) -> None:
+        # print('checking timestamp')
+        if args is not None:
+            ts = int(args.get('timestamp', 0))
+            now = round(datetime.now().timestamp()*1000)
+
+            if (now - ts) > 1:
+                self.timestamp.status = 'UNHEALTHY'
+            else:
+                self.timestamp.status = 'HEALTHY'
+        else:
+            raise ValueError("Must pass in 'timestamp' arg")
+
+    async def check_symbol(self, args: dict | None = None) -> None:
+        # print('checking symbol')
+
+        if args is not None:
+            symbol = utils.symbol_to_extend_fmt(args.get('algo_symbol', ''))
+            vk_symbol = utils.symbol_to_extend_fmt(args.get('vk_symbol', ''))
+            if symbol == vk_symbol:
+                self.symbol.status = 'HEALTHY'
+            else:
+                self.symbol.status = 'UNHEALTHY'
+        else:
+            raise ValueError("Must pass in 'algo_symbol' and 'vk_symbol' args")
+
+    
+class VK_Obj(BaseModel):
+    vk_name: str
+    timestamp: int = round(datetime.now().timestamp()*1000)
+    status: str = 'HEALTHY' # HEALTHY | UNHEALTHY | DEAD
+    checks: VK_Checks = VK_Checks()
+    data: Any = None
+
+    async def get(self, VK_CON: valkey.Valkey) -> None:
+        print('getting')
+        vk_get: str = VK_CON.get(self.vk_name)  # ty:ignore[invalid-assignment]
+        vk_dict: dict = json.loads(vk_get)
+        self.__init__(**vk_dict)
+    
+    async def set(self, VK_CON: valkey.Valkey, data_override: Any = None) -> None:
+        print('setting')
+        if data_override is not None:
+            self.data = data_override
+        
+        self.timestamp: int = round(datetime.now().timestamp()*1000)
+        
+        j: str = self.model_dump_json()
+        VK_CON.set(self.vk_name, j)
+
+### Valkey Archetypes ###
+# Engine - Health
+
+# Engine - Orchestrator
+class VK_Orchestrator_Output(VK_Obj):
+    vk_name: str = 'fr_orchestrator_output'
+    checks: VK_Checks = VK_Checks(
+        symbol = VK_Check(method = None)
+    )
+
+# Algo
+class VK_Working_Symbol(VK_Obj):
+    vk_name: str = 'fr_algo_working_symbol'
+    checks: VK_Checks = VK_Checks(
+        symbol = VK_Check(method = None)
+    )
+
+class VK_Algo_Status(VK_Obj):
+    vk_name: str = 'algo_status'
+    checks: VK_Checks = VK_Checks(
+        symbol = VK_Check(method = None)
+    )
+
+# User - Orders
+class VK_User_Orders_Aster(VK_Obj):
+    vk_name: str = 'fr_aster_user_orders'
+    checks: VK_Checks = VK_Checks(
+        symbol = VK_Check(method = None)
+    )
+
+class VK_User_Orders_Extend(VK_Obj):
+    vk_name: str = 'fr_extended_user_orders'
+    checks: VK_Checks = VK_Checks(
+        symbol = VK_Check(method = None)
+    )
+
+# User - Trades
+class VK_User_Trades_Extend(VK_Obj):
+    vk_name: str = 'fr_extended_user_trades'
+    checks: VK_Checks = VK_Checks(
+        symbol = VK_Check(method = None)
+    )
+
+# User - Balances
+class VK_User_Balances_Aster(VK_Obj):
+    vk_name: str = 'fr_aster_user_balances'
+    checks: VK_Checks = VK_Checks(
+        symbol = VK_Check(method = None)
+    )
+
+class VK_User_Balances_Extend(VK_Obj):
+    vk_name: str = 'fr_extended_user_balances'
+    checks: VK_Checks = VK_Checks(
+        symbol = VK_Check(method = None)
+    )
+
+# User - Positions
+class VK_User_Positions_Aster(VK_Obj):
+    vk_name: str = 'fr_aster_user_positions'
+    checks: VK_Checks = VK_Checks(
+        symbol = VK_Check(method = None)
+    )
+
+class VK_User_Positions_Extend(VK_Obj):
+    vk_name: str = 'fr_extended_user_positions'
+    checks: VK_Checks = VK_Checks(
+        symbol = VK_Check(method = None)
+    )
+
+# Fund Rates
+class VK_FR_Aster(VK_Obj):
+    vk_name: str = 'fund_rate_aster'
+    checks: VK_Checks = VK_Checks(
+        symbol = VK_Check(method = None)
+    )
+    
+
+class VK_FR_All_Aster(VK_Obj):
+    vk_name: str = 'fund_rate_aster_all'
+    checks: VK_Checks = VK_Checks(
+        symbol = VK_Check(method = None)
+    )
+
+class VK_FR_Extend(VK_Obj):
+    vk_name: str = 'fund_rate_extended'
+    checks: VK_Checks = VK_Checks(
+        symbol = VK_Check(method = None)
+    )
+
+class VK_FR_All_Extend(VK_Obj):
+    vk_name: str = 'fund_rate_extended_all'
+    checks: VK_Checks = VK_Checks(
+        symbol = VK_Check(method = None)
+    )
+
+# Tickers
+class VK_Ticker_Aster(VK_Obj):
+    vk_name: str = 'fut_ticker_aster'
+    checks: VK_Checks = VK_Checks(
+        symbol = VK_Check(method = None)
+    )
+
+class VK_Ticker_Extend(VK_Obj):
+    vk_name: str = 'fut_ticker_extended'
+    checks: VK_Checks = VK_Checks(
+        symbol = VK_Check(method = None)
+    )
+
+# Trades
+class VK_Trade_Aster(VK_Obj):
+    vk_name: str = 'fut_last_trade_aster'
+    checks: VK_Checks = VK_Checks(
+        symbol = VK_Check(method = None)
+    )
+
+class VK_Trade_Extend(VK_Obj):
+    vk_name: str = 'fut_last_trade_extended'
+    checks: VK_Checks = VK_Checks(
+        symbol = VK_Check(method = None)
+    )
+
+
+### Algo Objects ###
+class Algo_Status(BaseModel):
+    last_update_ts_ms: int
+    status: str # 'WORKING' | 'STOPPED' ///// # ENUM: 'HEALTHY' | 'UNHEALTHY' | 'DEAD'
+    expected_alpha: float
+    model_ratio: float
+    current_ratio: float
+
+
+class Algo_Config_Overrides(BaseModel):
     Allow_Ordering_Aster: bool
     Allow_Ordering_Extend: bool
+    Allow_Symbol_Change: bool
+    Flatten_Open_Positions: bool
+    Flatten_Open_Positions_Opportunistic: bool
+    Flip_Side_For_Testing: bool
+
+
+class Algo_Config_Config(BaseModel):
     Loop_Sleep_Sec: int
+    Max_Order_Over_Notional_Ratio: float
     Max_Target_Notional: float
     Min_Time_To_Funding_Minutes: int
-    Price_Worsener_Aster: float
+    Min_Fund_Rate_Pct_To_Trade: float
-    Price_Worsener_Extend: float
+    Price_Worsener_Aster: int
+    Price_Worsener_Extend: int
+    Switch_To_Taker_Seconds: int
     Target_Open_Cash_Position: int
 
-    Print_Summary_Each_Loop: bool = False
+
+class Algo_Config_Logging(BaseModel):
+    Log_Summary_Each_Loop: bool
+    Print_Summary_Each_Loop: bool
+
+
+class Algo_Config(BaseModel):
+    Updated_Timestamp: int
+    Config: Algo_Config_Config
+    Logging: Algo_Config_Logging
+    Overrides: Algo_Config_Overrides
+
 
 @dataclass(kw_only=True)
 class Flags:
@@ -33,8 +349,8 @@ class Valkey_Stream:
     none_fill: Any = None
 
     async def update(self):
-        r = self.client.get(self.channel)
+        r: str = self.client.get(name=self.channel)  # ty:ignore[invalid-assignment]
-        self.data = json.loads(r) if r is not None else self.none_fill
+        self.data = json.loads(s=r) if r is not None else self.none_fill
 
 
 @dataclass(kw_only=True)
@@ -142,42 +458,57 @@ class Perpetual_Exchange:
     mult: int
     lh_asset: str
     rh_asset: str
+    symbol: str = ''
     symbol_asset_separator: str = ''
+    initial_funding_rate: float = 0
+    fund_rate_at_same_time: bool = False
+    min_price: float = 0
+    min_order_size: float = 0
+    min_lot_size: float = 0
+    min_notional: float = 0
+    buy_ratio: float = 0
+    buy_ratio_std: float = 0
     
-    async def update(self):
+    notional_obj: dict = field(default_factory=dict)
-        await self.Collateral_Updates.update()
+    notional_position: float = 0
-        await self.Order_Updates.update()
+    unrealized_pnl: float = 0
-        await self.Position_Updates.update()
+    just_rejected_count: int = 0
-        await self.Funding_Rate.update()
+    cancel_request_pending: bool = False
+
+    # async def update(self):
+    #     await self.Collateral_Updates.update()
+    #     await self.Order_Updates.update()
+    #     await self.Position_Updates.update()
+    #     await self.Funding_Rate.update()
 
     def __post_init__(self) -> None:
         self.symbol = f'{self.lh_asset.upper()}{self.symbol_asset_separator}{self.rh_asset.upper()}'
 
 
-@dataclass(kw_only=True)
+# @dataclass(kw_only=True)
-class Aster(Perpetual_Exchange):
+# class Aster(Perpetual_Exchange):
-    name: str = 'Aster'
+#     name: str = 'Aster'
-    lh_asset: str = 'ETH'
+#     lh_asset: str = 'ETH'
-    rh_asset: str = 'USDT'
+#     rh_asset: str = 'USDT'
 
-    def __post_init__(self):
+#     def __post_init__(self):
-        super().__post_init__()
+#         super().__post_init__()
-        self.Order_Updates = Order_Updates(Valkey=Valkey_Stream(channel = 'fr_aster_user_balances', none_fills = []))
+#         self.Order_Updates = Order_Updates(Valkey=Valkey_Stream(channel = 'fr_aster_user_balances', none_fills = []))
-        self.Collateral_Updates = Collateral(Valkey=Valkey_Stream(channel = 'fr_aster_user_orders', none_fills = []))
+#         self.Collateral_Updates = Collateral(Valkey=Valkey_Stream(channel = 'fr_aster_user_orders', none_fills = []))
-        self.Position_Updates = Open_Positions(Valkey=Valkey_Stream(channel = 'fr_aster_user_positions', none_fills = []))
+#         self.Position_Updates = Open_Positions(Valkey=Valkey_Stream(channel = 'fr_aster_user_positions', none_fills = []))
-        self.Funding_Rate - Funding_Rate(Valkey=Valkey_Stream(channel = 'fund_rate_aster', none_fills = None))
+#         self.Funding_Rate - Funding_Rate(Valkey=Valkey_Stream(channel = 'fund_rate_aster', none_fills = None))
 
 
-@dataclass(kw_only=True)
+# @dataclass(kw_only=True)
-class Extend(Perpetual_Exchange):
+# class Extend(Perpetual_Exchange):
-    name: str = 'Extended'
+#     name: str = 'Extended'
-    lh_asset: str = 'ETH'
+#     lh_asset: str = 'ETH'
-    rh_asset: str = 'USD'
+#     rh_asset: str = 'USD'
-    symbol_asset_separator: str = '-'
+#     symbol_asset_separator: str = '-'
 
-    def __post_init__(self):
+#     def __post_init__(self):
-        super().__post_init__()
+#         super().__post_init__()
-        self.Order_Updates = Order_Updates(Valkey=Valkey_Stream(channel = 'fr_aster_user_balances', none_fills = []))
+#         self.Order_Updates = Order_Updates(Valkey=Valkey_Stream(channel = 'fr_aster_user_balances', none_fills = []))
-        self.Collateral_Updates = Collateral(Valkey=Valkey_Stream(channel = 'fr_aster_user_orders', none_fills = []))
+#         self.Collateral_Updates = Collateral(Valkey=Valkey_Stream(channel = 'fr_aster_user_orders', none_fills = []))
-        self.Position_Updates = Open_Positions(Valkey=Valkey_Stream(channel = 'fr_aster_user_positions', none_fills = []))
+#         self.Position_Updates = Open_Positions(Valkey=Valkey_Stream(channel = 'fr_aster_user_positions', none_fills = []))
-        self.Funding_Rate - Funding_Rate(Valkey=Valkey_Stream(channel = 'fund_rate_aster', none_fills = None))
+#         self.Funding_Rate - Funding_Rate(Valkey=Valkey_Stream(channel = 'fund_rate_aster', none_fills = None))

modules/utils.py

@@ -2,13 +2,22 @@ import logging
 from dotenv import load_dotenv
 import requests
 import os
+import json
+from decimal import Decimal
 
 load_dotenv()
 
-def upsert_list_of_dicts_by_id(list_of_dicts, new_dict, id='id', seq_check_field: str | None = None):
+
+class JSONEncoder_Decimal(json.JSONEncoder):
+    def default(self, o):
+        if isinstance(o, Decimal):
+            return float(o)
+        return super(JSONEncoder_Decimal, self).default(o)
+
+def upsert_list_of_dicts_by_id(list_of_dicts, new_dict, id='id', seq_check_field: str | None = None, reset_seq_id: bool = False) -> list[dict]:
     for index, item in enumerate(list_of_dicts):
         if item.get(id) == new_dict.get(id):
-            if seq_check_field is not None:
+            if ( seq_check_field is not None ) and ( not(reset_seq_id) ):
                 if item.get(seq_check_field) > new_dict.get(seq_check_field):
                     logging.info('Skipping out of sequence msg')
                     return list_of_dicts
@@ -26,3 +35,24 @@ def send_tg_alert(msg: str):
     response = requests.post(url, json={'text': str(str(msg)[:250])}, timeout=10)
 
     return response.json()
+
+def rec_set_dict(orig_dict, new_dict, allow_new_fields: bool = False) -> dict:
+    for k, v in new_dict.items():
+        if isinstance(v, dict):
+            rec_set_dict(orig_dict=orig_dict[k], new_dict=v)
+        else:
+            if allow_new_fields:
+                orig_dict[k] = v
+            else:
+                if orig_dict.get(k, None) is not None:
+                    orig_dict[k] = v
+                else:
+                    logging.warning(msg=f'rec_set_dict: encountered nonexistent key: "{k}"; skipping')
+    
+    return orig_dict
+
+def symbol_to_aster_fmt(symbol: str) -> str:
+    return (symbol+'T' if symbol[-1].upper()!='T' else symbol).replace('-','').upper()
+
+def symbol_to_extend_fmt(symbol: str) -> str:
+    return (symbol[0:-1] if symbol[-1].upper()=='T' else symbol).replace('-','').upper().split('USD')[0]+'-'+'USD'

ng.py

@@ -1,15 +1,343 @@
 import os
-from nicegui import ui, app
+from nicegui import ui, app, html
-from sqlalchemy import create_engine
+from sqlalchemy import create_engine, text
+# import requests
+import pandas as pd
 import json
+# import time
+# import re
 import valkey
+import asyncio
+from datetime import datetime
+from dataclasses import dataclass, field
+from sqlalchemy.ext.asyncio import create_async_engine
+from typing import AsyncContextManager
+# from random import random
+# from nicegui_modules import data
+# from nicegui_modules import ui_components
 
-
+ALLOW_BODY_SCROLL: bool = True
-VALKEY_R = valkey.Valkey(host='localhost', port=6379, db=0, decode_responses=True)
+LOOKBACK: int = 60
+LOOKBACK_RT_TV_MAX_POINTS: int = 3000
+REFRESH_INTERVAL_SEC: float = 10
+REFRESH_INTERVAL_RT_SEC: float = 1/10
+# CON: AsyncContextManager
+# ENGINE = create_async_engine('mysql+asyncmy://root:pwd@localhost/fund_rate')
+ENGINE = create_engine('mysql+pymysql://root:pwd@localhost/fund_rate')
 
 
 
-def root():
+VALKEY = valkey.Valkey(host='localhost', port=6379, db=0, decode_responses=True)
+
+CHARTS = [
+    {
+        'type': 'AREA',
+        'autoscaleInfoProvider': False,
+        'data': [],
+        'options': {
+            'color': '#94fcdf',
+            'priceScaleId': 'right',
+            'topColor': '#94fcdf', 
+            'bottomColor': 'rgba(112, 249, 210, 0.28)',
+            'invertFilledArea': True
+        }
+    },
+    {
+        'type': 'AREA',
+        'autoscaleInfoProvider': False,
+        'data': [],
+        'options': {
+            'color': '#dd7525',
+            'priceScaleId': 'right',
+            'topColor': '#94fcdf', 
+            'bottomColor': 'rgba(249, 167, 112, 0.28)',
+            'invertFilledArea': False
+        },
+    },
+    {
+        'type': 'LINE',
+        'autoscaleInfoProvider': [-0.1, 0.1],
+        'data': [],
+        'options': {
+            'color': '#ea0707',
+            'priceScaleId': 'left',
+        },
+    },    
+    {
+        'type': 'LINE',
+        'autoscaleInfoProvider': False,
+        'data': [],
+        'options': {
+            'color': '#009b12',
+            'priceScaleId': 'left',
+        },
+    },    
+    {
+        'type': 'LINE',
+        'autoscaleInfoProvider': False,
+        'data': [],
+        'options': {
+            'color': '#ffffff',
+            'priceScaleId': 'left',
+        },
+    },    
+]
+
+CHARTS_OPTIONS = { 
+    'crosshair': 'NORMAL',
+    'autoSize': True,
+    'toolbox': True,
+    'timeScale': {
+        'timeVisible': True,   # // Shows HH:mm on x-axis
+        'secondsVisible': True # // Optional: show seconds
+    },
+    'rightPriceScale': {
+        'visible': True,
+        'autoScale': True
+    },
+    'leftPriceScale': {
+        'visible': True
+    },        
+    'layout': {
+        'background': { 'type': 'solid', 'color': '#222' },
+        'textColor': '#DDD',
+    },
+    'grid': {
+        'vertLines': {
+            'color': '#e1e1e1', # // Set vertical line color
+            'visible': True,
+            'style': 2, # // 0: Solid, 1: Dashed, 2: Dotted, 3: LargeDashed, 4: SparseDotted
+        },
+        'horzLines': {
+            'color': '#e1e1e1', # // Set horizontal line color
+            'visible': True,
+            'style': 2,
+        },
+    }
+}
+
+### Data ###
+async def get_bfr_master_data() -> pd.DataFrame:
+    df = pd.DataFrame(json.loads(VALKEY.get('fr_engine_best_fund_rate_master')))  # ty:ignore[invalid-argument-type]    
+    df.reset_index(drop=True)
+    df['id'] = df.index
+    return df
+
+async def get_trades_hist() -> pd.DataFrame:
+    start_ts = (round(datetime.now().timestamp()*1000)-(60*60*24*1000))
+    ### ASTER ###
+    aster_orders = text(f'''
+    SELECT *
+    FROM fr_aster_user_order_trade
+    WHERE timestamp_arrival > {start_ts}
+    ''')
+    df_aster_orders = pd.read_sql(aster_orders, con=ENGINE)
+    if len(df_aster_orders) < 1:
+        return pd.DataFrame()
+    df_aster_orders['timestamp_dt'] = pd.to_datetime(df_aster_orders['timestamp_transaction'], unit='ms')
+    df_aster_orders_fill = df_aster_orders.loc[df_aster_orders['execution_type']=='TRADE',:]
+    df_aster_orders_fill = df_aster_orders_fill[['timestamp_transaction','order_trade_time_ts','timestamp_dt','order_id','trade_id','client_order_id','order_status','side','last_filled_qty','filled_accumulated_qty','commission','last_filled_price','realized_profit']].reset_index(drop=True)
+
+    df_aster_trades = df_aster_orders_fill.groupby('order_id').agg({'timestamp_transaction': 'first','order_trade_time_ts':'last','order_status':'last','side':'last','last_filled_qty':'sum','filled_accumulated_qty':'last','commission':'sum','last_filled_price':'mean','realized_profit':'sum'}).reset_index()
+    df_aster_trades['is_mkt_maker'] = df_aster_trades['commission'] == 0.00
+    df_aster_trades['timestamp_ts'] = pd.to_datetime(df_aster_trades['order_trade_time_ts'], unit='ms')
+
+    df_aster_trades = df_aster_trades.rename({'order_status':'status','filled_accumulated_qty':'filled_qty','commission':'payed_fee','last_filled_price':'price'}, axis=1)
+
+    ### EXTEND ###
+    # Load and Transform Orders
+    extend_orders = text(f'''
+        SELECT *
+        FROM fr_extended_user_order
+        WHERE timestamp_arrival > {start_ts}
+    ''')
+    df_extend_orders = pd.read_sql(extend_orders, con=ENGINE)
+    if len(df_extend_orders) < 1:
+        return pd.DataFrame()
+    df_extend_orders['timestamp_dt'] = pd.to_datetime(df_extend_orders['updated_time_ts'], unit='ms')
+    df_extend_orders_fill = df_extend_orders.loc[df_extend_orders['status'].isin(['FILLED','PARTIALLY_FILLED']),:]
+    df_extend_orders_fill = df_extend_orders_fill[['created_time_ts','updated_time_ts','timestamp_dt','order_id','external_id','status','side','qty','filled_qty','payed_fee','price','averagePrice']].reset_index(drop=True)
+
+    # Trades
+    df_extend_trades = df_extend_orders_fill.groupby('order_id').agg({'created_time_ts':'first','updated_time_ts':'last','status': 'last','side': 'last', 'filled_qty':'last','payed_fee':'sum','price':'last'}).reset_index()
+    df_extend_trades['duration_sec_ast'] = ( df_extend_trades['updated_time_ts'] - df_extend_trades['created_time_ts'] ) / 1000
+    df_extend_trades['is_mkt_maker'] = df_extend_trades['payed_fee'] == 0.00
+    df_extend_trades['timestamp_ts'] = pd.to_datetime(df_extend_trades['updated_time_ts'], unit='ms')
+    
+    def tie_trades_together_get_extend_from_aster(row):
+        row = row.to_frame().T
+        row.index=[1]
+
+        extend_row = df_extend_trades[['order_id','timestamp_ts','status','side','filled_qty','payed_fee','price','is_mkt_maker']].loc[df_extend_trades['timestamp_ts']>row['timestamp_ts'].iloc[0],:].iloc[0]
+        extend_row = extend_row.to_frame().T
+        extend_row.index=[1]
+
+        return_row = row.merge(extend_row, left_index=True, right_index=True, suffixes=('_ast','_ext'))
+
+        return return_row.iloc[0]
+
+    df_comb_trades = df_aster_trades[['order_id','timestamp_ts','status','side','filled_qty','payed_fee','price','is_mkt_maker']].apply(tie_trades_together_get_extend_from_aster, axis=1)
+    df_comb_trades['buy_price'] = df_comb_trades['price_ast'].where(df_comb_trades['side_ast']=='BUY', df_comb_trades['price_ext'])
+    df_comb_trades['sell_price'] = df_comb_trades['price_ast'].where(df_comb_trades['side_ast']=='SELL', df_comb_trades['price_ext'])
+    df_comb_trades['buy_qty'] = df_comb_trades['filled_qty_ast'].where(df_comb_trades['side_ast']=='BUY', df_comb_trades['filled_qty_ext'])
+    df_comb_trades['sell_qty'] = df_comb_trades['filled_qty_ast'].where(df_comb_trades['side_ast']=='SELL', df_comb_trades['filled_qty_ext'])
+    df_comb_trades['buy_side'] = df_comb_trades['order_id_ast'].where(df_comb_trades['side_ast']=='BUY', df_comb_trades['order_id_ext'])
+    df_comb_trades['buy_side'] = df_comb_trades['order_id_ast'] == df_comb_trades['buy_side']
+    df_comb_trades['buy_side'] = df_comb_trades['buy_side'].replace(True, 'ASTER').replace(False,'EXTEND')
+
+    df_comb_trades['per_trade_pnl'] = ( ( df_comb_trades['sell_price'] - df_comb_trades['buy_price'] ) * df_comb_trades['sell_qty'] ) - df_comb_trades['payed_fee_ast'] - df_comb_trades['payed_fee_ext']
+    df_comb_trades['per_trade_pnl_pct'] = ( (df_comb_trades['sell_price']*df_comb_trades['sell_qty']) - (df_comb_trades['buy_price']*df_comb_trades['buy_qty']) ) / (df_comb_trades['buy_price']*df_comb_trades['buy_qty'])
+    
+        
+    df = df_comb_trades.apply(lambda x: x.dt.strftime('%Y-%m-%d %H:%M:%S.%f') if hasattr(x, 'dt') else x)
+    
+    df.reset_index(drop=True)
+    df['id'] = df.index
+    return df
+
+### Utils ###
+def update_body_scroll(e=None, bool_override=False):
+    if e is None:
+        if bool_override:
+            ui.query('body').style('height: 100%; overflow-y: auto;')
+        else:
+            ui.query('body').style('height: 100%; overflow-y: hidden;')
+    else:
+        if e.value:
+            ui.query('body').style('height: 100%; overflow-y: auto;')
+        else:
+            ui.query('body').style('height: 100%; overflow-y: hidden;')
+
+### Callbacks ###
+async def update_tv():
+    series_update_aster_tob = json.loads(VALKEY.get('fut_ticker_aster'))  # ty:ignore[invalid-argument-type]
+    series_update_extend_tob = json.loads(VALKEY.get('fut_ticker_extended')) # ty:ignore[invalid-argument-type]
+    series_update_algo_status = json.loads(VALKEY.get('algo_status')) # ty:ignore[invalid-argument-type] 
+    
+    timestamp_aster_tob = round( ( series_update_aster_tob['timestamp_transaction'] / 1000 ) , 2)
+    timestamp_extend_tob = round( ( series_update_extend_tob['timestamp_msg'] / 1000 ) , 2)
+    timestamp_algo_status = round( ( series_update_algo_status['last_update_ts_ms'] / 1000 ) , 2)
+    
+    value_aster_tob = ( float(series_update_aster_tob['best_ask_px']) + float(series_update_aster_tob['best_bid_px']) ) / 2 
+    value_extend_tob = ( float(series_update_extend_tob['best_ask_px']) + float(series_update_extend_tob['best_bid_px']) ) / 2
+    value_algo_model_ratio = float(series_update_algo_status['model_ratio'])*1_000
+    value_algo_current_ratio = float(series_update_algo_status['current_ratio'])*1_000
+    value_algo_expected_alpha = float(series_update_algo_status['expected_alpha'])*1_000
+
+    data_list = [
+        {
+            'timestamp': timestamp_aster_tob,
+            'value': value_aster_tob,
+        },
+        {
+            'timestamp': timestamp_extend_tob,
+            'value': value_extend_tob,
+        },
+        {
+            'timestamp': timestamp_algo_status,
+            'value': value_algo_model_ratio,
+        },
+        {
+            'timestamp': timestamp_algo_status,
+            'value': value_algo_current_ratio,
+        },
+        {
+            'timestamp': timestamp_algo_status,
+            'value': value_algo_expected_alpha,
+        },
+    ]
+
+    ui.run_javascript(f'await update_tv(data_list={data_list}, lookback_max_points={LOOKBACK_RT_TV_MAX_POINTS});')
+
+async def create_bfr_aggrid() -> ui.aggrid:
+    df = await get_bfr_master_data()
+    col_extras = {
+        'symbol_ast': {
+            'editable': False, 
+            'sortable': True
+        }
+    }
+    cols = [ {'field': v, **col_extras.get(v, {})} for v in df.columns ]
+    rows = df.to_dict(orient='records')
+    grid = ui.aggrid(
+        {
+            'columnDefs': cols,
+            'rowData': rows,
+            'autoSizeStrategy': {
+                'type': 'fitCellContents',
+            },
+            # 'rowSelection': {'mode': 'multiRow'},
+            # 'stopEditingWhenCellsLoseFocus': True,
+        }
+    ).classes('auto-fit flex-grow w-full col-span-2 md:col-span-1')
+    
+    return grid
+
+async def create_pnl_aggrid() -> ui.aggrid:
+    df = await get_trades_hist()
+    col_extras = {}
+    cols = [ {'field': v, **col_extras.get(v, {})} for v in df.columns ]
+    rows = df.to_dict(orient='records')
+    grid = ui.aggrid(
+        {
+            'columnDefs': cols,
+            'rowData': rows,
+            'autoSizeStrategy': {
+                'type': 'fitCellContents',
+            },
+            # 'rowSelection': {'mode': 'multiRow'},
+            # 'stopEditingWhenCellsLoseFocus': True,
+        }
+    ).classes('auto-fit flex-grow w-full col-span-2 md:col-span-1')
+    
+    return grid
+
+
+### Pages ###
+async def rt_chart_page():
+    global LOOKBACK
+    
+    LOOKBACK = app.storage.user.get('lookback', LOOKBACK)
+    timer_tv = ui.timer(REFRESH_INTERVAL_RT_SEC, update_tv)
+    # timer_sql = ui.timer(REFRESH_INTERVAL_SEC)
+    
+    # ui.query('.q-page').classes('flex flex-col h-screen')
+
+    # with ui.row():
+    #     with ui.column():
+    #         ui.switch('☸︎', value=ALLOW_BODY_SCROLL, on_change=lambda e: update_body_scroll(e))
+    #     with ui.column():
+    #         ui.switch('▶️', value=True).bind_value_to(timer, 'active')
+    #     with ui.column().style('position: absolute; right: 20px; font-family: monospace; align-self: center;'):
+    #         ui.label('Atwater Trading - Funding Rate')
+    ui.query('.nicegui-content').classes('p-0 w-full')
+    ui.query('.q-page').classes('flex')
+
+    with ui.grid(columns=2, rows=2).classes('h-screen w-full flex-grow gap-2 auto-fit '):
+        # aggrid = await create_bfr_aggrid()
+        # with ui.element(tag='div').classes('auto-fit flex-grow w-full').style("height:100%; width: 100%;"):
+            # with ui.tabs().classes('w-full') as tabs:
+            #         one = ui.tab('One').classes('auto-fit flex-grow w-full').style("height:100%; width: 100%;")
+            #         two = ui.tab('Two').classes('auto-fit flex-grow w-full').style("height:100%; width: 100%;")
+            # with ui.tab_panels(tabs, value=two).classes('auto-fit flex-grow w-full').style("height:100%; width: 100%;"):
+            #     with ui.tab_panel(one).classes('auto-fit flex-grow w-full').style("height:100%; width: 100%;"):
+            #         ui.label('First tab')
+            #     with ui.tab_panel(two).classes('auto-fit flex-grow w-full').style("height:100%; width: 100%;"):
+        ui.html('<div id="tv" style="height:100%; width: 100%;"></div>', sanitize=False).classes('auto-fit flex-grow w-full col-span-2 md:col-span-1')
+        ui.run_javascript(f'await create_tv(charts_list={CHARTS}, create_chart_options={CHARTS_OPTIONS});')
+        
+        with ui.element(tag='div').classes('col-span-2').style("height:100%; width: 100%;"):
+            with ui.tabs().props('align=justify').classes('justify-start') as tabs:
+                tab_pnl = ui.tab('PnL').classes('justify-start')
+                tab_bfr = ui.tab('BFR').classes('justify-start')
+            with ui.tab_panels(tabs, value=tab_pnl).classes('w-full').style("height:100%; width: 100%;"):
+                with ui.tab_panel(tab_pnl):
+                    ag_pnl = await create_pnl_aggrid()
+                with ui.tab_panel(tab_bfr):
+                    ag_bfr = await create_bfr_aggrid()
+
+
+async def root():
     app.add_static_files(max_cache_age=0, url_path='/static', local_directory=os.path.join(os.path.dirname(__file__), 'nicegui_modules/static'))
     ui.add_head_html('''
     <meta name="darkreader-lock">
@@ -18,19 +346,13 @@ def root():
     <script src="/static/script.js"></script>
     '''
     )
+
     # ui.add_head_html('<meta name="darkreader-lock">')
-    # update_body_scroll(bool_override=ALLOW_BODY_SCROLL)
+    update_body_scroll(bool_override=ALLOW_BODY_SCROLL)
 
     ui.sub_pages({
-        '/': controls_grid,
+        '/': rt_chart_page,
     }).classes('w-full')
 
 
-async def controls_grid():
+ui.run(root, storage_secret="123ABC", reload=True, dark=True, title='Atwater Trading', port=8060) 
-    with ui.grid(columns=16).classes('w-full gap-0 auto-fit'):
-        with ui.card().tight().classes('w-full col-span-full no-shadow border border-black-200').style('overflow: auto;'):
-            ui.html('<div id="tv" style="width:100%; height:800px;"></div>', sanitize=False).classes('w-full')
-            ui.run_javascript('await create_tv();')
-
-
-ui.run(root, storage_secret="123ABC", reload=True, dark=True, title='Atwater_Trading')

nicegui_modules/static/script.js

@@ -0,0 +1,170 @@
+async function waitForVariable(variableName, timeout = 5000) {
+  const startTime = Date.now();
+  while (typeof window[variableName] === 'undefined') {
+    if (Date.now() - startTime > timeout) {
+      throw new Error(`Variable '${variableName}' not defined within ${timeout}ms`);
+    }
+    await new Promise(resolve => setTimeout(resolve, 100));
+  }
+  console.log(`Variable '${variableName}' is now defined.`);
+}
+
+async function update_tv(data_list, lookback_max_points) {
+    data_list.forEach(function (item, index) {
+        // console.log(item, index);
+        window.charts_arr[index].data.push({ time: item.timestamp, value: item.value });
+        window.charts_arr[index].series.update({ time: item.timestamp, value: item.value });
+
+        if (window.charts_arr[index].series.data().length > lookback_max_points) {
+            window.charts_arr[index].series.setData(window.charts_arr[index].series.data().slice(-lookback_max_points));
+        };
+    });
+
+    // midPriceLine.applyOptions({
+    //     price: data_dict.mid_px,
+    //     color: '#c78228',
+    //     lineWidth: 3,
+    //     lineStyle: LightweightCharts.LineStyle.Dashed,
+    //     axisLabelVisible: true,
+    // });
+
+    window.chart.timeScale().scrollToRealTime();
+    window.chart.timeScale().fitContent();
+    const currentRange = window.chart.timeScale().getVisibleLogicalRange();   
+    window.chart.timeScale().setVisibleLogicalRange(currentRange);
+};
+
+
+async function create_tv(charts_list, create_chart_options) {
+    const container = document.getElementById('tv');
+    
+    if (create_chart_options.crosshair == 'NORMAL') {
+        create_chart_options.crosshair = { mode: LightweightCharts.CrosshairMode.Normal }
+    };
+    window.chart = LightweightCharts.createChart(container, create_chart_options);
+    window.charts_arr = [];
+    
+    charts_list.forEach(function (item, index) {
+        // console.log(item, index);
+        charts_dict = {};
+        if ((Array.isArray(item.autoscaleInfoProvider) && item.autoscaleInfoProvider.length !== 0)) {
+            item.options.autoscaleInfoProvider = () => ({
+                priceRange: {
+                minValue: item.autoscaleInfoProvider[0],
+                maxValue: item.autoscaleInfoProvider[1]
+                }
+            })
+        };
+        if (item.type == "AREA" ) {
+            charts_dict.series = chart.addSeries(LightweightCharts.AreaSeries, item.options);
+        } else {
+            charts_dict.series = chart.addSeries(LightweightCharts.LineSeries, item.options);
+        };
+        charts_dict.data = [];
+        charts_dict.series.setData(charts_dict.data);
+        window.charts_arr.push(charts_dict);
+    });
+
+    window.chart.timeScale().fitContent();
+
+    // Handle responsiveness: Resize chart when container size changes
+    const resizeObserver = new ResizeObserver(entries => {{
+        for (let entry of entries) {{
+            window.chart.applyOptions({ 
+                width: entry.contentRect.width, 
+                height: entry.contentRect.height 
+            });
+        }}
+    }});
+    resizeObserver.observe(container);
+
+    console.log("TV Created!")
+
+    // window.midPriceLine_Config = {
+    //     price: 0,
+    //     color: '#c78228',
+    //     lineWidth: 3,
+    //     lineStyle: LightweightCharts.LineStyle.Dashed,
+    //     axisLabelVisible: false,
+    // };
+    // window.midPriceLine = window.lineSeries.createPriceLine(midPriceLine_Config);
+
+    
+    // Create and style the tooltip html element
+    // const container = document.getElementById('tv');
+
+    // window.toolTipWidth = 200;
+
+    // const toolTip = document.createElement('div');
+    // toolTip.style = `width: ${window.toolTipWidth}px; height: 100%; position: absolute; display: none; padding: 8px; box-sizing: border-box; font-size: 12px; text-align: left; z-index: 1000; top: 12px; left: 12px; pointer-events: none; border-radius: 4px 4px 0px 0px; border-bottom: none; box-shadow: 0 2px 5px 0 rgba(117, 134, 150, 0.45);font-family: -apple-system, BlinkMacSystemFont, 'Trebuchet MS', Roboto, Ubuntu, sans-serif; -webkit-font-smoothing: antialiased; -moz-osx-font-smoothing: grayscale;`;
+    // toolTip.style.background = `rgba(${'0, 0, 0'}, 0.25)`;
+    // toolTip.style.color = 'white';
+    // toolTip.style.borderColor = 'rgba( 239, 83, 80, 1)';
+    // container.appendChild(toolTip);
+
+    // // update tooltip
+    // window.chart.subscribeCrosshairMove(async param => {
+        
+    //     if (
+    //         param.point === undefined ||
+    //         !param.time ||
+    //         param.point.x < 0 ||
+    //         param.point.x > container.clientWidth ||
+    //         param.point.y < 0 ||
+    //         param.point.y > container.clientHeight
+    //     ) {
+    //         toolTip.style.display = 'none';
+    //     } else {
+            
+    //         // toolTip.style.height = '100%';
+    //         toolTip.style.alignContent = 'center';
+
+    //         const dateStr = new Date(param.time*1000).toISOString();
+            
+    //         let data = await param.seriesData.get(window.lineSeries);
+    //         if (data === undefined) {
+    //             data = {}
+    //             data.value = 0
+    //             console.log('data is UNDEFINED, SETTING TO 0')
+    //         };
+            
+    //         let data_b = await param.seriesData.get(window.lineSeries_b);
+    //         if (data_b === undefined) {
+    //             data_b = {}
+    //             data_b.value = 0
+    //             console.log('data is UNDEFINED, SETTING TO 0')
+    //         };
+
+    //         const value_px = data.value
+    //         const value_px_b = window.data_b.value
+    //         const value_px_c = window.data_c.value
+    //         // const value_px_tgt = window.data_tgt.value
+            
+    //         toolTip.style.display = 'block';
+    //         // <div style="color: ${'rgba( 239, 83, 80, 1)'}">
+    //         //     Atwater Trading
+    //         // </div>
+    //         toolTip.innerHTML = `
+    //         <div style="font-size: 24px; margin: 4px 0px; color: ${'white'}">
+    //             Chainlink: ${Math.round(100 * value_px) / 100}
+    //             Binance: ${Math.round(100 * value_px_b) / 100}
+    //             </div>
+    //             <div style="color: ${'white'}">
+    //                 ${dateStr}
+    //             </div>
+    //         `;
+
+    //         let left = param.point.x; // relative to timeScale
+    //         const timeScaleWidth = chart.timeScale().width();
+    //         const priceScaleWidth = chart.priceScale('left').width();
+    //         const halfTooltipWidth = toolTipWidth / 2;
+    //         left += priceScaleWidth - halfTooltipWidth;
+    //         left = Math.min(left, priceScaleWidth + timeScaleWidth - toolTipWidth);
+    //         left = Math.max(left, priceScaleWidth);
+
+    //         toolTip.style.left = left + 'px';
+    //         toolTip.style.top = 0 + 'px';
+    //     }
+    // });
+
+};

nicegui_modules/static/styles.css

@@ -0,0 +1,33 @@
+/* Sticky Quasar Table for Dark Mode */
+.table-sticky-dark .q-table__top,
+.table-sticky-dark .q-table__bottom,
+.table-sticky-dark thead tr:first-child th {
+    background-color: black;
+}
+.table-sticky-dark thead tr th {
+    position: sticky;
+    z-index: 1;
+}
+.table-sticky-dark thead tr:first-child th {
+    top: 0;
+}
+.table-sticky-dark tbody {
+    scroll-margin-top: 48px;
+}
+
+/* Sticky Quasar Table for Light Mode */
+/* .table-sticky-light .q-table__top,
+.table-sticky-light .q-table__bottom,
+.table-sticky-light thead tr:first-child th {
+    background-color: rgb(229, 223, 223);
+}
+.table-sticky-light thead tr th {
+    position: sticky;
+    z-index: 1;
+}
+.table-sticky-light thead tr:first-child th {
+    top: 0;
+}
+.table-sticky-light tbody {
+    scroll-margin-top: 48px;
+} */

order_engine.ipynb

@@ -0,0 +1,272 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "id": "68966247",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import requests\n",
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import json\n",
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "02cd5305",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "### Extended Trades History ###\n",
+    "candleType = 'trades'\n",
+    "market = 'ETH-USD'\n",
+    "params = {\n",
+    "    'interval': \"1m\",\n",
+    "    'limit': 100,\n",
+    "}\n",
+    "r = requests.get(f'https://api.starknet.extended.exchange/api/v1/info/candles/{market}/{candleType}', params=params)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "id": "5603b04d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "### Aster Trades History ###\n",
+    "params = {\n",
+    "    'symbol': \"ETHUSDT\",\n",
+    "    'limit': 1000,\n",
+    "}\n",
+    "r = requests.get('https://fapi.asterdex.com/fapi/v3/trades', params=params)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "id": "a3ad1819",
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>id</th>\n",
+       "      <th>price</th>\n",
+       "      <th>qty</th>\n",
+       "      <th>quoteQty</th>\n",
+       "      <th>time</th>\n",
+       "      <th>isBuyerMaker</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>74506547</td>\n",
+       "      <td>2311.02</td>\n",
+       "      <td>0.044</td>\n",
+       "      <td>101.68</td>\n",
+       "      <td>2026-04-27 14:22:45.650</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>74506548</td>\n",
+       "      <td>2311.00</td>\n",
+       "      <td>0.004</td>\n",
+       "      <td>9.24</td>\n",
+       "      <td>2026-04-27 14:22:45.650</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>74506549</td>\n",
+       "      <td>2310.91</td>\n",
+       "      <td>0.003</td>\n",
+       "      <td>6.93</td>\n",
+       "      <td>2026-04-27 14:22:45.650</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>74506550</td>\n",
+       "      <td>2310.90</td>\n",
+       "      <td>0.004</td>\n",
+       "      <td>9.24</td>\n",
+       "      <td>2026-04-27 14:22:45.650</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>74506551</td>\n",
+       "      <td>2310.80</td>\n",
+       "      <td>0.004</td>\n",
+       "      <td>9.24</td>\n",
+       "      <td>2026-04-27 14:22:45.700</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>995</th>\n",
+       "      <td>74507542</td>\n",
+       "      <td>2312.10</td>\n",
+       "      <td>0.004</td>\n",
+       "      <td>9.24</td>\n",
+       "      <td>2026-04-27 14:34:12.500</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>996</th>\n",
+       "      <td>74507543</td>\n",
+       "      <td>2312.18</td>\n",
+       "      <td>2.442</td>\n",
+       "      <td>5646.34</td>\n",
+       "      <td>2026-04-27 14:34:13.443</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>997</th>\n",
+       "      <td>74507544</td>\n",
+       "      <td>2312.24</td>\n",
+       "      <td>10.099</td>\n",
+       "      <td>23351.31</td>\n",
+       "      <td>2026-04-27 14:34:13.600</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>998</th>\n",
+       "      <td>74507545</td>\n",
+       "      <td>2312.13</td>\n",
+       "      <td>3.120</td>\n",
+       "      <td>7213.84</td>\n",
+       "      <td>2026-04-27 14:34:14.568</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>999</th>\n",
+       "      <td>74507546</td>\n",
+       "      <td>2312.19</td>\n",
+       "      <td>6.228</td>\n",
+       "      <td>14400.31</td>\n",
+       "      <td>2026-04-27 14:34:15.988</td>\n",
+       "      <td>True</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>1000 rows × 6 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "           id    price     qty  quoteQty                    time  isBuyerMaker\n",
+       "0    74506547  2311.02   0.044    101.68 2026-04-27 14:22:45.650          True\n",
+       "1    74506548  2311.00   0.004      9.24 2026-04-27 14:22:45.650          True\n",
+       "2    74506549  2310.91   0.003      6.93 2026-04-27 14:22:45.650          True\n",
+       "3    74506550  2310.90   0.004      9.24 2026-04-27 14:22:45.650          True\n",
+       "4    74506551  2310.80   0.004      9.24 2026-04-27 14:22:45.700          True\n",
+       "..        ...      ...     ...       ...                     ...           ...\n",
+       "995  74507542  2312.10   0.004      9.24 2026-04-27 14:34:12.500          True\n",
+       "996  74507543  2312.18   2.442   5646.34 2026-04-27 14:34:13.443          True\n",
+       "997  74507544  2312.24  10.099  23351.31 2026-04-27 14:34:13.600          True\n",
+       "998  74507545  2312.13   3.120   7213.84 2026-04-27 14:34:14.568          True\n",
+       "999  74507546  2312.19   6.228  14400.31 2026-04-27 14:34:15.988          True\n",
+       "\n",
+       "[1000 rows x 6 columns]"
+      ]
+     },
+     "execution_count": 34,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "l = json.loads(r.text)\n",
+    "df = pd.DataFrame(l)\n",
+    "df['time'] = pd.to_datetime(df['time'], unit='ms')\n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3c908942",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "60f4608a",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "76624896",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5ade3c15",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "py_313",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.13.13"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

pyproject.toml

@@ -0,0 +1,2 @@
+[tool.ty.src]
+exclude = ["*.ipynb"]

requirements.txt

@@ -23,3 +23,6 @@ nicegui
 x10-python-trading-starknet
 eth-keys
 eth-account
+pydantic
+plotly
+docker

ws_aster.py

@@ -5,7 +5,7 @@ import socket
 import traceback
 from datetime import datetime
 from typing import AsyncContextManager
-
+import time
 import numpy as np
 import pandas as pd
 import requests.packages.urllib3.util.connection as urllib3_cn  # type: ignore
@@ -15,7 +15,9 @@ from sqlalchemy.ext.asyncio import create_async_engine
 import valkey
 import os
 from dotenv import load_dotenv
-
+import modules.db as db
+import modules.aster_db as aster_db
+import sys
 
 ### Allow only ipv4 ###
 def allowed_gai_family():
@@ -23,103 +25,85 @@ def allowed_gai_family():
 urllib3_cn.allowed_gai_family = allowed_gai_family
 
 ### Database ###
-USE_DB: bool = False
+USE_DB: bool = True
 USE_VK: bool = True
+CON: AsyncContextManager
+VAL_KEY: valkey.Valkey
 VK_FUND_RATE = 'fund_rate_aster'
 VK_TICKER = 'fut_ticker_aster'
-CON: AsyncContextManager | None = None
+VK_LAST_TRADE = 'fut_last_trade_aster'
-VAL_KEY = None
 
 ### Logging ###
 load_dotenv()
-LOG_FILEPATH: str = os.getenv("LOGS_PATH") + '/Fund_Rate_Aster.log'
+LOG_FILEPATH: str = f'{os.getenv(key="LOGS_PATH")}/Fund_Rate_Aster.log'
 
 ### CONSTANTS ###
-SYMBOL: str = 'ETHUSDT'
+SYMBOL: str = 'ENAUSDT'
-STREAM_MARKPRICE: str = f'{SYMBOL.lower()}@markPrice@1s'
+
+STREAM_MARKPRICE: str = '!markPrice@arr@1s'
+# STREAM_MARKPRICE: str = f'{SYMBOL.lower()}@markPrice@1s'
 STREAM_BOOKTICKER: str = f'{SYMBOL.lower()}@bookTicker'
+STREAM_TRADES: str = f'{SYMBOL.lower()}@aggTrade'
 
 ### Globals ###
-WSS_URL = f"wss://fstream.asterdex.com/stream?streams={STREAM_MARKPRICE}/{STREAM_BOOKTICKER}"
+WSS_URL: str = f"wss://fstream.asterdex.com/stream?streams={STREAM_MARKPRICE}/{STREAM_BOOKTICKER}/{STREAM_TRADES}"
-# WSS_URL = f"wss://fstream.asterdex.com/stream?streams={STREAM_MARKPRICE}"
+ALLOW_SYMBOL_CHG: bool = True
 
-# HIST_TRADES = np.empty((0, 3))
+### Funcs ###
-# HIST_TRADES_LOOKBACK_SEC = 6
+async def subscribe_streams(websocket, streams: list[str]) -> None:
+    logging.info(f'Trying to sub: {streams}')
+    msg = {
+        "method": "SUBSCRIBE", 
+        "params": streams,
+        "id": int(round(number=datetime.now().timestamp()*1000)) 
+    }
+    await websocket.send(json.dumps(obj=msg))
+    logging.info(f'Success sub: {streams}')
 
-# ### Database Funcs ###
-# async def create_rtds_btcusd_table(
-#         CON: AsyncContextManager,
-#         engine: str = 'mysql', # mysql | duckdb
-#     ) -> None:
-#     if CON is None:
-#         logging.info("NO DB CONNECTION, SKIPPING Create Statements")
-#     else:
-#         if engine == 'mysql':
-#             logging.info('Creating Table if Does Not Exist: binance_btcusd_trades')
-#             await CON.execute(text("""
-#             CREATE TABLE IF NOT EXISTS binance_btcusd_trades (
-#                 timestamp_arrival BIGINT,
-#                 timestamp_msg BIGINT,
-#                 timestamp_value BIGINT,
-#                 value DOUBLE,
-#                 qty DOUBLE
-#             );
-#             """))
-#             await CON.commit()
-#         else:
-#             raise ValueError('Only MySQL engine is implemented')
 
-# async def insert_rtds_btcusd_table(
+async def unsubscribe_streams(websocket, streams: list[str]) -> None:
-#         timestamp_arrival: int,
+    logging.info(f'Trying to unsub: {streams}')
-#         timestamp_msg: int,
+    msg = {
-#         timestamp_value: int,
+        "method": "UNSUBSCRIBE", 
-#         value: float,
+        "params": streams,
-#         qty: float,
+        "id": int(round(number=datetime.now().timestamp()*1000)) 
-#         CON: AsyncContextManager,
+    }
-#         engine: str = 'mysql', # mysql | duckdb
+    await websocket.send(json.dumps(obj=msg))
-#     ) -> None:
+    logging.info(f'Success unsub: {streams}')
-#     params={
+
-#         'timestamp_arrival': timestamp_arrival,
-#         'timestamp_msg': timestamp_msg,
-#         'timestamp_value': timestamp_value,
-#         'value': value,
-#         'qty': qty,
-#     }
-#     if CON is None:
-#         logging.info("NO DB CONNECTION, SKIPPING Insert Statements")
-#     else:
-#         if engine == 'mysql':
-#             await CON.execute(text("""
-#             INSERT INTO binance_btcusd_trades 
-#             (
-#                 timestamp_arrival,
-#                 timestamp_msg,
-#                 timestamp_value,
-#                 value,
-#                 qty
-#             )
-#             VALUES 
-#                 (
-#                 :timestamp_arrival,
-#                 :timestamp_msg,
-#                 :timestamp_value,
-#                 :value,
-#                 :qty
-#                 )
-#             """),
-#             parameters=params
-#             )
-#             await CON.commit()
-#         else:
-#             raise ValueError('Only MySQL engine is implemented')
 
 ### Websocket ###
 async def ws_stream():
-    async for websocket in websockets.connect(WSS_URL):
+    global SYMBOL
-        logging.info(f"Connected to {WSS_URL}")
+    global STREAM_MARKPRICE
+    global STREAM_BOOKTICKER
+    global STREAM_TRADES
 
+    async for websocket in websockets.connect(WSS_URL, ping_interval=5):
+        logging.info(msg=f"Connected to {WSS_URL}")
         try:
             async for message in websocket:
+                ### Update Symbol if Algo Outputs Change ###
+                if ALLOW_SYMBOL_CHG:
+                    fr_algo_working_symbol = VAL_KEY.get(name='fr_algo_working_symbol')
+                    if not fr_algo_working_symbol:
+                        logging.critical(f'fr_algo_working_symbol is empty - killing: {fr_algo_working_symbol}')
+                        sys.exit(1)
+                        
+                    best_symbol_by_exchange: dict = json.loads(fr_algo_working_symbol)  # ty:ignore[invalid-argument-type]
+                    
+                    best_symbol: str = best_symbol_by_exchange['ASTER']['symbol']
+                    if best_symbol != SYMBOL:
+                        logging.info(f'Symbol Change: {SYMBOL} -> {best_symbol}')
+                        SYMBOL = best_symbol
+                        await unsubscribe_streams(websocket = websocket, streams=[STREAM_BOOKTICKER,STREAM_TRADES])
+                        
+                        # STREAM_MARKPRICE = f'{SYMBOL.lower()}@markPrice@1s'
+                        STREAM_BOOKTICKER = f'{SYMBOL.lower()}@bookTicker'
+                        STREAM_TRADES = f'{SYMBOL.lower()}@aggTrade'
+                        await subscribe_streams(websocket = websocket, streams=[STREAM_BOOKTICKER,STREAM_TRADES])
+                        continue
+                
                 ts_arrival = round(datetime.now().timestamp()*1000)
                 if isinstance(message, str):
                     try:
@@ -128,18 +112,25 @@ async def ws_stream():
                         if channel is not None:
                             match channel:
                                 case c if c == STREAM_MARKPRICE:
-                                    # print(f'MP: {data}')
+                                    if data.get('data'):
-                                    VAL_KEY_OBJ = json.dumps({
+                                        VAL_KEY.set('fund_rate_aster_all', json.dumps(data['data']))
-                                        'timestamp_arrival': ts_arrival,
+                                    else:
-                                        'timestamp_msg': data['data']['E'],
+                                        logging.warning(f'Data["data"] is None: {data}')
-                                        'symbol': data['data']['s'],
+                                    single_ticker_fr = [d for d in data['data'] if d.get('s')==SYMBOL]
-                                        'mark_price': data['data']['p'],
+                                    if single_ticker_fr:
-                                        'index_price': data['data']['i'],
+                                        d = single_ticker_fr[0]
-                                        'estimated_settle_price': data['data']['P'],
+                                        VAL_KEY_OBJ = json.dumps({
-                                        'funding_rate': data['data']['r'],
+                                            'timestamp_arrival': ts_arrival,
-                                        'next_funding_time_ts_ms': data['data']['T'],
+                                            'timestamp_msg': d['E'],
-                                    })
+                                            'symbol': d['s'],
-                                    VAL_KEY.set(VK_FUND_RATE, VAL_KEY_OBJ)
+                                            'mark_price': d['p'],
+                                            'index_price': d['i'],
+                                            'estimated_settle_price': d['P'],
+                                            'funding_rate': d['r'],
+                                            'next_funding_time_ts_ms': d['T'],
+                                        })
+                                        VAL_KEY.set(VK_FUND_RATE, VAL_KEY_OBJ)
+                                        # print(f'MP: {d}')
                                     continue
                                 case c if c == STREAM_BOOKTICKER:
                                     # print(f'BT: {data}')
@@ -156,6 +147,24 @@ async def ws_stream():
                                     })
                                     VAL_KEY.set(VK_TICKER, VAL_KEY_OBJ)
                                     continue
+                                case c if c == STREAM_TRADES:
+                                    # # print(f'MKT_TRADE: {data}')
+                                    # trade_obj = {
+                                    #     'timestamp_arrival': ts_arrival,
+                                    #     'timestamp_msg': data['data']['E'],
+                                    #     'timestamp_trade': data['data']['T'],
+                                    #     'symbol': data['data']['s'],
+                                    #     'aggregate_trade_id': data['data']['a'],
+                                    #     'price': float(data['data']['p']),
+                                    #     'qty': float(data['data']['q']),
+                                    #     'first_trade_id': data['data']['f'],
+                                    #     'last_trade_id': data['data']['l'],
+                                    #     'is_buyer_mkt_maker': bool(data['data']['m']),
+                                    # }
+                                    # # VAL_KEY.set(VK_LAST_TRADE, json.dumps(trade_obj))
+                                    # if USE_DB:
+                                    #     await db.insert_df_to_mysql(table_name='fr_aster_mkt_trades', params=trade_obj, CON=CON)
+                                    continue
                                 case _:
                                     logging.warning(f'UNMATCHED OTHER MSG: {data}')
                         else:
@@ -182,18 +191,17 @@ async def main():
     if USE_VK:
         VAL_KEY = valkey.Valkey(host='localhost', port=6379, db=0)
     else:
-        VAL_KEY = None
         logging.warning("VALKEY NOT BEING USED, NO DATA WILL BE PUBLISHED")
+        raise NotImplementedError('Cannot run without VK')
 
     if USE_DB:
         engine = create_async_engine('mysql+asyncmy://root:pwd@localhost/fund_rate')
         async with engine.connect() as CON:
-            # await create_rtds_btcusd_table(CON=CON)
+            await aster_db.create_fr_aster_mkt_trades(CON=CON)
             await ws_stream()
     else:
-        CON = None
         logging.warning("DATABASE NOT BEING USED, NO DATA WILL BE RECORDED")
-        await ws_stream()
+        raise NotImplementedError('Cannot run without DB')
 
 
 if __name__ == '__main__':

ws_aster/.dockerignore

@@ -0,0 +1 @@
+../rust/

ws_aster_user.py

@@ -1,20 +1,19 @@
 import asyncio
 import json
 import logging
+import os
 import socket
 import traceback
 from datetime import datetime
 from typing import AsyncContextManager
 
-import numpy as np
-import pandas as pd
 import requests.packages.urllib3.util.connection as urllib3_cn  # type: ignore
-from sqlalchemy import text
-import websockets
-from sqlalchemy.ext.asyncio import create_async_engine
 import valkey
-import os
+
+import websockets
 from dotenv import load_dotenv
+from sqlalchemy.ext.asyncio import create_async_engine
+
 import modules.aster_auth as aster_auth
 import modules.aster_db as aster_db
 import modules.db as db
@@ -28,84 +27,84 @@ urllib3_cn.allowed_gai_family = allowed_gai_family
 ### Database ###
 USE_DB: bool = True
 USE_VK: bool = True
-VK_ORDERS_TRADES = 'fr_aster_user_orders'
+CON: AsyncContextManager
-VK_MARGIN_CALLS = 'fr_aster_user_margin_calls'
+VAL_KEY: valkey.Valkey
-VK_BALANCES = 'fr_aster_user_balances'
+VK_ORDERS_TRADES: str = 'fr_aster_user_orders'
-VK_POSITIONS = 'fr_aster_user_positions'
+VK_MARGIN_CALLS: str = 'fr_aster_user_margin_calls'
-CON: AsyncContextManager | None = None
+VK_BALANCES: str = 'fr_aster_user_balances'
-VAL_KEY = None
+VK_POSITIONS: str = 'fr_aster_user_positions'
 
 ### Logging ###
 load_dotenv()
-LOG_FILEPATH: str = os.getenv("LOGS_PATH") + '/Fund_Rate_Aster_User.log'
+LOG_FILEPATH: str = f'{os.getenv(key="LOGS_PATH")}/Fund_Rate_Aster_User.log'
 
 ### CONSTANTS ###
-WSS_URL = "wss://fstream.asterdex.com/ws/"
+WSS_URL: str = "wss://fstream.asterdex.com/ws/"
-LOCAL_RECENT_UPDATES_LOOKBACK_SEC = 30
+LOCAL_RECENT_UPDATES_LOOKBACK_SEC: int = 30
 
 ### Globals ###
-LISTEN_KEY: str | None = None
+Listen_Key: str
-LISTEN_KEY_LAST_UPDATE_TS_S: int = 0
+Listen_Key_Last_Update_TS_S: int = 0
-LISTEN_KEY_PUT_INTERVAL_SEC = 1800
+Listen_Key_Put_Interval_Sec: int = 1800
 
-LOCAL_RECENT_ORDERS: list = []
+Local_Recent_Orders: list[dict] = []
-LOCAL_RECENT_MARGIN_CALLS: list = [] 
+Local_Recent_Margin_Calls: list[dict] = [] 
-LOCAL_RECENT_BALANCES: list = []
+Local_Recent_Balances: list[dict] = []
-LOCAL_RECENT_POSITIONS: list = []
+Local_Recent_Positions: list[dict] = []
 
 
 async def get_new_listen_key() -> str:
-    global LISTEN_KEY_LAST_UPDATE_TS_S
+    global Listen_Key_Last_Update_TS_S
     
-    listen_key_request = {
+    listen_key_request: dict = {
         "url": "/fapi/v3/listenKey",
         "method": "POST",
         "params": {}
     }
-    r = await aster_auth.post_authenticated_url(listen_key_request)
+    r: dict = await aster_auth.post_authenticated_url(listen_key_request)  # ty:ignore[invalid-assignment]
-    listen_key = r.get('listenKey', None)
+    listen_key: str = r.get('listenKey', '')
     print(f'LISTEN KEY: {listen_key}')
-    if listen_key is not None:
+    if listen_key:
-        LISTEN_KEY_LAST_UPDATE_TS_S = round(datetime.now().timestamp())
+        Listen_Key_Last_Update_TS_S = round(number=datetime.now().timestamp())
         return listen_key
     else:
-        raise ValueError(f'Listen Key is None; Failed to Update. response: {r}')
+        raise ValueError(f'Listen Key is empty; Failed to Update. response: {r}')
 
 async def listen_key_interval():
-    global LISTEN_KEY
+    global Listen_Key
 
     while True:
-        await asyncio.sleep(LISTEN_KEY_PUT_INTERVAL_SEC)
+        await asyncio.sleep(delay=Listen_Key_Put_Interval_Sec)
-        LISTEN_KEY = await get_new_listen_key()
+        Listen_Key = await get_new_listen_key()
 
 ### Websocket ###
 async def ws_stream():
-    global LISTEN_KEY
+    global Listen_Key
-    global LOCAL_RECENT_ORDERS
+    global Local_Recent_Orders
-    global LOCAL_RECENT_MARGIN_CALLS
+    global Local_Recent_Margin_Calls
-    global LOCAL_RECENT_BALANCES
+    global Local_Recent_Balances
-    global LOCAL_RECENT_POSITIONS
+    global Local_Recent_Positions
 
-    LISTEN_KEY = await get_new_listen_key()
+    Listen_Key = await get_new_listen_key()
     
-    async for websocket in websockets.connect(WSS_URL+LISTEN_KEY):
+    async for websocket in websockets.connect(uri=WSS_URL+Listen_Key, ping_interval=5):
-        logging.info(f"Connected to {WSS_URL}")
+        logging.info(msg=f"Connected to {WSS_URL}")
-        asyncio.create_task(listen_key_interval())
+        asyncio.create_task(coro=listen_key_interval())
         try:
             async for message in websocket:
-                ts_arrival = round(datetime.now().timestamp()*1000)
+                ts_arrival: int = round(number=datetime.now().timestamp()*1000)
                     
                 if isinstance(message, str):
                     try: 
-                        data = json.loads(message)
+                        data: dict = json.loads(s=message)
-                        channel = data.get('e', None)
+                        channel: str = data.get('e', '')
-                        if channel is not None:
+                        if channel:
-                            LOOKBACK_MIN_TS_MS = ts_arrival - (LOCAL_RECENT_UPDATES_LOOKBACK_SEC*1000)
+                            lookback_min_ts_ms: int = ts_arrival - (LOCAL_RECENT_UPDATES_LOOKBACK_SEC*1000)
 
                             match channel:
                                 case 'ORDER_TRADE_UPDATE':
                                     # logging.info(f'ORDER_TRADE_UPDATE: {data}')
-                                    new_order_update = {
+                                    new_order_update: dict = {
                                         'timestamp_arrival': ts_arrival,
                                         'timestamp_msg': data['E'],
                                         'timestamp_transaction': data['T'],
@@ -141,11 +140,12 @@ async def ws_stream():
                                         'callback_rate': float(data['o'].get("cr", 0)), # :"5.0",                 // Callback Rate, only puhed with TRAILING_STOP_MARKET order
                                         'realized_profit': float(data['o']["rp"]), # :"0"                            // Realized Profit of the trade
                                     }
-                                    LOCAL_RECENT_ORDERS = utils.upsert_list_of_dicts_by_id(LOCAL_RECENT_ORDERS, new_order_update, id='order_id', seq_check_field='timestamp_msg')
+                                    Local_Recent_Orders = utils.upsert_list_of_dicts_by_id(Local_Recent_Orders, new_order_update, id='order_id', seq_check_field='timestamp_msg')
-                                    LOCAL_RECENT_ORDERS = [t for t in LOCAL_RECENT_ORDERS if t.get('timestamp_arrival', 0) >= LOOKBACK_MIN_TS_MS]
+                                    Local_Recent_Orders = [t for t in Local_Recent_Orders if t.get('timestamp_arrival', 0) >= lookback_min_ts_ms]
 
-                                    VAL_KEY_OBJ = json.dumps(LOCAL_RECENT_ORDERS)
+                                    VAL_KEY_OBJ: str = json.dumps(obj=Local_Recent_Orders)
-                                    VAL_KEY.set(VK_ORDERS_TRADES, VAL_KEY_OBJ)
+                                    VAL_KEY.publish(channel=VK_ORDERS_TRADES, message=VAL_KEY_OBJ)
+                                    VAL_KEY.set(name=VK_ORDERS_TRADES, value=VAL_KEY_OBJ)
 
                                     await db.insert_df_to_mysql(table_name='fr_aster_user_order_trade', params=new_order_update, CON=CON)
                                     continue
@@ -153,7 +153,7 @@ async def ws_stream():
                                     # logging.info(f'MARGIN_CALL: {data}')
                                     list_for_df = []
                                     for p in list(data['p']):
-                                        margin_call_update = {
+                                        margin_call_update: dict = {
                                             'timestamp_arrival': ts_arrival,
                                             'timestamp_msg': data['E'],
                                             'cross_wallet_balance': float(data.get('cw', 0)),
@@ -168,11 +168,11 @@ async def ws_stream():
                                             'maint_margin_required': float(p["mm"]), # :"1.614445"     // Maintenance Margin Required
                                         }
                                         list_for_df.append(margin_call_update)
-                                        LOCAL_RECENT_MARGIN_CALLS = utils.upsert_list_of_dicts_by_id(LOCAL_RECENT_MARGIN_CALLS, margin_call_update, id='symbol',  seq_check_field='timestamp_msg')
+                                        Local_Recent_Margin_Calls = utils.upsert_list_of_dicts_by_id(Local_Recent_Margin_Calls, margin_call_update, id='symbol',  seq_check_field='timestamp_msg')
-                                        LOCAL_RECENT_MARGIN_CALLS = [t for t in LOCAL_RECENT_MARGIN_CALLS if t.get('timestamp_arrival', 0) >= LOOKBACK_MIN_TS_MS]
+                                        Local_Recent_Margin_Calls = [t for t in Local_Recent_Margin_Calls if t.get('timestamp_arrival', 0) >= lookback_min_ts_ms]
                                     
-                                    VAL_KEY_OBJ = json.dumps(LOCAL_RECENT_MARGIN_CALLS)
+                                    VAL_KEY_OBJ: str = json.dumps(obj=Local_Recent_Margin_Calls)
-                                    VAL_KEY.set(VK_MARGIN_CALLS, VAL_KEY_OBJ)
+                                    VAL_KEY.set(name=VK_MARGIN_CALLS, value=VAL_KEY_OBJ)
 
                                     await db.insert_df_to_mysql(table_name='fr_aster_user_margin', params=list_for_df, CON=CON)
                                     continue
@@ -183,7 +183,7 @@ async def ws_stream():
                                     ### Balance Updates ###
                                     if len(list(data['a']['B'])) > 0:
                                         for b in list(data['a']['B']):
-                                            balance_update = {
+                                            balance_update: dict = {
                                                 'timestamp_arrival': ts_arrival,
                                                 'timestamp_msg': data['E'],
                                                 'timestamp_transaction': data['T'],
@@ -196,13 +196,13 @@ async def ws_stream():
                                                 'balance_change_excl_pnl_comms': float(b['bc']),
                                             }
                                             list_for_df_bal.append(balance_update)
-                                            LOCAL_RECENT_BALANCES = utils.upsert_list_of_dicts_by_id(LOCAL_RECENT_BALANCES, balance_update, id='asset', seq_check_field='timestamp_msg')
+                                            Local_Recent_Balances = utils.upsert_list_of_dicts_by_id(Local_Recent_Balances, balance_update, id='asset', seq_check_field='timestamp_msg')
-                                            LOCAL_RECENT_BALANCES = [t for t in LOCAL_RECENT_BALANCES if t.get('timestamp_arrival', 0) >= LOOKBACK_MIN_TS_MS]
+                                            Local_Recent_Balances = [t for t in Local_Recent_Balances if t.get('timestamp_arrival', 0) >= lookback_min_ts_ms]
-                                        VAL_KEY.set(VK_BALANCES, json.dumps(LOCAL_RECENT_BALANCES))
+                                        VAL_KEY.set(name=VK_BALANCES, value=json.dumps(obj=Local_Recent_Balances))
                                     ### Position Updates ###
                                     if len(list(data['a']['P'])) > 0:
                                         for p in list(data['a']['P']):
-                                            position_update = {
+                                            position_update: dict = {
                                                 'timestamp_arrival': ts_arrival,
                                                 'timestamp_msg': data['E'],
                                                 'timestamp_transaction': data['T'],
@@ -219,33 +219,35 @@ async def ws_stream():
                                                 'position_side': p['ps'],
                                             }
                                             list_for_df_pos.append(position_update)
-                                            LOCAL_RECENT_POSITIONS = utils.upsert_list_of_dicts_by_id(LOCAL_RECENT_POSITIONS, position_update, id='symbol', seq_check_field='timestamp_msg')
+                                            Local_Recent_Positions = utils.upsert_list_of_dicts_by_id(Local_Recent_Positions, position_update, id='symbol', seq_check_field='timestamp_msg')
-                                            LOCAL_RECENT_POSITIONS = [t for t in LOCAL_RECENT_POSITIONS if t.get('timestamp_arrival', 0) >= LOOKBACK_MIN_TS_MS]
+                                            Local_Recent_Positions = [t for t in Local_Recent_Positions if t.get('timestamp_arrival', 0) >= lookback_min_ts_ms]
-                                        VAL_KEY.set(VK_POSITIONS, json.dumps(LOCAL_RECENT_POSITIONS))
+                                        VAL_KEY.publish(channel=VK_POSITIONS, message=json.dumps(obj=Local_Recent_Positions))
+                                        VAL_KEY.set(name=VK_POSITIONS, value=json.dumps(obj=Local_Recent_Positions))
                                     if list_for_df_bal:
                                         await db.insert_df_to_mysql(table_name='fr_aster_user_account_bal', params=list_for_df_bal, CON=CON)
                                     if list_for_df_pos:
                                         await db.insert_df_to_mysql(table_name='fr_aster_user_account_pos', params=list_for_df_pos, CON=CON)
                                     continue
                                 case 'listenKeyExpired':
-                                    raise('Listen Key Has Expired; Failed to Update Properly. Restarting.')
+                                    raise ValueError('Listen Key Has Expired; Failed to Update Properly. Restarting.')
                                 case _:
-                                    logging.warning(f'UNMATCHED OTHER MSG: {data}')
+                                    logging.warning(msg=f'UNMATCHED OTHER MSG: {data}')
                         else:
-                            logging.info(f'Initial or unexpected data struct, skipping: {data}')
+                            logging.info(msg=f'Initial or unexpected data struct, skipping: {data}')
                             continue
                     except (json.JSONDecodeError, ValueError):
-                        logging.warning(f'Message not in JSON format, skipping: {message}')
+                        logging.warning(msg=f'Message not in JSON format, skipping: {message}')
                         continue
                 else:
                     raise ValueError(f'Type: {type(data)}  not expected: {message}')
         except websockets.ConnectionClosed as e:
-            logging.error(f'Connection closed: {e}')
+            logging.error(msg=f'Connection closed: {e}')
-            logging.error(traceback.format_exc())
+            logging.error(msg=traceback.format_exc())
-            continue
+            utils.send_tg_alert(msg=f'WS_Aster_User - Failure: {e}')
         except Exception as e:
-            logging.error(f'Connection closed: {e}')
+            logging.error(msg=f'Connection closed: {e}')
-            logging.error(traceback.format_exc())
+            logging.error(msg=traceback.format_exc())
+            utils.send_tg_alert(msg=f'WS_Aster_User - Failure: {e}')
 
 
 async def main():
@@ -255,8 +257,8 @@ async def main():
     if USE_VK:
         VAL_KEY = valkey.Valkey(host='localhost', port=6379, db=0)
     else:
-        VAL_KEY = None
         logging.warning("VALKEY NOT BEING USED, NO DATA WILL BE PUBLISHED")
+        raise NotImplementedError('Cannot run without Valkey')
 
     if USE_DB:
         engine = create_async_engine('mysql+asyncmy://root:pwd@localhost/fund_rate')
@@ -267,15 +269,14 @@ async def main():
             await aster_db.create_fr_aster_user_account_pos(CON=CON)
             await ws_stream()
     else:
-        CON = None
         logging.warning("DATABASE NOT BEING USED, NO DATA WILL BE RECORDED")
-        await ws_stream()
+        raise NotImplementedError('Cannot run without DB')
 
 
 if __name__ == '__main__':
-    START_TIME = round(datetime.now().timestamp()*1000)
+    START_TIME: int = round(number=datetime.now().timestamp()*1000)
 
-    logging.info(f'Log FilePath: {LOG_FILEPATH}')
+    logging.info(msg=f'Log FilePath: {LOG_FILEPATH}')
 
     logging.basicConfig(
         force=True,
@@ -284,9 +285,9 @@ if __name__ == '__main__':
         format='%(asctime)s - %(levelname)s - %(message)s',
         filemode='w'
     )
-    logging.info(f"STARTED: {START_TIME}")
+    logging.info(msg=f"STARTED: {START_TIME}")
     
     try:
         asyncio.run(main())
     except KeyboardInterrupt:
-        logging.info("Stream stopped")
+        logging.info(msg="Stream stopped")

ws_aster_user/.dockerignore

@@ -0,0 +1 @@
+../rust/

ws_extended_fund_rate.py

@@ -16,6 +16,8 @@ from sqlalchemy.ext.asyncio import create_async_engine
 import valkey
 import os
 from dotenv import load_dotenv
+import sys
+import modules.utils as utils
 
 
 ### Allow only ipv4 ###
@@ -28,23 +30,19 @@ USE_DB: bool = False
 USE_VK: bool = True
 VK_FUND_RATE = 'fund_rate_extended'
 
-CON: AsyncContextManager | None = None
+CON: AsyncContextManager
-VAL_KEY = None
+VAL_KEY: valkey.Valkey
 
 ### Logging ###
 load_dotenv()
-LOG_FILEPATH: str = os.getenv("LOGS_PATH") + '/Fund_Rate_Extended_FR.log'
+LOG_FILEPATH: str = f'{os.getenv("LOGS_PATH")}/Fund_Rate_Extended_FR.log'
 
 ### CONSTANTS ###
-WS_SYMBOL: str = 'ETH-USD'
+SYMBOL: str = 'ENA-USD'
-FUNDING_RATE_INTERVAL_MIN = 60
 
 ### Globals ###
-WSS_URL = f"wss://api.starknet.extended.exchange/stream.extended.exchange/v1/funding/{WS_SYMBOL}"
+ALLOW_SYMBOL_CHG: bool = True
-
+LOCAL_FUNDING_RATES = []
-
-# HIST_TRADES = np.empty((0, 3))
-# HIST_TRADES_LOOKBACK_SEC = 6
 
 def time_round_down(dt, interval_mins=5) -> int: # returns timestamp in seconds
     interval_secs = interval_mins * 60
@@ -53,113 +51,72 @@ def time_round_down(dt, interval_mins=5) -> int: # returns timestamp in seconds
     
     return rounded_seconds
 
-
-# ### Database Funcs ###
-# async def create_rtds_btcusd_table(
-#         CON: AsyncContextManager,
-#         engine: str = 'mysql', # mysql | duckdb
-#     ) -> None:
-#     if CON is None:
-#         logging.info("NO DB CONNECTION, SKIPPING Create Statements")
-#     else:
-#         if engine == 'mysql':
-#             logging.info('Creating Table if Does Not Exist: binance_btcusd_trades')
-#             await CON.execute(text("""
-#             CREATE TABLE IF NOT EXISTS binance_btcusd_trades (
-#                 timestamp_arrival BIGINT,
-#                 timestamp_msg BIGINT,
-#                 timestamp_value BIGINT,
-#                 value DOUBLE,
-#                 qty DOUBLE
-#             );
-#             """))
-#             await CON.commit()
-#         else:
-#             raise ValueError('Only MySQL engine is implemented')
-
-# async def insert_rtds_btcusd_table(
-#         timestamp_arrival: int,
-#         timestamp_msg: int,
-#         timestamp_value: int,
-#         value: float,
-#         qty: float,
-#         CON: AsyncContextManager,
-#         engine: str = 'mysql', # mysql | duckdb
-#     ) -> None:
-#     params={
-#         'timestamp_arrival': timestamp_arrival,
-#         'timestamp_msg': timestamp_msg,
-#         'timestamp_value': timestamp_value,
-#         'value': value,
-#         'qty': qty,
-#     }
-#     if CON is None:
-#         logging.info("NO DB CONNECTION, SKIPPING Insert Statements")
-#     else:
-#         if engine == 'mysql':
-#             await CON.execute(text("""
-#             INSERT INTO binance_btcusd_trades 
-#             (
-#                 timestamp_arrival,
-#                 timestamp_msg,
-#                 timestamp_value,
-#                 value,
-#                 qty
-#             )
-#             VALUES 
-#                 (
-#                 :timestamp_arrival,
-#                 :timestamp_msg,
-#                 :timestamp_value,
-#                 :value,
-#                 :qty
-#                 )
-#             """),
-#             parameters=params
-#             )
-#             await CON.commit()
-#         else:
-#             raise ValueError('Only MySQL engine is implemented')
-
 ### Websocket ###
 async def ws_stream():
-    async for websocket in websockets.connect(WSS_URL):
+    global SYMBOL
-        logging.info(f"Connected to {WSS_URL}")
+    global LOCAL_FUNDING_RATES
-        try:
+    
-            async for message in websocket:
+    while True:
-                ts_arrival = round(datetime.now().timestamp()*1000)
+        # CHANGE_SYMBOL = False
-                if isinstance(message, str):
+        WSS_URL = "wss://api.starknet.extended.exchange/stream.extended.exchange/v1/funding/"
-                    try: 
+        async for websocket in websockets.connect(WSS_URL):
-                        data = json.loads(message)
+            # if CHANGE_SYMBOL:
-                        if data.get('data', None) is not None:
+                # break
-                            # print(f'FR: {data}')
+            logging.info(f"Connected to {WSS_URL}")
-                            fr_next_update_ts = (time_round_down(dt=datetime.now(timezone.utc), interval_mins=60)+(60*60))*1000
+            try:
-                            VAL_KEY_OBJ = json.dumps({
+                async for message in websocket:
-                                'sequence_id': data['seq'],
+                    ### Update Symbol if Algo Outputs Change ###
-                                'timestamp_arrival': ts_arrival,
+                    if ALLOW_SYMBOL_CHG:
-                                'timestamp_msg': data['ts'],
+                        fr_algo_working_symbol = VAL_KEY.get(name='fr_algo_working_symbol')
-                                'symbol': data['data']['m'],
+                        if not fr_algo_working_symbol:
-                                'funding_rate': float(data['data']['f']),
+                            logging.critical(f'fr_algo_working_symbol is empty - killing: {fr_algo_working_symbol}')
-                                'funding_rate_updated_ts_ms': data['data']['T'],
+                            sys.exit(1)
-                                'next_funding_time_ts_ms': fr_next_update_ts,
+                        best_symbol_by_exchange: dict = json.loads(fr_algo_working_symbol)  # ty:ignore[invalid-argument-type]
-                            })
+                        best_symbol: str = best_symbol_by_exchange['EXTEND']['symbol']
-                            VAL_KEY.set(VK_FUND_RATE, VAL_KEY_OBJ)
+                        if best_symbol != SYMBOL:
+                            logging.info(f'Symbol Change: {SYMBOL} -> {best_symbol}')
+                            SYMBOL = best_symbol
+                            # CHANGE_SYMBOL = True
+                            # await websocket.close()
+                            # break
+                    
+                    ts_arrival = round(datetime.now().timestamp()*1000)
+                    if isinstance(message, str):
+                        try: 
+                            data = json.loads(message)
+                            if data.get('data', None) is not None:
+                                print(f'FR: {data}')
+                                fr_next_update_ts = (time_round_down(dt=datetime.now(timezone.utc), interval_mins=60)+(60*60))*1000
+                                fr_update = {
+                                    'sequence_id': data['seq'],
+                                    'timestamp_arrival': ts_arrival,
+                                    'timestamp_msg': data['ts'],
+                                    'symbol': data['data']['m'],
+                                    'funding_rate': float(data['data']['f']),
+                                    'funding_rate_updated_ts_ms': data['data']['T'],
+                                    'next_funding_time_ts_ms': fr_next_update_ts,
+                                }
+                                if fr_update.get('symbol') == SYMBOL:
+                                    VAL_KEY_OBJ = json.dumps(fr_update)
+                                    VAL_KEY.set(VK_FUND_RATE, VAL_KEY_OBJ)
+                                LOCAL_FUNDING_RATES = utils.upsert_list_of_dicts_by_id(LOCAL_FUNDING_RATES, fr_update, id='symbol', seq_check_field=None)
+                                VAL_KEY.set('fund_rate_extended_all', json.dumps(LOCAL_FUNDING_RATES))
+                                continue
+                            else:
+                                logging.info(f'Initial or unexpected data struct, skipping: {data}')
+                                continue
+                        except (json.JSONDecodeError, ValueError):
+                            logging.warning(f'Message not in JSON format, skipping: {message}')
                             continue
-                        else:
+                    else:
-                            logging.info(f'Initial or unexpected data struct, skipping: {data}')
+                        raise ValueError(f'Type: {type(data)}  not expected: {message}')
-                            continue
+            except websockets.ConnectionClosed as e:
-                    except (json.JSONDecodeError, ValueError):
+                logging.error(f'Connection closed: {e}')
-                        logging.warning(f'Message not in JSON format, skipping: {message}')
+                logging.error(traceback.format_exc())
-                        continue
+                continue
-                else:
+            except Exception as e:
-                    raise ValueError(f'Type: {type(data)}  not expected: {message}')
+                logging.error(f'Connection closed: {e}')
-        except websockets.ConnectionClosed as e:
+                logging.error(traceback.format_exc())
-            logging.error(f'Connection closed: {e}')
-            logging.error(traceback.format_exc())
-            continue
-        except Exception as e:
-            logging.error(f'Connection closed: {e}')
-            logging.error(traceback.format_exc())
 
 
 async def main():
@@ -169,16 +126,16 @@ async def main():
     if USE_VK:
         VAL_KEY = valkey.Valkey(host='localhost', port=6379, db=0)
     else:
-        VAL_KEY = None
         logging.warning("VALKEY NOT BEING USED, NO DATA WILL BE PUBLISHED")
+        raise NotImplementedError('Cannot run without VK')
 
     if USE_DB:
-        engine = create_async_engine('mysql+asyncmy://root:pwd@localhost/fund_rate')
+        raise NotImplementedError('DB not implemented')
-        async with engine.connect() as CON:
+        # engine = create_async_engine('mysql+asyncmy://root:pwd@localhost/fund_rate')
-            # await create_rtds_btcusd_table(CON=CON)
+        # async with engine.connect() as CON:
-            await ws_stream()
+        #     # await create_rtds_btcusd_table(CON=CON)
+        #     await ws_stream()
     else:
-        CON = None
         logging.warning("DATABASE NOT BEING USED, NO DATA WILL BE RECORDED")
         await ws_stream()
 

ws_extended_fund_rate/.dockerignore

@@ -0,0 +1 @@
+../rust/

ws_extended_orderbook.py

@@ -27,58 +27,80 @@ USE_DB: bool = False
 USE_VK: bool = True
 
 VK_TICKER = 'fut_ticker_extended'
-CON: AsyncContextManager | None = None
+CON: AsyncContextManager
-VAL_KEY = None
+VAL_KEY: valkey.Valkey
 
 ### Logging ###
 load_dotenv()
-LOG_FILEPATH: str = os.getenv("LOGS_PATH") + '/Fund_Rate_Extended_OB.log'
+LOG_FILEPATH: str = f'{os.getenv("LOGS_PATH")}/Fund_Rate_Extended_OB.log'
 
 ### CONSTANTS ###
-WS_SYMBOL: str = 'ETH-USD'
+SYMBOL: str = 'ETH-USD'
 
 ### Globals ###
-WSS_URL = f"wss://api.starknet.extended.exchange/stream.extended.exchange/v1/orderbooks/{WS_SYMBOL}?depth=1"
+ALLOW_SYMBOL_CHG: bool = True
 
 ### Websocket ###
 async def ws_stream():
-    async for websocket in websockets.connect(WSS_URL):
+    global SYMBOL
-        logging.info(f"Connected to {WSS_URL}")
+    
-        try:
+    while True:
-            async for message in websocket:
+        CHANGE_SYMBOL = False
-                ts_arrival = round(datetime.now().timestamp()*1000)
+        WSS_URL = f"wss://api.starknet.extended.exchange/stream.extended.exchange/v1/orderbooks/{SYMBOL}?depth=1"
-                if isinstance(message, str):
+        async for websocket in websockets.connect(WSS_URL):
-                    try: 
+            if CHANGE_SYMBOL:
-                        data = json.loads(message)
+                break
-                        if data.get('type', None) is not None:
+            logging.info(f"Connected to {WSS_URL}")
-                            # print(f'OB: {data}')
+            try:
-                            VAL_KEY_OBJ = json.dumps({
+                async for message in websocket:
-                                'sequence_id': data['seq'],
+                    ### Update Symbol if Algo Outputs Change ###
-                                'timestamp_arrival': ts_arrival,
+                    if ALLOW_SYMBOL_CHG:
-                                'timestamp_msg': data['ts'],
+                        vk_get: str = VAL_KEY.get(name='fr_algo_working_symbol')  # ty:ignore[invalid-assignment]
-                                'symbol': data['data']['m'],
+                        if vk_get:
-                                'best_bid_px': float(data['data']['b'][0]['p']),
+                            best_symbol_by_exchange: dict = json.loads(s=vk_get)
-                                'best_bid_qty': float(data['data']['b'][0]['q']),
+                            best_symbol: str = best_symbol_by_exchange['EXTEND']['symbol']
-                                'best_ask_px': float(data['data']['a'][0]['p']),
+                            if best_symbol != SYMBOL:
-                                'best_ask_qty': float(data['data']['a'][0]['q']),
+                                logging.info(f'Symbol Change: {SYMBOL} -> {best_symbol}')
-                            })
+                                SYMBOL = best_symbol
-                            VAL_KEY.set(VK_TICKER, VAL_KEY_OBJ)
+                                CHANGE_SYMBOL = True
-                            continue
+                                await websocket.close()
+                                break
                         else:
-                            logging.info(f'Initial or unexpected data struct, skipping: {data}')
+                            logging.warning('Extend Orderbook WS: "fr_algo_working_symbol" is None; not switching to new symbol...')
+                    
+                    ts_arrival = round(datetime.now().timestamp()*1000)
+                    if isinstance(message, str):
+                        try: 
+                            data = json.loads(message)
+                            if data.get('type', None) is not None:
+                                # print(f'OB: {data}')
+                                VAL_KEY_OBJ = json.dumps({
+                                    'sequence_id': data['seq'],
+                                    'timestamp_arrival': ts_arrival,
+                                    'timestamp_msg': data['ts'],
+                                    'symbol': data['data']['m'],
+                                    'best_bid_px': float(data['data']['b'][0]['p']),
+                                    'best_bid_qty': float(data['data']['b'][0]['q']),
+                                    'best_ask_px': float(data['data']['a'][0]['p']),
+                                    'best_ask_qty': float(data['data']['a'][0]['q']),
+                                })
+                                VAL_KEY.set(VK_TICKER, VAL_KEY_OBJ)
+                                continue
+                            else:
+                                logging.info(f'Initial or unexpected data struct, skipping: {data}')
+                                continue
+                        except (json.JSONDecodeError, ValueError):
+                            logging.warning(f'Message not in JSON format, skipping: {message}')
                             continue
-                    except (json.JSONDecodeError, ValueError):
+                    else:
-                        logging.warning(f'Message not in JSON format, skipping: {message}')
+                        raise ValueError(f'Type: {type(data)}  not expected: {message}')
-                        continue
+            except websockets.ConnectionClosed as e:
-                else:
+                logging.error(f'Connection closed: {e}')
-                    raise ValueError(f'Type: {type(data)}  not expected: {message}')
+                logging.error(traceback.format_exc())
-        except websockets.ConnectionClosed as e:
+                continue
-            logging.error(f'Connection closed: {e}')
+            except Exception as e:
-            logging.error(traceback.format_exc())
+                logging.error(f'Connection closed: {e}')
-            continue
+                logging.error(traceback.format_exc())
-        except Exception as e:
-            logging.error(f'Connection closed: {e}')
-            logging.error(traceback.format_exc())
 
 
 async def main():
@@ -88,16 +110,16 @@ async def main():
     if USE_VK:
         VAL_KEY = valkey.Valkey(host='localhost', port=6379, db=0)
     else:
-        VAL_KEY = None
         logging.warning("VALKEY NOT BEING USED, NO DATA WILL BE PUBLISHED")
+        raise NotImplementedError('Cannot run without VK')
 
     if USE_DB:
-        engine = create_async_engine('mysql+asyncmy://root:pwd@localhost/fund_rate')
+        raise NotImplementedError('DB not implemented')
-        async with engine.connect() as CON:
+        # engine = create_async_engine('mysql+asyncmy://root:pwd@localhost/fund_rate')
-            # await create_rtds_btcusd_table(CON=CON)
+        # async with engine.connect() as CON:
-            await ws_stream()
+        #     # await create_rtds_btcusd_table(CON=CON)
+        #     await ws_stream()
     else:
-        CON = None
         logging.warning("DATABASE NOT BEING USED, NO DATA WILL BE RECORDED")
         await ws_stream()
 

ws_extended_orderbook/.dockerignore

@@ -0,0 +1,2 @@
+../rust/
+/rust/

ws_extended_trades.py

@@ -0,0 +1,158 @@
+import asyncio
+import json
+import logging
+import os
+import socket
+import traceback
+from datetime import datetime
+from typing import AsyncContextManager
+
+import numpy as np
+import pandas as pd
+import requests.packages.urllib3.util.connection as urllib3_cn  # type: ignore
+import valkey
+import websockets
+from dotenv import load_dotenv
+from sqlalchemy.ext.asyncio import create_async_engine
+
+import modules.db as db
+import modules.extended_db as extended_db
+
+
+### Allow only ipv4 ###
+def allowed_gai_family():
+    return socket.AF_INET
+urllib3_cn.allowed_gai_family = allowed_gai_family
+
+### Database ###
+USE_DB: bool = True
+USE_VK: bool = True
+
+VK_LAST_TRADE = 'fut_last_trade_extended'
+CON: AsyncContextManager
+VAL_KEY: valkey.Valkey
+
+### Logging ###
+load_dotenv()
+LOG_FILEPATH: str = f'{os.getenv("LOGS_PATH")}/Fund_Rate_Extended_Trades.log'
+
+### CONSTANTS ###
+SYMBOL: str = 'ETH-USD'
+
+### Globals ###
+ALLOW_SYMBOL_CHG: bool = True
+
+
+### Websocket ###
+async def ws_stream():
+    global SYMBOL 
+    
+    while True:
+        CHANGE_SYMBOL: bool = False
+        WSS_URL = f"wss://api.starknet.extended.exchange/stream.extended.exchange/v1/publicTrades/{SYMBOL}"
+        async for websocket in websockets.connect(WSS_URL):
+            if CHANGE_SYMBOL:
+                break
+            logging.info(f"Connected to {WSS_URL}")
+            try:
+                async for message in websocket:
+                    ### Update Symbol if Algo Outputs Change ###
+                    if ALLOW_SYMBOL_CHG:
+                        vk_get: str = VAL_KEY.get(name='fr_algo_working_symbol')  # ty:ignore[invalid-assignment]
+                        if vk_get:
+                            best_symbol_by_exchange: dict = json.loads(s=vk_get)
+                            best_symbol: str = best_symbol_by_exchange['EXTEND']['symbol']
+                            if best_symbol != SYMBOL:
+                                logging.info(f'Symbol Change: {SYMBOL} -> {best_symbol}')
+                                SYMBOL = best_symbol
+                                CHANGE_SYMBOL = True
+                                await websocket.close()
+                                break
+                        else:
+                            logging.warning('Extend Trades WS: "fr_algo_working_symbol" is None; not switching to new symbol...')
+                    
+                    ts_arrival = round(datetime.now().timestamp()*1000)
+                    if isinstance(message, str):
+                        try: 
+                            data = json.loads(message)
+                            if data.get('data', None) is not None:
+                                # print(data)
+                                if data['seq'] == 1: # Skip first msg that has historical trades
+                                    continue
+                                list_for_df = []
+                                for t in data['data']:
+                                    trade_obj = {
+                                        'sequence_id': data['seq'],
+                                        'timestamp_arrival': ts_arrival,
+                                        'timestamp_msg': data['ts'],
+                                        'timestamp_trade': t['T'],
+                                        'symbol': t['m'],
+                                        'side_taker': t['S'],
+                                        'trade_type': t['tT'],
+                                        'price': float(t['p']),
+                                        'qty': float(t['q']),
+                                        'trade_id': t['i'],
+                                        'is_buyer_mkt_maker': True if t['S']=='SELL' else False,
+                                    }
+                                    list_for_df.append(trade_obj)
+                                # VAL_KEY.set(VK_LAST_TRADE, json.dumps(trade_obj))
+                                if USE_DB:
+                                    await db.insert_df_to_mysql(table_name='fr_extended_mkt_trades', params=list_for_df, CON=CON)
+                                    pass
+                                continue
+                            else:
+                                logging.info(f'Initial or unexpected data struct, skipping: {data}')
+                                continue
+                        except (json.JSONDecodeError, ValueError):
+                            logging.warning(f'Message not in JSON format, skipping: {message}')
+                            continue
+                    else:
+                        raise ValueError(f'Type: {type(data)}  not expected: {message}')
+            except websockets.ConnectionClosed as e:
+                logging.error(f'Connection closed: {e}')
+                logging.error(traceback.format_exc())
+                continue
+            except Exception as e:
+                logging.error(f'Connection closed: {e}')
+                logging.error(traceback.format_exc())
+
+
+async def main():
+    global VAL_KEY
+    global CON
+    
+    if USE_VK:
+        VAL_KEY = valkey.Valkey(host='localhost', port=6379, db=0)
+    else:
+        logging.warning("VALKEY NOT BEING USED, NO DATA WILL BE PUBLISHED")
+        raise NotImplementedError('Cannot run without VK')
+
+    if USE_DB:
+        engine = create_async_engine('mysql+asyncmy://root:pwd@localhost/fund_rate')
+        async with engine.connect() as CON:
+            await extended_db.create_fr_extended_mkt_trades(CON=CON)
+            await ws_stream()
+    else:
+        logging.warning("DATABASE NOT BEING USED, NO DATA WILL BE RECORDED")
+        raise NotImplementedError('Cannot run without DB')
+
+
+
+if __name__ == '__main__':
+    START_TIME = round(datetime.now().timestamp()*1000)
+
+    logging.info(f'Log FilePath: {LOG_FILEPATH}')
+
+    logging.basicConfig(
+        force=True,
+        filename=LOG_FILEPATH,
+        level=logging.INFO,
+        format='%(asctime)s - %(levelname)s - %(message)s',
+        filemode='w'
+    )
+    logging.info(f"STARTED: {START_TIME}")
+    
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        logging.info("Stream stopped")

ws_extended_trades/.dockerignore

@@ -0,0 +1 @@
+../rust/

ws_extended_trades/Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.13-slim
+
+RUN apt-get update && \
+    apt-get install -y build-essential
+
+RUN gcc --version
+RUN rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY requirements.txt .
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+# Finally, run gunicorn.
+CMD [ "python", "ws_extended_trades.py"]
+# CMD [ "gunicorn", "--workers=5", "--threads=1", "-b 0.0.0.0:8000", "app:server"]

ws_extended_user.py

@@ -32,16 +32,16 @@ VK_ORDERS = 'fr_extended_user_orders'
 VK_TRADES = 'fr_extended_user_trades'
 VK_BALANCES = 'fr_extended_user_balances'
 VK_POSITIONS = 'fr_extended_user_positions'
-CON: AsyncContextManager | None = None
+CON: AsyncContextManager
-VAL_KEY = None
+VAL_KEY: valkey.Valkey
 
 ### Logging ###
 load_dotenv()
-LOG_FILEPATH: str = os.getenv("LOGS_PATH") + '/Fund_Rate_Extended_User.log'
+LOG_FILEPATH: str = f'{os.getenv("LOGS_PATH")}/Fund_Rate_Extended_User.log'
 
 ### CONSTANTS ###
 WSS_URL = "wss://api.starknet.extended.exchange/stream.extended.exchange/v1/account"
-API_KEY = os.getenv('EXTENDED_API_KEY')
+API_KEY: str = os.getenv('EXTENDED_API_KEY')  # ty:ignore[invalid-assignment]
 LOCAL_RECENT_UPDATES_LOOKBACK_SEC = 30
 
 ### Globals ###
@@ -50,12 +50,15 @@ LOCAL_RECENT_TRADES: list = []
 LOCAL_RECENT_BALANCES: list = []
 LOCAL_RECENT_POSITIONS: list = []
 
+RESET_SEQ: bool = False
+
 ### Websocket ###
 async def ws_stream():
     global LOCAL_RECENT_ORDERS
     global LOCAL_RECENT_TRADES
     global LOCAL_RECENT_BALANCES
     global LOCAL_RECENT_POSITIONS
+    global RESET_SEQ
     
     async for websocket in websockets.connect(WSS_URL, extra_headers={'X-Api-Key': API_KEY}):
         logging.info(f"Connected to {WSS_URL}")
@@ -101,13 +104,13 @@ async def ws_stream():
                                             'expire_time_ts': o['expireTime'],
                                         }
                                         list_for_df.append(order_update)
-                                        LOCAL_RECENT_ORDERS = utils.upsert_list_of_dicts_by_id(LOCAL_RECENT_ORDERS, order_update, id='order_id', seq_check_field='sequence_id')
+                                        LOCAL_RECENT_ORDERS = utils.upsert_list_of_dicts_by_id(LOCAL_RECENT_ORDERS, order_update, id='order_id', seq_check_field='sequence_id', reset_seq_id=RESET_SEQ)
                                         LOCAL_RECENT_ORDERS = [t for t in LOCAL_RECENT_ORDERS if t.get('timestamp_arrival', 0) >= LOOKBACK_MIN_TS_MS]
                                     
                                     VAL_KEY_OBJ = json.dumps(LOCAL_RECENT_ORDERS)
-                                    VAL_KEY.set(VK_ORDERS, VAL_KEY_OBJ)
+                                    VAL_KEY.publish(channel=VK_ORDERS, message=VAL_KEY_OBJ)
+                                    VAL_KEY.set(name=VK_ORDERS, value=VAL_KEY_OBJ)
                                     await db.insert_df_to_mysql(table_name='fr_extended_user_order', params=list_for_df, CON=CON)
-                                    continue
                                 case 'TRADE':
                                     list_for_df = []
                                     for t in data['data']['trades']:
@@ -131,13 +134,12 @@ async def ws_stream():
                                             'is_taker': t['isTaker'],
                                         }
                                         list_for_df.append(trade_update)
-                                        LOCAL_RECENT_TRADES = utils.upsert_list_of_dicts_by_id(LOCAL_RECENT_TRADES, trade_update, id='trade_id', seq_check_field='sequence_id')
+                                        LOCAL_RECENT_TRADES = utils.upsert_list_of_dicts_by_id(LOCAL_RECENT_TRADES, trade_update, id='trade_id', seq_check_field='sequence_id', reset_seq_id=RESET_SEQ)
                                         LOCAL_RECENT_TRADES = [t for t in LOCAL_RECENT_TRADES if t.get('timestamp_arrival', 0) >= LOOKBACK_MIN_TS_MS]
                                     
                                     VAL_KEY_OBJ = json.dumps(LOCAL_RECENT_TRADES)
                                     VAL_KEY.set(VK_TRADES, VAL_KEY_OBJ)
                                     await db.insert_df_to_mysql(table_name='fr_extended_user_trade', params=list_for_df, CON=CON)
-                                    continue
                                 case 'BALANCE':
                                     balance_update = {
                                         'sequence_id': data['seq'],
@@ -156,13 +158,12 @@ async def ws_stream():
                                         'exposure': float(data['data']['balance']['exposure']),
                                         'leverage': float(data['data']['balance']['leverage']),
                                     }
-                                    LOCAL_RECENT_BALANCES = utils.upsert_list_of_dicts_by_id(LOCAL_RECENT_BALANCES, balance_update, id='collateral_name', seq_check_field='sequence_id')
+                                    LOCAL_RECENT_BALANCES = utils.upsert_list_of_dicts_by_id(LOCAL_RECENT_BALANCES, balance_update, id='collateral_name', seq_check_field='sequence_id', reset_seq_id=RESET_SEQ)
                                     LOCAL_RECENT_BALANCES = [t for t in LOCAL_RECENT_BALANCES if t.get('timestamp_arrival', 0) >= LOOKBACK_MIN_TS_MS]
                                     
                                     VAL_KEY_OBJ = json.dumps(LOCAL_RECENT_BALANCES)
                                     VAL_KEY.set(VK_BALANCES, VAL_KEY_OBJ)
                                     await db.insert_df_to_mysql(table_name='fr_extended_user_balance', params=balance_update, CON=CON)
-                                    continue
                                 case 'POSITION':
                                     list_for_df = []
                                     for p in data['data']['positions']:
@@ -193,31 +194,40 @@ async def ws_stream():
                                             'updated_at_ts': p['updatedAt'],
                                         }
                                         list_for_df.append(position_update)
-                                        LOCAL_RECENT_POSITIONS = utils.upsert_list_of_dicts_by_id(LOCAL_RECENT_POSITIONS, position_update, id='position_id', seq_check_field='sequence_id')
+                                        LOCAL_RECENT_POSITIONS = utils.upsert_list_of_dicts_by_id(LOCAL_RECENT_POSITIONS, position_update, id='market', seq_check_field='sequence_id', reset_seq_id=RESET_SEQ)
                                         LOCAL_RECENT_POSITIONS = [t for t in LOCAL_RECENT_POSITIONS if t.get('timestamp_arrival', 0) >= LOOKBACK_MIN_TS_MS]
                                     
                                     VAL_KEY_OBJ = json.dumps(LOCAL_RECENT_POSITIONS)
-                                    VAL_KEY.set(VK_POSITIONS, VAL_KEY_OBJ)
+                                    VAL_KEY.publish(channel=VK_POSITIONS, message=VAL_KEY_OBJ)
-                                    
+                                    VAL_KEY.set(name=VK_POSITIONS, value=VAL_KEY_OBJ)
                                     await db.insert_df_to_mysql(table_name='fr_extended_user_position', params=list_for_df, CON=CON)
-                                    continue
                                 case _:
                                     logging.warning(f'UNMATCHED OTHER MSG: {data}')
+                                    RESET_SEQ = True
+                            
+                            ### END OF GOOD MAIN LOOP - SEQ WILL HAVE BEEN RESET IF A FAILURE HAD OCCURRED; FLIPPING BOOL BACK TO NORMAL ###
+                            RESET_SEQ = False 
+                            continue
                         else:
                             logging.info(f'Initial or unexpected data struct, skipping: {data}')
+                            RESET_SEQ = True
                             continue
                     except (json.JSONDecodeError, ValueError):
                         logging.warning(f'Message not in JSON format, skipping: {message}')
+                        RESET_SEQ = True
                         continue
                 else:
+                    RESET_SEQ = True
                     raise ValueError(f'Type: {type(data)}  not expected: {message}')
         except websockets.ConnectionClosed as e:
             logging.error(f'Connection closed: {e}')
             logging.error(traceback.format_exc())
+            RESET_SEQ = True
             continue
         except Exception as e:
             logging.error(f'Connection closed: {e}')
             logging.error(traceback.format_exc())
+            RESET_SEQ = True
 
 
 async def main():
@@ -227,8 +237,8 @@ async def main():
     if USE_VK:
         VAL_KEY = valkey.Valkey(host='localhost', port=6379, db=0)
     else:
-        VAL_KEY = None
         logging.warning("VALKEY NOT BEING USED, NO DATA WILL BE PUBLISHED")
+        raise NotImplementedError('Cannot run without VK')
 
     if USE_DB:
         engine = create_async_engine('mysql+asyncmy://root:pwd@localhost/fund_rate')
@@ -239,9 +249,9 @@ async def main():
             await extended_db.create_fr_extended_user_trade(CON=CON)
             await ws_stream()
     else:
-        CON = None
         logging.warning("DATABASE NOT BEING USED, NO DATA WILL BE RECORDED")
-        await ws_stream()
+        raise NotImplementedError('Cannot run without DB')
+        # await ws_stream()
 
 
 if __name__ == '__main__':

ws_extended_user/.dockerignore

@@ -0,0 +1 @@
+../rust/