moving to extravm

2026-05-08 03:02:02 +00:00
parent e4bdb3f6a0
commit f5f43be1a1
4 changed files with 517 additions and 447 deletions
--- a/ng.py
+++ b/ng.py
@@ -1,7 +1,8 @@
 import os
 from nicegui import ui, app, html
-from sqlalchemy import create_engine
+from sqlalchemy import create_engine, text
 # import requests
+import pandas as pd
 import json
 # import time
 # import re
@@ -9,6 +10,8 @@ import valkey
 import asyncio
 from datetime import datetime
 from dataclasses import dataclass, field
+from sqlalchemy.ext.asyncio import create_async_engine
+from typing import AsyncContextManager
 # from random import random
 # from nicegui_modules import data
 # from nicegui_modules import ui_components
@@ -16,13 +19,16 @@ from dataclasses import dataclass, field
 ALLOW_BODY_SCROLL: bool = True
 LOOKBACK: int = 60
 LOOKBACK_RT_TV_MAX_POINTS: int = 3000
-# REFRESH_INTERVAL_SEC: float = 10
+REFRESH_INTERVAL_SEC: float = 10
 REFRESH_INTERVAL_RT_SEC: float = 1/10
-
+# CON: AsyncContextManager
+# ENGINE = create_async_engine('mysql+asyncmy://root:pwd@localhost/fund_rate')
 ENGINE = create_engine('mysql+pymysql://root:pwd@localhost/fund_rate')
-VALKEY = valkey.Valkey(host='localhost', port=6379, db=0, decode_responses=True)


+
+VALKEY = valkey.Valkey(host='localhost', port=6379, db=0, decode_responses=True)
+
 CHARTS = [
    {
        'type': 'AREA',
@@ -110,6 +116,81 @@ CHARTS_OPTIONS = {
    }
 }

+### Data ###
+async def get_bfr_master_data() -> pd.DataFrame:
+    df = pd.DataFrame(json.loads(VALKEY.get('fr_engine_best_fund_rate_master')))  # ty:ignore[invalid-argument-type]    
+    df.reset_index(drop=True)
+    df['id'] = df.index
+    return df
+
+async def get_trades_hist() -> pd.DataFrame:
+    start_ts = (round(datetime.now().timestamp()*1000)-(60*60*24*1000))
+    ### ASTER ###
+    aster_orders = text(f'''
+    SELECT *
+    FROM fr_aster_user_order_trade
+    WHERE timestamp_arrival > {start_ts}
+    ''')
+    df_aster_orders = pd.read_sql(aster_orders, con=ENGINE)
+    df_aster_orders['timestamp_dt'] = pd.to_datetime(df_aster_orders['timestamp_transaction'], unit='ms')
+    df_aster_orders_fill = df_aster_orders.loc[df_aster_orders['execution_type']=='TRADE',:]
+    df_aster_orders_fill = df_aster_orders_fill[['timestamp_transaction','order_trade_time_ts','timestamp_dt','order_id','trade_id','client_order_id','order_status','side','last_filled_qty','filled_accumulated_qty','commission','last_filled_price','realized_profit']].reset_index(drop=True)
+
+    df_aster_trades = df_aster_orders_fill.groupby('order_id').agg({'timestamp_transaction': 'first','order_trade_time_ts':'last','order_status':'last','side':'last','last_filled_qty':'sum','filled_accumulated_qty':'last','commission':'sum','last_filled_price':'mean','realized_profit':'sum'}).reset_index()
+    df_aster_trades['is_mkt_maker'] = df_aster_trades['commission'] == 0.00
+    df_aster_trades['timestamp_ts'] = pd.to_datetime(df_aster_trades['order_trade_time_ts'], unit='ms')
+
+    df_aster_trades = df_aster_trades.rename({'order_status':'status','filled_accumulated_qty':'filled_qty','commission':'payed_fee','last_filled_price':'price'}, axis=1)
+
+    ### EXTEND ###
+    # Load and Transform Orders
+    extend_orders = text(f'''
+        SELECT *
+        FROM fr_extended_user_order
+        WHERE timestamp_arrival > {start_ts}
+    ''')
+    df_extend_orders = pd.read_sql(extend_orders, con=ENGINE)
+    df_extend_orders['timestamp_dt'] = pd.to_datetime(df_extend_orders['updated_time_ts'], unit='ms')
+    df_extend_orders_fill = df_extend_orders.loc[df_extend_orders['status'].isin(['FILLED','PARTIALLY_FILLED']),:]
+    df_extend_orders_fill = df_extend_orders_fill[['created_time_ts','updated_time_ts','timestamp_dt','order_id','external_id','status','side','qty','filled_qty','payed_fee','price','averagePrice']].reset_index(drop=True)
+
+    # Trades
+    df_extend_trades = df_extend_orders_fill.groupby('order_id').agg({'created_time_ts':'first','updated_time_ts':'last','status': 'last','side': 'last', 'filled_qty':'last','payed_fee':'sum','price':'last'}).reset_index()
+    df_extend_trades['duration_sec_ast'] = ( df_extend_trades['updated_time_ts'] - df_extend_trades['created_time_ts'] ) / 1000
+    df_extend_trades['is_mkt_maker'] = df_extend_trades['payed_fee'] == 0.00
+    df_extend_trades['timestamp_ts'] = pd.to_datetime(df_extend_trades['updated_time_ts'], unit='ms')
+    
+    def tie_trades_together_get_extend_from_aster(row):
+        row = row.to_frame().T
+        row.index=[1]
+
+        extend_row = df_extend_trades[['order_id','timestamp_ts','status','side','filled_qty','payed_fee','price','is_mkt_maker']].loc[df_extend_trades['timestamp_ts']>row['timestamp_ts'].iloc[0],:].iloc[0]
+        extend_row = extend_row.to_frame().T
+        extend_row.index=[1]
+
+        return_row = row.merge(extend_row, left_index=True, right_index=True, suffixes=('_ast','_ext'))
+
+        return return_row.iloc[0]
+
+    df_comb_trades = df_aster_trades[['order_id','timestamp_ts','status','side','filled_qty','payed_fee','price','is_mkt_maker']].apply(tie_trades_together_get_extend_from_aster, axis=1)
+    df_comb_trades['buy_price'] = df_comb_trades['price_ast'].where(df_comb_trades['side_ast']=='BUY', df_comb_trades['price_ext'])
+    df_comb_trades['sell_price'] = df_comb_trades['price_ast'].where(df_comb_trades['side_ast']=='SELL', df_comb_trades['price_ext'])
+    df_comb_trades['buy_qty'] = df_comb_trades['filled_qty_ast'].where(df_comb_trades['side_ast']=='BUY', df_comb_trades['filled_qty_ext'])
+    df_comb_trades['sell_qty'] = df_comb_trades['filled_qty_ast'].where(df_comb_trades['side_ast']=='SELL', df_comb_trades['filled_qty_ext'])
+    df_comb_trades['buy_side'] = df_comb_trades['order_id_ast'].where(df_comb_trades['side_ast']=='BUY', df_comb_trades['order_id_ext'])
+    df_comb_trades['buy_side'] = df_comb_trades['order_id_ast'] == df_comb_trades['buy_side']
+    df_comb_trades['buy_side'] = df_comb_trades['buy_side'].replace(True, 'ASTER').replace(False,'EXTEND')
+
+    df_comb_trades['per_trade_pnl'] = ( ( df_comb_trades['sell_price'] - df_comb_trades['buy_price'] ) * df_comb_trades['sell_qty'] ) - df_comb_trades['payed_fee_ast'] - df_comb_trades['payed_fee_ext']
+    df_comb_trades['per_trade_pnl_pct'] = ( (df_comb_trades['sell_price']*df_comb_trades['sell_qty']) - (df_comb_trades['buy_price']*df_comb_trades['buy_qty']) ) / (df_comb_trades['buy_price']*df_comb_trades['buy_qty'])
+    
+        
+    df = df_comb_trades.apply(lambda x: x.dt.strftime('%Y-%m-%d %H:%M:%S.%f') if hasattr(x, 'dt') else x)
+    
+    df.reset_index(drop=True)
+    df['id'] = df.index
+    return df
+
 ### Utils ###
 def update_body_scroll(e=None, bool_override=False):
    if e is None:
@@ -128,7 +209,6 @@ async def update_tv():
    series_update_aster_tob = json.loads(VALKEY.get('fut_ticker_aster'))  # ty:ignore[invalid-argument-type]
    series_update_extend_tob = json.loads(VALKEY.get('fut_ticker_extended')) # ty:ignore[invalid-argument-type]
    series_update_algo_status = json.loads(VALKEY.get('algo_status')) # ty:ignore[invalid-argument-type] 
-    master_data = json.loads(VALKEY.get(name='fr_engine_best_fund_rate_master')) # ty:ignore[invalid-argument-type]
    
    timestamp_aster_tob = round( ( series_update_aster_tob['timestamp_transaction'] / 1000 ) , 2)
    timestamp_extend_tob = round( ( series_update_extend_tob['timestamp_msg'] / 1000 ) , 2)
@@ -165,13 +245,57 @@ async def update_tv():

    ui.run_javascript(f'await update_tv(data_list={data_list}, lookback_max_points={LOOKBACK_RT_TV_MAX_POINTS});')

+async def create_bfr_aggrid() -> ui.aggrid:
+    df = await get_bfr_master_data()
+    col_extras = {
+        'symbol_ast': {
+            'editable': False, 
+            'sortable': True
+        }
+    }
+    cols = [ {'field': v, **col_extras.get(v, {})} for v in df.columns ]
+    rows = df.to_dict(orient='records')
+    grid = ui.aggrid(
+        {
+            'columnDefs': cols,
+            'rowData': rows,
+            'autoSizeStrategy': {
+                'type': 'fitCellContents',
+            },
+            # 'rowSelection': {'mode': 'multiRow'},
+            # 'stopEditingWhenCellsLoseFocus': True,
+        }
+    ).classes('auto-fit flex-grow w-full col-span-2 md:col-span-1')
+    
+    return grid
+
+async def create_pnl_aggrid() -> ui.aggrid:
+    df = await get_trades_hist()
+    col_extras = {}
+    cols = [ {'field': v, **col_extras.get(v, {})} for v in df.columns ]
+    rows = df.to_dict(orient='records')
+    grid = ui.aggrid(
+        {
+            'columnDefs': cols,
+            'rowData': rows,
+            'autoSizeStrategy': {
+                'type': 'fitCellContents',
+            },
+            # 'rowSelection': {'mode': 'multiRow'},
+            # 'stopEditingWhenCellsLoseFocus': True,
+        }
+    ).classes('auto-fit flex-grow w-full col-span-2 md:col-span-1')
+    
+    return grid
+

 ### Pages ###
 async def rt_chart_page():
    global LOOKBACK
    
    LOOKBACK = app.storage.user.get('lookback', LOOKBACK)
-    timer = ui.timer(REFRESH_INTERVAL_RT_SEC, update_tv)
+    timer_tv = ui.timer(REFRESH_INTERVAL_RT_SEC, update_tv)
+    # timer_sql = ui.timer(REFRESH_INTERVAL_SEC)
    
    # ui.query('.q-page').classes('flex flex-col h-screen')

@@ -182,22 +306,11 @@ async def rt_chart_page():
    #         ui.switch('▶️', value=True).bind_value_to(timer, 'active')
    #     with ui.column().style('position: absolute; right: 20px; font-family: monospace; align-self: center;'):
    #         ui.label('Atwater Trading - Funding Rate')
-    
+    ui.query('.nicegui-content').classes('p-0 w-full')
+    ui.query('.q-page').classes('flex')
+
    with ui.grid(columns=2, rows=2).classes('h-screen w-full flex-grow gap-2 auto-fit '):
-        aggrid = ui.aggrid({
-            'columnDefs': [
-                {'field': 'name', 'editable': True, 'sortable': True},
-                {'field': 'age', 'editable': True},
-                {'field': 'id'},
-            ],
-            'rowData': [
-                {'id': 0, 'name': 'Alice', 'age': 18},
-                {'id': 1, 'name': 'Bob', 'age': 21},
-                {'id': 2, 'name': 'Carol', 'age': 20},
-            ],
-            'rowSelection': {'mode': 'multiRow'},
-            'stopEditingWhenCellsLoseFocus': True,
-        }).classes('auto-fit flex-grow w-full col-span-2 md:col-span-1')
+        # aggrid = await create_bfr_aggrid()
        # with ui.element(tag='div').classes('auto-fit flex-grow w-full').style("height:100%; width: 100%;"):
            # with ui.tabs().classes('w-full') as tabs:
            #         one = ui.tab('One').classes('auto-fit flex-grow w-full').style("height:100%; width: 100%;")
@@ -210,31 +323,17 @@ async def rt_chart_page():
        ui.run_javascript(f'await create_tv(charts_list={CHARTS}, create_chart_options={CHARTS_OPTIONS});')
        
        with ui.element(tag='div').classes('col-span-2').style("height:100%; width: 100%;"):
-            with ui.tabs().classes('w-full') as tabs:
-                one = ui.tab('One')
-                two = ui.tab('Two')
-            with ui.tab_panels(tabs, value=two).classes('w-full').style("height:100%; width: 100%;"):
-                with ui.tab_panel(one):
-                    ui.label('First tab')
-                with ui.tab_panel(two):
-                    aggrid_2 = ui.aggrid({
-                        'columnDefs': [
-                            {'field': 'name', 'editable': True, 'sortable': True},
-                            {'field': 'age', 'editable': True},
-                            {'field': 'id'},
-                        ],
-                        'rowData': [
-                            {'id': 0, 'name': 'Alice', 'age': 18},
-                            {'id': 1, 'name': 'Bob', 'age': 21},
-                            {'id': 2, 'name': 'Carol', 'age': 20},
-                        ],
-                        'rowSelection': {'mode': 'multiRow'},
-                        'stopEditingWhenCellsLoseFocus': True,
-                    })
+            with ui.tabs().props('align=justify').classes('justify-start') as tabs:
+                tab_pnl = ui.tab('PnL').classes('justify-start')
+                tab_bfr = ui.tab('BFR').classes('justify-start')
+            with ui.tab_panels(tabs, value=tab_pnl).classes('w-full').style("height:100%; width: 100%;"):
+                with ui.tab_panel(tab_pnl):
+                    ag_pnl = await create_pnl_aggrid()
+                with ui.tab_panel(tab_bfr):
+                    ag_bfr = await create_bfr_aggrid()


-
-def root():
+async def root():
    app.add_static_files(max_cache_age=0, url_path='/static', local_directory=os.path.join(os.path.dirname(__file__), 'nicegui_modules/static'))
    ui.add_head_html('''
    <meta name="darkreader-lock">
@@ -251,4 +350,5 @@ def root():
        '/': rt_chart_page,
    }).classes('w-full')

-ui.run(root, storage_secret="123ABC", reload=True, dark=True, title='Atwater Trading') 
+
+ui.run(root, storage_secret="123ABC", reload=True, dark=True, title='Atwater Trading', port=9090)