Funding_Rate/ng.py

import os
from nicegui import ui, app, html
from sqlalchemy import create_engine, text
# import requests
import pandas as pd
import json
# import time
# import re
import valkey
import asyncio
from datetime import datetime
from dataclasses import dataclass, field
from sqlalchemy.ext.asyncio import create_async_engine
from typing import AsyncContextManager
# from random import random
# from nicegui_modules import data
# from nicegui_modules import ui_components

ALLOW_BODY_SCROLL: bool = True
LOOKBACK: int = 60
LOOKBACK_RT_TV_MAX_POINTS: int = 3000
REFRESH_INTERVAL_SEC: float = 10
REFRESH_INTERVAL_RT_SEC: float = 1/10
# CON: AsyncContextManager
# ENGINE = create_async_engine('mysql+asyncmy://root:pwd@localhost/fund_rate')
ENGINE = create_engine('mysql+pymysql://root:pwd@localhost/fund_rate')



VALKEY = valkey.Valkey(host='localhost', port=6379, db=0, decode_responses=True)

CHARTS = [
    {
        'type': 'AREA',
        'autoscaleInfoProvider': False,
        'data': [],
        'options': {
            'color': '#94fcdf',
            'priceScaleId': 'right',
            'topColor': '#94fcdf', 
            'bottomColor': 'rgba(112, 249, 210, 0.28)',
            'invertFilledArea': True
        }
    },
    {
        'type': 'AREA',
        'autoscaleInfoProvider': False,
        'data': [],
        'options': {
            'color': '#dd7525',
            'priceScaleId': 'right',
            'topColor': '#94fcdf', 
            'bottomColor': 'rgba(249, 167, 112, 0.28)',
            'invertFilledArea': False
        },
    },
    {
        'type': 'LINE',
        'autoscaleInfoProvider': [-0.1, 0.1],
        'data': [],
        'options': {
            'color': '#ea0707',
            'priceScaleId': 'left',
        },
    },    
    {
        'type': 'LINE',
        'autoscaleInfoProvider': False,
        'data': [],
        'options': {
            'color': '#009b12',
            'priceScaleId': 'left',
        },
    },    
    {
        'type': 'LINE',
        'autoscaleInfoProvider': False,
        'data': [],
        'options': {
            'color': '#ffffff',
            'priceScaleId': 'left',
        },
    },    
]

CHARTS_OPTIONS = { 
    'crosshair': 'NORMAL',
    'autoSize': True,
    'toolbox': True,
    'timeScale': {
        'timeVisible': True,   # // Shows HH:mm on x-axis
        'secondsVisible': True # // Optional: show seconds
    },
    'rightPriceScale': {
        'visible': True,
        'autoScale': True
    },
    'leftPriceScale': {
        'visible': True
    },        
    'layout': {
        'background': { 'type': 'solid', 'color': '#222' },
        'textColor': '#DDD',
    },
    'grid': {
        'vertLines': {
            'color': '#e1e1e1', # // Set vertical line color
            'visible': True,
            'style': 2, # // 0: Solid, 1: Dashed, 2: Dotted, 3: LargeDashed, 4: SparseDotted
        },
        'horzLines': {
            'color': '#e1e1e1', # // Set horizontal line color
            'visible': True,
            'style': 2,
        },
    }
}

### Data ###
async def get_bfr_master_data() -> pd.DataFrame:
    df = pd.DataFrame(json.loads(VALKEY.get('fr_engine_best_fund_rate_master')))  # ty:ignore[invalid-argument-type]    
    df.reset_index(drop=True)
    df['id'] = df.index
    return df

async def get_trades_hist() -> pd.DataFrame:
    start_ts = (round(datetime.now().timestamp()*1000)-(60*60*24*1000))
    ### ASTER ###
    aster_orders = text(f'''
    SELECT *
    FROM fr_aster_user_order_trade
    WHERE timestamp_arrival > {start_ts}
    ''')
    df_aster_orders = pd.read_sql(aster_orders, con=ENGINE)
    if len(df_aster_orders) < 1:
        return pd.DataFrame()
    df_aster_orders['timestamp_dt'] = pd.to_datetime(df_aster_orders['timestamp_transaction'], unit='ms')
    df_aster_orders_fill = df_aster_orders.loc[df_aster_orders['execution_type']=='TRADE',:]
    df_aster_orders_fill = df_aster_orders_fill[['timestamp_transaction','order_trade_time_ts','timestamp_dt','order_id','trade_id','client_order_id','order_status','side','last_filled_qty','filled_accumulated_qty','commission','last_filled_price','realized_profit']].reset_index(drop=True)

    df_aster_trades = df_aster_orders_fill.groupby('order_id').agg({'timestamp_transaction': 'first','order_trade_time_ts':'last','order_status':'last','side':'last','last_filled_qty':'sum','filled_accumulated_qty':'last','commission':'sum','last_filled_price':'mean','realized_profit':'sum'}).reset_index()
    df_aster_trades['is_mkt_maker'] = df_aster_trades['commission'] == 0.00
    df_aster_trades['timestamp_ts'] = pd.to_datetime(df_aster_trades['order_trade_time_ts'], unit='ms')

    df_aster_trades = df_aster_trades.rename({'order_status':'status','filled_accumulated_qty':'filled_qty','commission':'payed_fee','last_filled_price':'price'}, axis=1)

    ### EXTEND ###
    # Load and Transform Orders
    extend_orders = text(f'''
        SELECT *
        FROM fr_extended_user_order
        WHERE timestamp_arrival > {start_ts}
    ''')
    df_extend_orders = pd.read_sql(extend_orders, con=ENGINE)
    if len(df_extend_orders) < 1:
        return pd.DataFrame()
    df_extend_orders['timestamp_dt'] = pd.to_datetime(df_extend_orders['updated_time_ts'], unit='ms')
    df_extend_orders_fill = df_extend_orders.loc[df_extend_orders['status'].isin(['FILLED','PARTIALLY_FILLED']),:]
    df_extend_orders_fill = df_extend_orders_fill[['created_time_ts','updated_time_ts','timestamp_dt','order_id','external_id','status','side','qty','filled_qty','payed_fee','price','averagePrice']].reset_index(drop=True)

    # Trades
    df_extend_trades = df_extend_orders_fill.groupby('order_id').agg({'created_time_ts':'first','updated_time_ts':'last','status': 'last','side': 'last', 'filled_qty':'last','payed_fee':'sum','price':'last'}).reset_index()
    df_extend_trades['duration_sec_ast'] = ( df_extend_trades['updated_time_ts'] - df_extend_trades['created_time_ts'] ) / 1000
    df_extend_trades['is_mkt_maker'] = df_extend_trades['payed_fee'] == 0.00
    df_extend_trades['timestamp_ts'] = pd.to_datetime(df_extend_trades['updated_time_ts'], unit='ms')
    
    def tie_trades_together_get_extend_from_aster(row):
        row = row.to_frame().T
        row.index=[1]

        extend_row = df_extend_trades[['order_id','timestamp_ts','status','side','filled_qty','payed_fee','price','is_mkt_maker']].loc[df_extend_trades['timestamp_ts']>row['timestamp_ts'].iloc[0],:].iloc[0]
        extend_row = extend_row.to_frame().T
        extend_row.index=[1]

        return_row = row.merge(extend_row, left_index=True, right_index=True, suffixes=('_ast','_ext'))

        return return_row.iloc[0]

    df_comb_trades = df_aster_trades[['order_id','timestamp_ts','status','side','filled_qty','payed_fee','price','is_mkt_maker']].apply(tie_trades_together_get_extend_from_aster, axis=1)
    df_comb_trades['buy_price'] = df_comb_trades['price_ast'].where(df_comb_trades['side_ast']=='BUY', df_comb_trades['price_ext'])
    df_comb_trades['sell_price'] = df_comb_trades['price_ast'].where(df_comb_trades['side_ast']=='SELL', df_comb_trades['price_ext'])
    df_comb_trades['buy_qty'] = df_comb_trades['filled_qty_ast'].where(df_comb_trades['side_ast']=='BUY', df_comb_trades['filled_qty_ext'])
    df_comb_trades['sell_qty'] = df_comb_trades['filled_qty_ast'].where(df_comb_trades['side_ast']=='SELL', df_comb_trades['filled_qty_ext'])
    df_comb_trades['buy_side'] = df_comb_trades['order_id_ast'].where(df_comb_trades['side_ast']=='BUY', df_comb_trades['order_id_ext'])
    df_comb_trades['buy_side'] = df_comb_trades['order_id_ast'] == df_comb_trades['buy_side']
    df_comb_trades['buy_side'] = df_comb_trades['buy_side'].replace(True, 'ASTER').replace(False,'EXTEND')

    df_comb_trades['per_trade_pnl'] = ( ( df_comb_trades['sell_price'] - df_comb_trades['buy_price'] ) * df_comb_trades['sell_qty'] ) - df_comb_trades['payed_fee_ast'] - df_comb_trades['payed_fee_ext']
    df_comb_trades['per_trade_pnl_pct'] = ( (df_comb_trades['sell_price']*df_comb_trades['sell_qty']) - (df_comb_trades['buy_price']*df_comb_trades['buy_qty']) ) / (df_comb_trades['buy_price']*df_comb_trades['buy_qty'])
    
        
    df = df_comb_trades.apply(lambda x: x.dt.strftime('%Y-%m-%d %H:%M:%S.%f') if hasattr(x, 'dt') else x)
    
    df.reset_index(drop=True)
    df['id'] = df.index
    return df

### Utils ###
def update_body_scroll(e=None, bool_override=False):
    if e is None:
        if bool_override:
            ui.query('body').style('height: 100%; overflow-y: auto;')
        else:
            ui.query('body').style('height: 100%; overflow-y: hidden;')
    else:
        if e.value:
            ui.query('body').style('height: 100%; overflow-y: auto;')
        else:
            ui.query('body').style('height: 100%; overflow-y: hidden;')

### Callbacks ###
async def update_tv():
    series_update_aster_tob = json.loads(VALKEY.get('fut_ticker_aster'))  # ty:ignore[invalid-argument-type]
    series_update_extend_tob = json.loads(VALKEY.get('fut_ticker_extended')) # ty:ignore[invalid-argument-type]
    series_update_algo_status = json.loads(VALKEY.get('algo_status')) # ty:ignore[invalid-argument-type] 
    
    timestamp_aster_tob = round( ( series_update_aster_tob['timestamp_transaction'] / 1000 ) , 2)
    timestamp_extend_tob = round( ( series_update_extend_tob['timestamp_msg'] / 1000 ) , 2)
    timestamp_algo_status = round( ( series_update_algo_status['last_update_ts_ms'] / 1000 ) , 2)
    
    value_aster_tob = ( float(series_update_aster_tob['best_ask_px']) + float(series_update_aster_tob['best_bid_px']) ) / 2 
    value_extend_tob = ( float(series_update_extend_tob['best_ask_px']) + float(series_update_extend_tob['best_bid_px']) ) / 2
    value_algo_model_ratio = float(series_update_algo_status['model_ratio'])*1_000
    value_algo_current_ratio = float(series_update_algo_status['current_ratio'])*1_000
    value_algo_expected_alpha = float(series_update_algo_status['expected_alpha'])*1_000

    data_list = [
        {
            'timestamp': timestamp_aster_tob,
            'value': value_aster_tob,
        },
        {
            'timestamp': timestamp_extend_tob,
            'value': value_extend_tob,
        },
        {
            'timestamp': timestamp_algo_status,
            'value': value_algo_model_ratio,
        },
        {
            'timestamp': timestamp_algo_status,
            'value': value_algo_current_ratio,
        },
        {
            'timestamp': timestamp_algo_status,
            'value': value_algo_expected_alpha,
        },
    ]

    ui.run_javascript(f'await update_tv(data_list={data_list}, lookback_max_points={LOOKBACK_RT_TV_MAX_POINTS});')

async def create_bfr_aggrid() -> ui.aggrid:
    df = await get_bfr_master_data()
    col_extras = {
        'symbol_ast': {
            'editable': False, 
            'sortable': True
        }
    }
    cols = [ {'field': v, **col_extras.get(v, {})} for v in df.columns ]
    rows = df.to_dict(orient='records')
    grid = ui.aggrid(
        {
            'columnDefs': cols,
            'rowData': rows,
            'autoSizeStrategy': {
                'type': 'fitCellContents',
            },
            # 'rowSelection': {'mode': 'multiRow'},
            # 'stopEditingWhenCellsLoseFocus': True,
        }
    ).classes('auto-fit flex-grow w-full col-span-2 md:col-span-1')
    
    return grid

async def create_pnl_aggrid() -> ui.aggrid:
    df = await get_trades_hist()
    col_extras = {}
    cols = [ {'field': v, **col_extras.get(v, {})} for v in df.columns ]
    rows = df.to_dict(orient='records')
    grid = ui.aggrid(
        {
            'columnDefs': cols,
            'rowData': rows,
            'autoSizeStrategy': {
                'type': 'fitCellContents',
            },
            # 'rowSelection': {'mode': 'multiRow'},
            # 'stopEditingWhenCellsLoseFocus': True,
        }
    ).classes('auto-fit flex-grow w-full col-span-2 md:col-span-1')
    
    return grid


### Pages ###
async def rt_chart_page():
    global LOOKBACK
    
    LOOKBACK = app.storage.user.get('lookback', LOOKBACK)
    timer_tv = ui.timer(REFRESH_INTERVAL_RT_SEC, update_tv)
    # timer_sql = ui.timer(REFRESH_INTERVAL_SEC)
    
    # ui.query('.q-page').classes('flex flex-col h-screen')

    # with ui.row():
    #     with ui.column():
    #         ui.switch('☸︎', value=ALLOW_BODY_SCROLL, on_change=lambda e: update_body_scroll(e))
    #     with ui.column():
    #         ui.switch('▶️', value=True).bind_value_to(timer, 'active')
    #     with ui.column().style('position: absolute; right: 20px; font-family: monospace; align-self: center;'):
    #         ui.label('Atwater Trading - Funding Rate')
    ui.query('.nicegui-content').classes('p-0 w-full')
    ui.query('.q-page').classes('flex')

    with ui.grid(columns=2, rows=2).classes('h-screen w-full flex-grow gap-2 auto-fit '):
        # aggrid = await create_bfr_aggrid()
        # with ui.element(tag='div').classes('auto-fit flex-grow w-full').style("height:100%; width: 100%;"):
            # with ui.tabs().classes('w-full') as tabs:
            #         one = ui.tab('One').classes('auto-fit flex-grow w-full').style("height:100%; width: 100%;")
            #         two = ui.tab('Two').classes('auto-fit flex-grow w-full').style("height:100%; width: 100%;")
            # with ui.tab_panels(tabs, value=two).classes('auto-fit flex-grow w-full').style("height:100%; width: 100%;"):
            #     with ui.tab_panel(one).classes('auto-fit flex-grow w-full').style("height:100%; width: 100%;"):
            #         ui.label('First tab')
            #     with ui.tab_panel(two).classes('auto-fit flex-grow w-full').style("height:100%; width: 100%;"):
        ui.html('<div id="tv" style="height:100%; width: 100%;"></div>', sanitize=False).classes('auto-fit flex-grow w-full col-span-2 md:col-span-1')
        ui.run_javascript(f'await create_tv(charts_list={CHARTS}, create_chart_options={CHARTS_OPTIONS});')
        
        with ui.element(tag='div').classes('col-span-2').style("height:100%; width: 100%;"):
            with ui.tabs().props('align=justify').classes('justify-start') as tabs:
                tab_pnl = ui.tab('PnL').classes('justify-start')
                tab_bfr = ui.tab('BFR').classes('justify-start')
            with ui.tab_panels(tabs, value=tab_pnl).classes('w-full').style("height:100%; width: 100%;"):
                with ui.tab_panel(tab_pnl):
                    ag_pnl = await create_pnl_aggrid()
                with ui.tab_panel(tab_bfr):
                    ag_bfr = await create_bfr_aggrid()


async def root():
    app.add_static_files(max_cache_age=0, url_path='/static', local_directory=os.path.join(os.path.dirname(__file__), 'nicegui_modules/static'))
    ui.add_head_html('''
    <meta name="darkreader-lock">
    <link rel="stylesheet" type="text/css" href="/static/styles.css">
    <script type="text/javascript" src="https://unpkg.com/lightweight-charts/dist/lightweight-charts.standalone.production.js"></script>
    <script src="/static/script.js"></script>
    '''
    )

    # ui.add_head_html('<meta name="darkreader-lock">')
    update_body_scroll(bool_override=ALLOW_BODY_SCROLL)

    ui.sub_pages({
        '/': rt_chart_page,
    }).classes('w-full')


ui.run(root, storage_secret="123ABC", reload=True, dark=True, title='Atwater Trading', port=8060)