通过Digital Ocean上的Celery Beat努力将对象保存到Django数据库

Question

I'm struggling to save objects through Celery Beat to my Django app (showing OHLC data).

This script works fine on local environment (saves 3M objects) but not on VPN like Digital Ocean. It saves a certain amount of objects (roughly 200K objects or 2GB) but then it removes other objects to add each new object, which is totally confusing.

My stack

• Django
• Redis
• Supervisor
• Ubuntu

I'm NOT using Supervisor on my local, so I think this is causing the issue but can't identify. Any feedback / help would be really appreciated.

Script

@periodic_task(
    # run_every=(crontab(minute='*/1')),
    run_every=(crontab(minute='*/60')),
    name="load_data",
    ignore_result=False
)
def load_data():
# Forex OHLC
TOKEN = MYTOKEN
con = fxcmpy.fxcmpy(access_token = TOKEN, log_level = 'error')
start = dt.datetime(2010, 1, 1)
stop = dt.datetime.today()
df = pd.DataFrame(list(DatasourceItem.objects.filter(datasource__sub_category__exact='Forex').values('symbol')))

for i in df['symbol']:

    datasource_item_obj = DatasourceItem.objects.get(symbol=i)

    Ohlc.objects.filter(datasource = datasource_item_obj).delete()

    if datasource_item_obj.base_symbol:
        base_symbol = datasource_item_obj.base_symbol
        tar_symbol = datasource_item_obj.tar_symbol
        mod_symbol = base_symbol + "/" + tar_symbol
        sys_symbol = base_symbol + tar_symbol
    else:
        sys_symbol = datasource_item_obj.symbol
        mod_symbol = datasource_item_obj.symbol

    data = con.get_candles(mod_symbol, period='D1', start=start, stop=stop)
    del data['askopen']
    del data['askclose']
    del data['askhigh']
    del data['asklow']
    del data['tickqty']
    data.columns = ['Open', 'Close', 'High', 'Low']
    data = data[['Open', 'High', 'Low',  'Close']]
    data.insert(loc=0, column='Symbol', value=sys_symbol)
    data.reset_index(level=0, inplace=True)
    data.dropna()
    # .values = return numpy array
    data_list = data.values.tolist()
    for row in data_list:
        new_price = Ohlc(time = row[0], symbol = row[1], open_price = row[2], high_price = row[3], low_price = row[4], close_price = row[5], datasource = datasource_item_obj)
        new_price.save()

# Stock OHLC
start = dt.datetime.now() - dt.timedelta(days=(365.25*5))
stop = dt.datetime.today()

df = pd.DataFrame(list(DatasourceItem.objects.filter(datasource__sub_category__exact='Stock').values('symbol')))
for i in df['symbol']:
    datasource_obj = DatasourceItem.objects.get(symbol=i)
    old_price = Ohlc.objects.filter(datasource = datasource_obj).delete()

    symbol = datasource_obj.symbol
    data = get_historical_data(symbol, start=start, stop=stop, output_format='pandas')
    del data['volume']
    data.columns = ['Open', 'High', 'Low', 'Close']
    data.insert(loc=0, column='Symbol', value=symbol)
    data.reset_index(level=0, inplace=True)
    data.dropna()
    data_list = data.values.tolist()
    for row in data_list:
        price = Ohlc(time = row[0], symbol = row[1], open_price = row[2], high_price = row[3], low_price = row[4], close_price = row[5], datasource = datasource_obj)
        price.save()

Answer 1

Hey it's happening due to number of transaction happening on database, so try to optimize the data creation query, for example you can use bulk create instead of crating each object individual.

price_list
for row in data_list:
    price = Ohlc(time = row[0], symbol = row[1], open_price = row[2], high_price = row[3], low_price = row[4], close_price = row[5], datasource = datasource_obj)
    price_list.append(price)
Ohlc.objects.bulk(price_list)

may be possibilty it will not create large set off data in one go, then break data in chunks 1000.