python 異步數據拉取錯誤：__aexit__/__enter__

Question

我正在嘗試使用 Python 的異步功能編寫代碼。 我有一個數據庫連接 class ，其中我有用於（斷開）連接數據庫以及獲取數據的代碼。 現在我想使用基於一個標識符的 fetch data 方法異步獲取數據。 代碼如下圖：

import pyexasol
import pandas as pd
import logging
from typing import Iterable
import asyncio
import tqdm


class Exa(object):
    def __init__(self, dsn: str = '1.2.3.4',
                 user: str = os.environ['UID'],
                 password: str = os.environ['PWD']):
        self.__dsn = dsn
        self.__user = user
        self.__password = password
        self.conn = None

    def __connect(self):
        if self.conn is None:
            try:
                self.conn = pyexasol.connect(dsn=self.__dsn, user=self.__user,
                    password=self.__password, encryption=True)
            except Exception as e:
                logging.error(f"Error in connecting with Exasol. Error is: {e}")

    def __disconnect(self):
        if self.conn is not None:
            try:
                self.conn.close()
            except Exception as e:
                logging.error(f"Exception in disconnecting DB. Error is {e}")
            self.conn = None

    def fetch(self, query: str, leave_connection_open: bool = False) -> pd.DataFrame:
        # connect and execute the query
        self.__connect()
        try:
            res = self.conn.export_to_pandas(query)
            res.columns = res.columns.str.lower()
        except Exception as e:
            self.__disconnect()
            return pd.DataFrame()
        if not leave_connection_open:
            self.__disconnect()
        return res

    def fetch_batch(self, pattern: str, replacement: Iterable,
                    query: str, batchsize: int = 5000) -> pd.DataFrame:
        res = asyncio.run(self._fetch_batch(pattern=pattern, replacement=replacement,
                                            query=query, batchsize=batchsize))
        return res

    async def _fetch_batch(self, pattern: str, replacement: Iterable,
                           query: str, batchsize: int = 5000) -> pd.DataFrame:

        replacement = list(replacement)
        # breaking into batches
        if any(isinstance(i, str) for i in replacement):
            batches = ["'" + "','".join(replacement[i:i + batchsize]) + "'"
                       for i in range(0, len(replacement), batchsize)]
        else:
            batches = [",".join(replacement[i:i + batchsize])
                       for i in range(0, len(replacement), batchsize)]
        # connecting and executing query in batches
        nbatches = len(batches)
        self.__connect()
        try:
            tasks = [self.__run_batch_query(query=query.replace(pattern, batches[i]),
                                            i=i, nbatches=nbatches) for i in range(nbatches)]
            # progress bar
            res = [await f for f in tqdm.tqdm(asyncio.as_completed(tasks), total=len(tasks))]
        except Exception as e:
            logging.error("Could not fetch batches of data. Error is: %s", e)
        '''finally:
            self.__disconnect()'''
        # dataframe concatenation
        res = pd.concat(res)
        res.columns = res.columns.str.lower()
        return res

    async def __run_batch_query(self, query: str,
                                i: int, nbatches: int) -> pd.DataFrame:
        logging.info("Fetching %d/%d", i + 1, nbatches)
        async with self.fetch(query=query, leave_connection_open=True) as resp:
            raw = await resp
        return raw

我正在運行此代碼：

from foo import Exa
db = Exa()

ids = db.fetch('select id from application limit 100')
ids1 = db.fetch_batch(pattern='IDS',
                         replacement=ids['id'],
                         query='select id from application where id in (IDS)',
                         batchsize=25)

但后來我得到如下錯誤：

ERROR:root:Could not fetch batches of data. Error is: __aexit__
Traceback (most recent call last):
  File "/home/priya/pydbutils/gitignored/foo2.py", line 85, in __run_batch_query
    async with self.fetch(query=query, leave_connection_open=True) as resp:
AttributeError: __aexit__

此外，如果我在沒有async的情況下將__run_batch_query()方法調用更改為self.fetch()方法，則錯誤將更改為：

ERROR:root:Could not fetch batches of data. Error is: __enter__
Traceback (most recent call last):
  File "/home/priya/pydbutils/gitignored/foo2.py", line 85, in __run_batch_query
    with self.fetch(query=query, leave_connection_open=True) as resp:
AttributeError: __enter__

如果這里有錯誤，請幫忙指出錯誤？

Answer 1

pyexasol 創造者在這里。

請注意，asyncio 不會為與 Exasol 相關的場景提供任何好處。 Ayncio 在單個 CPU 上運行並使用單個網絡連接，這會阻止任何有意義的擴展。

從 Exasol 服務器加載數據的最有效方法是：

• export_to_pandas()或export_to_callback()用於單個 Python 進程；
• export_parallel() + http_transport()用於多個 Python 進程；

請查看HTTP Transport (parallel)手冊頁以獲取解釋和示例。 這種方法可以線性擴展，您甚至可以在多個服務器上運行計算任務。

對於簡單的場景，如果您通過慢速（例如 WiFi）網絡傳輸大量數據，您可以考慮使用compression=True連接選項。