[英]Python mysql (using pymysql) auto reconnect
I'm not sure if this is possible, but I'm looking for a way to reconnect to mysql database when the connection is lost.我不确定这是否可能,但我正在寻找一种在连接丢失时重新连接到 mysql 数据库的方法。 All the connections are held in a gevent queue but that shouldn't matter I think.
所有连接都保存在 gevent 队列中,但我认为这无关紧要。 I'm sure if I put some time in, I can come up with a way to reconnect to the database.
我敢肯定,如果我花一些时间,我可以想出一种重新连接到数据库的方法。 However I was glancing pymysql code and I saw that there is a 'ping' method in Connection class, which I'm not sure exactly how to use.
但是,我浏览了 pymysql 代码,发现 Connection 类中有一个“ping”方法,我不确定如何使用。
The method looks like it will reconnect first time but after that it switched the reconnect flag to False again?该方法看起来会第一次重新连接,但之后它再次将重新连接标志切换为 False? Can I use this method, or is there a different way to establish connection if it is lost?
我可以使用这种方法吗,或者如果连接丢失,是否有不同的方法来建立连接? Even if it is not pymysql how do people tackle, database servers going down and having to re-establish connection to mysql server?
即使不是 pymysql,人们如何处理,数据库服务器宕机并且不得不重新建立与 mysql 服务器的连接?
def ping(self, reconnect=True):
''' Check if the server is alive '''
if self.socket is None:
if reconnect:
self._connect()
reconnect = False
else:
raise Error("Already closed")
try:
self._execute_command(COM_PING, "")
return self._read_ok_packet()
except Exception:
if reconnect:
self._connect()
return self.ping(False)
else:
raise
Well, I've got the same problem in my application and I found a method on the PyMySQL documentation that pings to the server and check if the connection was closed or not, if it was closed, then it reconnects again.好吧,我在我的应用程序中遇到了同样的问题,我在PyMySQL 文档上找到了一个方法,它可以 ping 到服务器并检查连接是否关闭,如果它已关闭,则它会再次重新连接。
from pymysql import connect
from pymysql.cursors import DictCursor
# create the connection
connection = connect(host='host', port='port', user='user',
password='password', db='db',
cursorclass=DictCursor)
# get the cursor
cursor = connection.cursor()
# if the connection was lost, then it reconnects
connection.ping(reconnect=True)
# execute the query
cursor.execute(query)
I hope it helps.我希望它有所帮助。
Finally got a working solution, might help someone.终于找到了一个可行的解决方案,可能会对某人有所帮助。
from gevent import monkey
monkey.patch_socket()
import logging
import gevent
from gevent.queue import Queue
import pymysql as db
logging.basicConfig(level=logging.DEBUG)
LOGGER = logging.getLogger("connection_pool")
class ConnectionPool:
def __init__(self, db_config, time_to_sleep=30, test_run=False):
self.username = db_config.get('user')
self.password = db_config.get('password')
self.host = db_config.get('host')
self.port = int(db_config.get('port'))
self.max_pool_size = 20
self.test_run = test_run
self.pool = None
self.time_to_sleep = time_to_sleep
self._initialize_pool()
def get_initialized_connection_pool(self):
return self.pool
def _initialize_pool(self):
self.pool = Queue(maxsize=self.max_pool_size)
current_pool_size = self.pool.qsize()
if current_pool_size < self.max_pool_size: # this is a redundant check, can be removed
for _ in xrange(0, self.max_pool_size - current_pool_size):
try:
conn = db.connect(host=self.host,
user=self.username,
passwd=self.password,
port=self.port)
self.pool.put_nowait(conn)
except db.OperationalError, e:
LOGGER.error("Cannot initialize connection pool - retrying in {} seconds".format(self.time_to_sleep))
LOGGER.exception(e)
break
self._check_for_connection_loss()
def _re_initialize_pool(self):
gevent.sleep(self.time_to_sleep)
self._initialize_pool()
def _check_for_connection_loss(self):
while True:
conn = None
if self.pool.qsize() > 0:
conn = self.pool.get()
if not self._ping(conn):
if self.test_run:
self.port = 3306
self._re_initialize_pool()
else:
self.pool.put_nowait(conn)
if self.test_run:
break
gevent.sleep(self.time_to_sleep)
def _ping(self, conn):
try:
if conn is None:
conn = db.connect(host=self.host,
user=self.username,
passwd=self.password,
port=self.port)
cursor = conn.cursor()
cursor.execute('select 1;')
LOGGER.debug(cursor.fetchall())
return True
except db.OperationalError, e:
LOGGER.warn('Cannot connect to mysql - retrying in {} seconds'.format(self.time_to_sleep))
LOGGER.exception(e)
return False
# test (pytest compatible) -------------------------------------------------------------------------------------------
import logging
from src.py.ConnectionPool import ConnectionPool
logging.basicConfig(level=logging.DEBUG)
LOGGER = logging.getLogger("test_connection_pool")
def test_get_initialized_connection_pool():
config = {
'user': 'root',
'password': '',
'host': '127.0.0.1',
'port': 3305
}
conn_pool = ConnectionPool(config, time_to_sleep=5, test_run=True)
pool = conn_pool.get_initialized_connection_pool()
# when in test run the port will be switched back to 3306
# so the queue size should be 20 - will be nice to work
# around this rather than test_run hack
assert pool.qsize() == 20
The easiest way is to check the connection right before sending a query.最简单的方法是在发送查询之前检查连接。
You can do this by creating a small class that contains two methods: connect
and query
:您可以通过创建一个包含两个方法的小类来做到这一点:
connect
和query
:
import pymysql
import pymysql.cursors
class DB:
def connect(self):
self.conn = pymysql.connect(
host=hostname,
user=username,
password=password,
db=dbname,
charset='utf8mb4',
cursorclass=pymysql.cursors.DictCursor,
port=3306)
def query(self, sql):
try:
cursor = self.conn.cursor()
cursor.execute(sql)
except pymysql.OperationalError:
self.connect()
cursor = self.conn.cursor()
cursor.execute(sql)
return cursor
db = DB()
Now, whenever you send a query using db.query("example SQL")
the request is automatically prepared to encounter a connection error and reconnects using self.connect()
if it needs to.现在,每当您使用
db.query("example SQL")
发送查询时,请求都会自动准备好遇到连接错误,并在需要时使用self.connect()
重新连接。
Remember: This is a simplified example.请记住:这是一个简化的示例。 Normally, you would want to let PyMySQL help you escape special characters in your queries.
通常,您希望让 PyMySQL 帮助您转义查询中的特殊字符。 To do that, you would have to add a 2nd parameter in the
query
method and go from there.为此,您必须在
query
方法中添加第二个参数并从那里开始。
the logic is quite simple, if connection close then try to reconnect for several times in this case I use max tries for 15 times to reconnect or ping.逻辑很简单,如果连接关闭然后尝试重新连接几次,在这种情况下我使用最大尝试 15 次重新连接或 ping。
import pymysql, pymysql.cursors
conn = pymysql.connect(
host=hostname,
user=username,
password=password,
db=dbname,
charset='utf8mb4',
cursorclass=pymysql.cursors.DictCursor,
)
cursor = conn.cursor()
# you can do transactions to database and when you need conn later, just make sure the server is still connected
if conn.open is False:
max_try = 15
try = 0
while conn.open is False:
if try < max_try:
conn.ping() # autoreconnect is true by default
try +=1
# check the conn again to make sure it connected
if conn.open:
# statements when conn is successfully reconnect to the server
else:
# it must be something wrong : server, network etc
Old but I encountered a similar problem for accessing hosted db within programs.旧的,但我在访问程序中的托管数据库时遇到了类似的问题。 The solution I ended up using was to create a decorator to automatically reconnect when making a query.
我最终使用的解决方案是创建一个装饰器,以便在进行查询时自动重新连接。
given a connection function:给定一个连接函数:
def connect(self):
self.conn = mysql.connector.connect(host=self.host, user=self.user,
database=self.database, password=self.password)
self.cursor = self.conn.cursor()
print("Established connectionn...")
I created我建立
def _reconnect(func):
@wraps(func)
def rec(self,*args,**kwargs):
try:
result = func(self,*args,**kwargs)
return result
except (mysql.connector.Error, mysql.connector.Warning) as e:
self.connect()
result = func(self,*args,**kwargs)
return result
return rec
Such that any function using the connection can now be decorated as so这样任何使用连接的函数现在都可以这样装饰
@_reconnect
def check_user_exists(self,user_id):
self.cursor.execute("SELECT COUNT(*) FROM _ where user_id={};".format(user_id))
if self.cursor.fetchall()[0][0]==0:
return False
else:
return True
This decorator will re-establish a connection and rerun any function involving a query to the db.此装饰器将重新建立连接并重新运行任何涉及到数据库查询的函数。
You can use a property to keep the connection alive every time you do querying:每次进行查询时,您都可以使用属性来保持连接处于活动状态:
import pymysql
import pymysql.cursors
import pandas as pd
class DB:
def __init__(self, hostname='1.1.1.1', username='root', password='password',
database=None, port=3306, charset="utf8mb4"):
self.hostname = hostname
self.database = database
self.username = username
self.password = password
self.port = port
self.charset = charset
self.connect()
@property
def conn(self):
if not self.connection.open:
print('Going to reconnect')
self.connection.ping(reconnect=True)
return self.connection
def connect(self):
self.connection = pymysql.connect(
host=self.hostname,
user=self.username,
password=self.password,
db=self.database,
charset=self.charset,
cursorclass=pymysql.cursors.DictCursor,
port=self.port)
def query(self, sql):
return pd.read_sql_query(sql, con=self.conn)
db = DB(hostname='1.1.1.1', username='root', password='password', database=None, port=3306, charset="utf8mb4")
声明:本站的技术帖子网页,遵循CC BY-SA 4.0协议,如果您需要转载,请注明本站网址或者原文地址。任何问题请咨询:yoyou2525@163.com.