[英]Database still in use after a selenium test in Django
I have a Django project in which I'm starting to write Selenium tests.我有一个 Django 项目,我开始在其中编写 Selenium 测试。 The first one looking like this:第一个看起来像这样:
from django.contrib.staticfiles.testing import StaticLiveServerTestCase
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from core.models import User
from example import settings
BACH_EMAIL = "johann.sebastian.bach@classics.com"
PASSWORD = "password"
class TestImportCRMData(StaticLiveServerTestCase):
@classmethod
def setUpClass(cls):
super().setUpClass()
cls.webdriver = webdriver.Chrome()
cls.webdriver.implicitly_wait(10)
@classmethod
def tearDownClass(cls):
cls.webdriver.close()
cls.webdriver.quit()
super().tearDownClass()
def setUp(self):
self.admin = User.objects.create_superuser(email=BACH_EMAIL, password=PASSWORD)
def test_admin_tool(self):
self.webdriver.get(f"http://{settings.ADMIN_HOST}:{self.server_thread.port}/admin")
self.webdriver.find_element_by_id("id_username").send_keys(BACH_EMAIL)
self.webdriver.find_element_by_id("id_password").send_keys(PASSWORD)
self.webdriver.find_element_by_id("id_password").send_keys(Keys.RETURN)
self.webdriver.find_element_by_link_text("Users").click()
When I run it, the test pass but still ends with this error:当我运行它时,测试通过但仍然以这个错误结束:
Traceback (most recent call last):
File "C:\Users\pupeno\Documents\Eligible\code\example\venv\lib\site-packages\django\db\backends\utils.py", line 83, in _execute
return self.cursor.execute(sql)
psycopg2.OperationalError: database "test_example" is being accessed by other users
DETAIL: There is 1 other session using the database.
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "C:\Program Files\JetBrains\PyCharm 2018.2.4\helpers\pycharm\django_test_manage.py", line 168, in <module>
utility.execute()
File "C:\Program Files\JetBrains\PyCharm 2018.2.4\helpers\pycharm\django_test_manage.py", line 142, in execute
_create_command().run_from_argv(self.argv)
File "C:\Users\pupeno\Documents\Eligible\code\example\venv\lib\site-packages\django\core\management\commands\test.py", line 26, in run_from_argv
super().run_from_argv(argv)
File "C:\Users\pupeno\Documents\Eligible\code\example\venv\lib\site-packages\django\core\management\base.py", line 316, in run_from_argv
self.execute(*args, **cmd_options)
File "C:\Users\pupeno\Documents\Eligible\code\example\venv\lib\site-packages\django\core\management\base.py", line 353, in execute
output = self.handle(*args, **options)
File "C:\Program Files\JetBrains\PyCharm 2018.2.4\helpers\pycharm\django_test_manage.py", line 104, in handle
failures = TestRunner(test_labels, **options)
File "C:\Program Files\JetBrains\PyCharm 2018.2.4\helpers\pycharm\django_test_runner.py", line 255, in run_tests
extra_tests=extra_tests, **options)
File "C:\Program Files\JetBrains\PyCharm 2018.2.4\helpers\pycharm\django_test_runner.py", line 156, in run_tests
return super(DjangoTeamcityTestRunner, self).run_tests(test_labels, extra_tests, **kwargs)
File "C:\Users\pupeno\Documents\Eligible\code\example\venv\lib\site-packages\django\test\runner.py", line 607, in run_tests
self.teardown_databases(old_config)
File "C:\Users\pupeno\Documents\Eligible\code\example\venv\lib\site-packages\django\test\runner.py", line 580, in teardown_databases
keepdb=self.keepdb,
File "C:\Users\pupeno\Documents\Eligible\code\example\venv\lib\site-packages\django\test\utils.py", line 297, in teardown_databases
connection.creation.destroy_test_db(old_name, verbosity, keepdb)
File "C:\Users\pupeno\Documents\Eligible\code\example\venv\lib\site-packages\django\db\backends\base\creation.py", line 257, in destroy_test_db
self._destroy_test_db(test_database_name, verbosity)
File "C:\Users\pupeno\Documents\Eligible\code\example\venv\lib\site-packages\django\db\backends\base\creation.py", line 274, in _destroy_test_db
% self.connection.ops.quote_name(test_database_name))
File "C:\Users\pupeno\Documents\Eligible\code\example\venv\lib\site-packages\django\db\backends\utils.py", line 68, in execute
return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
File "C:\Users\pupeno\Documents\Eligible\code\example\venv\lib\site-packages\django\db\backends\utils.py", line 77, in _execute_with_wrappers
return executor(sql, params, many, context)
File "C:\Users\pupeno\Documents\Eligible\code\example\venv\lib\site-packages\django\db\backends\utils.py", line 85, in _execute
return self.cursor.execute(sql, params)
File "C:\Users\pupeno\Documents\Eligible\code\example\venv\lib\site-packages\django\db\utils.py", line 89, in __exit__
raise dj_exc_value.with_traceback(traceback) from exc_value
File "C:\Users\pupeno\Documents\Eligible\code\example\venv\lib\site-packages\django\db\backends\utils.py", line 83, in _execute
return self.cursor.execute(sql)
django.db.utils.OperationalError: database "test_example" is being accessed by other users
DETAIL: There is 1 other session using the database.
The problem of course is that the next run of the tests, the database still exists, so, the tests don't run without confirming deletion of the database.问题当然是在下一次运行测试时,数据库仍然存在,因此,在没有确认删除数据库的情况下,测试不会运行。
If I comment out the last line:如果我注释掉最后一行:
self.webdriver.find_element_by_link_text("Users").click()
then I don't get this error.然后我没有收到此错误。 I guess just because the database connection is not established.我猜只是因为没有建立数据库连接。 Sometimes it's 1 other session, sometimes it's up to 4. In one of the cases of 4 sessions, these were the running sessions:有时是另外 1 个会话,有时最多 4 个。在 4 个会话的其中一个情况下,这些是正在运行的会话:
select * from pg_stat_activity where datname = 'test_example';
100123 test_example 29892 16393 pupeno "" ::1 61967 2018-11-15 17:28:19.552431 2018-11-15 17:28:19.562398 2018-11-15 17:28:19.564623 idle SELECT "core_user"."id", "core_user"."password", "core_user"."last_login", "core_user"."is_superuser", "core_user"."email", "core_user"."is_staff", "core_user"."is_active", "core_user"."date_joined" FROM "core_user" WHERE "core_user"."id" = 1
100123 test_example 33028 16393 pupeno "" ::1 61930 2018-11-15 17:28:18.792466 2018-11-15 17:28:18.843383 2018-11-15 17:28:18.851828 idle SELECT "django_admin_log"."id", "django_admin_log"."action_time", "django_admin_log"."user_id", "django_admin_log"."content_type_id", "django_admin_log"."object_id", "django_admin_log"."object_repr", "django_admin_log"."action_flag", "django_admin_log"."change_message", "core_user"."id", "core_user"."password", "core_user"."last_login", "core_user"."is_superuser", "core_user"."email", "core_user"."is_staff", "core_user"."is_active", "core_user"."date_joined", "django_content_type"."id", "django_content_type"."app_label", "django_content_type"."model" FROM "django_admin_log" INNER JOIN "core_user" ON ("django_admin_log"."user_id" = "core_user"."id") LEFT OUTER JOIN "django_content_type" ON ("django_admin_log"."content_type_id" = "django_content_type"."id") WHERE "django_admin_log"."user_id" = 1 ORDER BY "django_admin_log"."action_time" DESC LIMIT 10
100123 test_example 14128 16393 pupeno "" ::1 61988 2018-11-15 17:28:19.767225 2018-11-15 17:28:19.776150 2018-11-15 17:28:19.776479 idle SELECT "core_firm"."id", "core_firm"."name", "core_firm"."host_name" FROM "core_firm" WHERE "core_firm"."id" = 1
100123 test_example 9604 16393 pupeno "" ::1 61960 2018-11-15 17:28:19.469197 2018-11-15 17:28:19.478775 2018-11-15 17:28:19.478788 idle COMMIT
I've been trying to find the minimum reproducible example of this problem, but so far I haven't succeeded.我一直试图找到这个问题的最小可重现示例,但到目前为止我还没有成功。
Any ideas what could be causing this or how to find out more about what the issue could be?任何想法可能导致此问题或如何了解更多有关问题的信息?
There has been an issue reported long back for the same ( bug #22424 ).很久以前就报告了一个问题(错误#22424 )。
One thing you need to make sure is that CONN_MAX_AGE
is set to 0
and not None
您需要确保的一件事是CONN_MAX_AGE
设置为0
而不是None
Also, you can use something like below in your teardown此外,您可以在拆解中使用以下内容
from django.db import connections
from django.db.utils import OperationalError
@classmethod
def tearDownClass(cls):
# Workaround for https://code.djangoproject.com/ticket/22414
# Persistent connections not closed by LiveServerTestCase, preventing dropping test databases
# https://github.com/cjerdonek/django/commit/b07fbca02688a0f8eb159f0dde132e7498aa40cc
def close_sessions(conn):
close_sessions_query = """
SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE
datname = current_database() AND
pid <> pg_backend_pid();
"""
with conn.cursor() as cursor:
try:
cursor.execute(close_sessions_query)
except OperationalError:
# We get kicked out after closing.
pass
for alias in connections:
connections[alias].close()
close_sessions(connections[alias])
print "Forcefully closed database connections."
Above code is from below URL以上代码来自以下网址
https://github.com/cga-harvard/Hypermap-Registry/blob/cd4efad61f18194ddab2c662aa431aa21dec03f4/hypermap/tests/test_csw.py https://github.com/cga-harvard/Hypermap-Registry/blob/cd4efad61f18194ddab2c662aa431aa21dec03f4/hypermap/tests/test_csw.py
This error message...这个错误信息...
django.db.utils.OperationalError: database "test_example" is being accessed by other users
DETAIL: There is 1 other session using the database.
...implies that there is an existing session using the database and the new session can't access the database. ...暗示存在使用该数据库的现有会话,而新会话无法访问该数据库。
Some more information regarding the Django version , Database type and version along with Selenium , ChromeDriver and Chrome versions would have helped us to debug this issue in a better way.有关Django版本、数据库类型和版本以及Selenium 、 ChromeDriver和Chrome版本的更多信息将帮助我们以更好的方式调试此问题。
However, you need to take care of a couple of things from Selenium perspective as follows:但是,您需要从Selenium 的角度处理以下几件事:
As you are initiating a new session on the next run of the tests you need to remove the line of code cls.webdriver.close()
as the next line of code cls.webdriver.quit()
will be enough to terminate the existing session.当您在下一次运行测试时启动新会话时,您需要删除代码行cls.webdriver.close()
因为下一行代码cls.webdriver.quit()
足以终止现有会话. As per the best practices you should invoke the quit()
method within the tearDown() {}
.根据最佳实践,您应该在tearDown() {}
调用quit()
方法。 Invoking quit()
DELETE
s the current browsing session through sending "quit" command with {"flags":["eForceQuit"]} and finally sends the GET request on /shutdown EndPoint
.通过发送带有{"flags":["eForceQuit"]} 的“quit”命令调用quit()
DELETE
s当前浏览会话,最后在/shutdown EndPoint
上发送GET请求。 Here is an example below :下面是一个例子:
1503397488598 webdriver::server DEBUG -> DELETE /session/8e457516-3335-4d3b-9140-53fb52aa8b74 1503397488607 geckodriver::marionette TRACE -> 37:[0,4,"quit",{"flags":["eForceQuit"]}] 1503397488821 webdriver::server DEBUG -> GET /shutdown
So on invoking quit()
method the Web Browser
session and the WebDriver
instance gets killed completely.因此,在调用quit()
方法时, Web Browser
会话和WebDriver
实例被完全杀死。 Hence you don't have to incorporate any additional steps which will be an overhead.因此,您不必合并任何额外的步骤,这将是一个开销。
You can find a detailed discussion in Selenium : How to stop geckodriver process impacting PC memory, without calling driver.quit()?您可以在Selenium 中找到详细讨论:如何在不调用 driver.quit() 的情况下停止影响 PC 内存的 geckodriver 进程?
So you need to:所以你需要:
Remove the implicitly_wait(10)
:删除implicitly_wait(10)
:
cls.webdriver.implicitly_wait(10)
Induce WebDriverWait while interacting with elements:在与元素交互时引入WebDriverWait :
WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.ID, "id_username"))).send_keys(BACH_EMAIL) self.webdriver.find_element_by_id("id_password").send_keys(PASSWORD) self.webdriver.find_element_by_id("id_password").send_keys(Keys.RETURN) WebDriverWait(driver, 20).until(EC.element_to_be_clickable((By.LINK_TEXT, "Users"))).click()
Now, as per the discussion Persistent connections not closed by LiveServerTestCase, preventing dropping test databases this issue was observed, reported, discussed within Djangov1.6 and fixed.现在,根据LiveServerTestCase 未关闭的持久连接的讨论,防止删除测试数据库此问题已在Djangov1.6 中观察、报告、讨论并修复。 The main issue was:主要问题是:
Whenever a PostgreSQL connection is marked as persistent (
CONN_MAX_AGE = None
) and a LiveServerTestCase is executed, the connection from the server thread is never closed, leading to inability to drop the test database.每当 PostgreSQL 连接被标记为持久性 (CONN_MAX_AGE = None
) 并执行 LiveServerTestCase 时,来自服务器线程的连接永远不会关闭,从而导致无法删除测试数据库。
This is exactly the reason why you see:这正是您看到的原因:
select * from pg_stat_activity where datname = 'test_example';
100123 test_example 29892 16393 pupeno "" ::1 61967 2018-11-15 17:28:19.552431 2018-11-15 17:28:19.562398 2018-11-15 17:28:19.564623 idle SELECT "core_user"."id", "core_user"."password", "core_user"."last_login", "core_user"."is_superuser", "core_user"."email", "core_user"."is_staff", "core_user"."is_active", "core_user"."date_joined" FROM "core_user" WHERE "core_user"."id" = 1
100123 test_example 33028 16393 pupeno "" ::1 61930 2018-11-15 17:28:18.792466 2018-11-15 17:28:18.843383 2018-11-15 17:28:18.851828 idle SELECT "django_admin_log"."id", "django_admin_log"."action_time", "django_admin_log"."user_id", "django_admin_log"."content_type_id", "django_admin_log"."object_id", "django_admin_log"."object_repr", "django_admin_log"."action_flag", "django_admin_log"."change_message", "core_user"."id", "core_user"."password", "core_user"."last_login", "core_user"."is_superuser", "core_user"."email", "core_user"."is_staff", "core_user"."is_active", "core_user"."date_joined", "django_content_type"."id", "django_content_type"."app_label", "django_content_type"."model" FROM "django_admin_log" INNER JOIN "core_user" ON ("django_admin_log"."user_id" = "core_user"."id") LEFT OUTER JOIN "django_content_type" ON ("django_admin_log"."content_type_id" = "django_content_type"."id") WHERE "django_admin_log"."user_id" = 1 ORDER BY "django_admin_log"."action_time" DESC LIMIT 10
100123 test_example 14128 16393 pupeno "" ::1 61988 2018-11-15 17:28:19.767225 2018-11-15 17:28:19.776150 2018-11-15 17:28:19.776479 idle SELECT "core_firm"."id", "core_firm"."name", "core_firm"."host_name" FROM "core_firm" WHERE "core_firm"."id" = 1
100123 test_example 9604 16393 pupeno "" ::1 61960 2018-11-15 17:28:19.469197 2018-11-15 17:28:19.478775 2018-11-15 17:28:19.478788 idle COMMIT
It was further observed that, even with CONN_MAX_AGE=None
, after LiveServerTestCase.tearDownClass()
, querying PostgreSQL's pg_stat_activity
shows a lingering connection in state idle (which was the connection created by the previous test in your case).进一步观察到,即使使用CONN_MAX_AGE=None
,在LiveServerTestCase.tearDownClass()
,查询 PostgreSQL 的pg_stat_activity
显示处于空闲状态的挥之不去的连接(这是在您的情况下由之前的测试创建的连接)。 So it was pretty evident that the idle connections are not closed when the thread terminates and the needle of supection was at:所以很明显,当线程终止并且怀疑指针位于:
LiveServerThread(threading.Thread)
which control the threads for running a live http server while the tests are running: LiveServerThread(threading.Thread)
控制在测试运行时运行实时 http 服务器的线程:
class LiveServerThread(threading.Thread): def __init__(self, host, static_handler, connections_override=None): self.host = host self.port = None self.is_ready = threading.Event() self.error = None self.static_handler = static_handler self.connections_override = connections_override super(LiveServerThread, self).__init__() def run(self): """ Sets up the live server and databases, and then loops over handling http requests. """ if self.connections_override: # Override this thread's database connections with the ones # provided by the main thread. for alias, conn in self.connections_override.items(): connections[alias] = conn try: # Create the handler for serving static and media files handler = self.static_handler(_MediaFilesHandler(WSGIHandler())) self.httpd = self._create_server(0) self.port = self.httpd.server_address[1] self.httpd.set_app(handler) self.is_ready.set() self.httpd.serve_forever() except Exception as e: self.error = e self.is_ready.set() def _create_server(self, port): return WSGIServer((self.host, port), QuietWSGIRequestHandler, allow_reuse_address=False) def terminate(self): if hasattr(self, 'httpd'): # Stop the WSGI server self.httpd.shutdown() self.httpd.server_close()
LiveServerTestCase(TransactionTestCase)
which basically does the same as TransactionTestCase
but also launches a live http server in a separate thread so that the tests may use another testing framework to be used by Selenium , instead of the built-in dummy client: LiveServerTestCase(TransactionTestCase)
基本上与TransactionTestCase
相同,但也在单独的线程中启动实时 http 服务器,以便测试可以使用Selenium使用的另一个测试框架,而不是内置的虚拟客户端:
class LiveServerTestCase(TransactionTestCase): host = 'localhost' static_handler = _StaticFilesHandler @classproperty def live_server_url(cls): return 'http://%s:%s' % (cls.host, cls.server_thread.port) @classmethod def setUpClass(cls): super(LiveServerTestCase, cls).setUpClass() connections_override = {} for conn in connections.all(): # If using in-memory sqlite databases, pass the connections to # the server thread. if conn.vendor == 'sqlite' and conn.is_in_memory_db(conn.settings_dict['NAME']): # Explicitly enable thread-shareability for this connection conn.allow_thread_sharing = True connections_override[conn.alias] = conn cls._live_server_modified_settings = modify_settings( ALLOWED_HOSTS={'append': cls.host}, ) cls._live_server_modified_settings.enable() cls.server_thread = cls._create_server_thread(connections_override) cls.server_thread.daemon = True cls.server_thread.start() # Wait for the live server to be ready cls.server_thread.is_ready.wait() if cls.server_thread.error: # Clean up behind ourselves, since tearDownClass won't get called in # case of errors. cls._tearDownClassInternal() raise cls.server_thread.error @classmethod def _create_server_thread(cls, connections_override): return LiveServerThread( cls.host, cls.static_handler, connections_override=connections_override, ) @classmethod def _tearDownClassInternal(cls): # There may not be a 'server_thread' attribute if setUpClass() for some # reasons has raised an exception. if hasattr(cls, 'server_thread'): # Terminate the live server's thread cls.server_thread.terminate() cls.server_thread.join() # Restore sqlite in-memory database connections' non-shareability for conn in connections.all(): if conn.vendor == 'sqlite' and conn.is_in_memory_db(conn.settings_dict['NAME']): conn.allow_thread_sharing = False @classmethod def tearDownClass(cls): cls._tearDownClassInternal() cls._live_server_modified_settings.disable() super(LiveServerTestCase, cls).tearDownClass()
The solution was to close only the non-overriden connections and was incorporated from this pull request / commit .解决方案是仅关闭未覆盖的连接,并从此pull request / commit 中合并。 The changes were:这些变化是:
In django/test/testcases.py
add:在django/test/testcases.py
添加:
finally: connections.close_all()
Add a new file tests/servers/test_liveserverthread.py
:添加一个新文件tests/servers/test_liveserverthread.py
:
from django.db import DEFAULT_DB_ALIAS, connections from django.test import LiveServerTestCase, TestCase class LiveServerThreadTest(TestCase): def run_live_server_thread(self, connections_override=None): thread = LiveServerTestCase._create_server_thread(connections_override) thread.daemon = True thread.start() thread.is_ready.wait() thread.terminate() def test_closes_connections(self): conn = connections[DEFAULT_DB_ALIAS] if conn.vendor == 'sqlite' and conn.is_in_memory_db(): self.skipTest("the sqlite backend's close() method is a no-op when using an in-memory database") # Pass a connection to the thread to check they are being closed. connections_override = {DEFAULT_DB_ALIAS: conn} saved_sharing = conn.allow_thread_sharing try: conn.allow_thread_sharing = True self.assertTrue(conn.is_usable()) self.run_live_server_thread(connections_override) self.assertFalse(conn.is_usable()) finally: conn.allow_thread_sharing = saved_sharing
In tests/servers/tests.py
remove:在tests/servers/tests.py
删除:
finally: TestCase.tearDownClass()
In tests/servers/tests.py
add:在tests/servers/tests.py
添加:
finally: if hasattr(TestCase, 'server_thread'): TestCase.server_thread.terminate()
Steps:脚步:
You can find a similar discussion in django.db.utils.IntegrityError: FOREIGN KEY constraint failed while executing LiveServerTestCases through Selenium and Python Django您可以在django.db.utils.IntegrityError: FOREIGN KEY constraint failed while execution LiveServerTestCases through Selenium and Python Django 中找到类似的讨论
Check active connections to know what causes the problem select * from pg_stat_activity;
检查活动连接以了解导致问题的原因select * from pg_stat_activity;
You can disable extensions:您可以禁用扩展:
@classmethod
def setUpClass(cls):
super().setUpClass()
options = webdriver.chrome.options.Options()
options.add_argument("--disable-extensions")
cls.webdriver = webdriver.Chrome(chrome_options=options)
cls.webdriver.implicitly_wait(10)
Then in teardown:然后在拆解中:
@classmethod
def tearDownClass(cls):
cls.webdriver.stop_client()
cls.webdriver.quit()
connections.close_all()
super().tearDownClass()
声明:本站的技术帖子网页,遵循CC BY-SA 4.0协议,如果您需要转载,请注明本站网址或者原文地址。任何问题请咨询:yoyou2525@163.com.