Pytest - 如何在参数化测试中首先通过参数排序测试执行

Question

I have the following parametrized test:

import pytest
@pytest.mark.parametrize("url1", ["url0001", "url0002", "url0003"], indirect=False)
class Test01:
    def test01(self, url1):
        print(url1)
        assert url1 == f"{url1}"


    def test02(self, url1):
        print(url1)
        assert url1 == f"{url1}"

Result:

venv/test_url_fixt.py::Test01::test01[url0001] PASSED                    [ 16%]url0001

venv/test_url_fixt.py::Test01::test01[url0002] PASSED                    [ 33%]url0002

venv/test_url_fixt.py::Test01::test01[url0003] PASSED                    [ 50%]url0003

venv/test_url_fixt.py::Test01::test02[url0001] PASSED                    [ 66%]url0001

venv/test_url_fixt.py::Test01::test02[url0002] PASSED                    [ 83%]url0002

venv/test_url_fixt.py::Test01::test02[url0003] PASSED                    [100%]url0003


============================== 6 passed in 0.03s ==============================

Process finished with exit code 0

Target result:

venv/test_url_fixt.py::Test01::test01[url0001] PASSED                    [ 16%]url0001

venv/test_url_fixt.py::Test01::test02[url0001] PASSED                    [ 33%]url0001

venv/test_url_fixt.py::Test01::test01[url0002] PASSED                    [ 50%]url0002

venv/test_url_fixt.py::Test01::test02[url0002] PASSED                    [ 66%]url0002

venv/test_url_fixt.py::Test01::test01[url0003] PASSED                    [ 83%]url0003

venv/test_url_fixt.py::Test01::test02[url0003] PASSED                    [100%]url0003


============================== 6 passed in 0.03s ==============================

Process finished with exit code 0

Info:
I just need to execute test01 with url1 = "url0001" and test02 with url1 = "url0001" first, then loop over all tests with the next parameter, eg test01 with url1 = "url0002", test02 with url1 = "url0002" and so on. I have the list of urls and want to execute properly set of tests with that list of urls.

update

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from allure_commons.types import AttachmentType
import pytest
from selenium.common.exceptions import NoSuchElementException
import time
import logging
import allure


driver = webdriver.Chrome(executable_path=r'C:\webdrivers\chromedriver.exe')


# Logger

logging.basicConfig(filename="C:/LPsLogs/test.log",
                    format='%(asctime)s: %(levelname)s: %(message)s',
                    datefmt='%m/%d/%Y %H:%M:%S')


logger = logging.getLogger()
logger.setLevel(logging.INFO)


wait = WebDriverWait(driver, 10)
driver.implicitly_wait(5)

ec = EC
goxp = driver.find_element_by_xpath
goid = driver.find_element_by_id
keys = Keys
original_window = driver.current_window_handle

# Urls
sslurl = "https://www.sslshopper.com/ssl-checker.html"


# Locators

# xpath
sslpch = "//h1"
u1chb1 = "//div[@id='checkData']/descendant::td[1]"
u1chb2 = "//div[@id='checkData']/descendant::td[3]"
u1chb3 = "//div[@id='checkData']/descendant::td[5]"
u1chb4 = "//div[@id='checkData']/descendant::td[7]"
u1chb5 = "//div[@id='checkData']/descendant::td[11]"
u1chb6 = "//div[@id='checkData']/descendant::td[15]"

# id
hostname = "hostname"
expdate = "cert_expiration_days"


@allure.severity(allure.severity_level.BLOCKER)
def test_go_sslcheck():
    logger.info("Testing started")
    driver.maximize_window()
    driver.get(sslurl)
    wait.until(EC.visibility_of_element_located((By.XPATH, sslpch)))
    sslchecker = driver.find_element_by_xpath(sslpch).text
    if sslchecker == 'SSL Checker':
        assert True
    else:
        logging.error('Error - def test_go_sslcheck module', exc_info=True)
        allure.attach(driver.get_screenshot_as_png(), name="testLoginScreen", attachment_type=AttachmentType.PNG)
        assert False


@pytest.mark.parametrize("url", ["google.com",
                                 "https://expired.badssl.com"
                                 ])
class TestSSL:

    @allure.severity(allure.severity_level.CRITICAL)
    def test_url_input(self, url):
        try:
            input_hostname = goid(hostname)
            input_hostname.send_keys(url)
            input_hostname.send_keys(keys.ENTER)
            time.sleep(2)
            print(f"{url} has been entered")
            assert True
            input_hostname.clear()
        except (Exception, NameError, AssertionError):
            allure.attach(driver.get_screenshot_as_png(), name="testLoginScreen", attachment_type=AttachmentType.PNG)
            logging.error('Error - url_input module', exc_info=True)
            assert False


    @allure.severity(allure.severity_level.CRITICAL)
    def test_url_checkbox1(self, url):
        wait.until(EC.visibility_of_element_located((By.XPATH, u1chb1)))
        u1chb1ch = driver.find_element_by_xpath(u1chb1).get_attribute('class')
        if u1chb1ch == 'passed':
            print(f"{url} - test_url_checkbox1 - PASS")
            assert True
        else:
            allure.attach(driver.get_screenshot_as_png(), name="testLoginScreen", attachment_type=AttachmentType.PNG)
            logging.error(f'{url} - checkbox 1 - FAILED', exc_info=True)
            assert False

    @allure.severity(allure.severity_level.CRITICAL)
    def test_url_checkbox2(self, url):
        wait.until(EC.visibility_of_element_located((By.XPATH, u1chb2)))
        u1chb2ch = driver.find_element_by_xpath(u1chb2).get_attribute('class')
        if u1chb2ch == 'passed':
            print(f"{url} - test_url_checkbox2 - PASS")
            assert True
        else:
            allure.attach(driver.get_screenshot_as_png(), name="testLoginScreen", attachment_type=AttachmentType.PNG)
            logging.error(f'{url} - checkbox 2 - FAILED', exc_info=True)
            assert False

    @allure.severity(allure.severity_level.CRITICAL)
    def test_url_checkbox3(self, url):
        wait.until(EC.visibility_of_element_located((By.XPATH, u1chb3)))
        u1chb3ch = driver.find_element_by_xpath(u1chb3).get_attribute('class')
        if u1chb3ch == 'passed':
            print(f"{url} - test_url_checkbox3 - PASS")
            assert True
        else:
            allure.attach(driver.get_screenshot_as_png(), name="testLoginScreen", attachment_type=AttachmentType.PNG)
            logging.error(f'{url} - checkbox 3 - FAILED', exc_info=True)
            assert False

    @allure.severity(allure.severity_level.CRITICAL)
    def test_url_checkbox4(self, url):
        wait.until(EC.visibility_of_element_located((By.XPATH, u1chb4)))
        u1chb4ch = driver.find_element_by_xpath(u1chb4).get_attribute('class')
        if u1chb4ch == 'passed':
            print(f"{url} - test_url_checkbox4 - PASS")
            assert True
        else:
            allure.attach(driver.get_screenshot_as_png(), name="testLoginScreen", attachment_type=AttachmentType.PNG)
            logging.error(f'{url} - checkbox 4 - FAILED', exc_info=True)
            assert False

    @allure.severity(allure.severity_level.CRITICAL)
    def test_url_checkbox5(self, url):
        wait.until(EC.visibility_of_element_located((By.XPATH, u1chb5)))
        u1chb5ch = driver.find_element_by_xpath(u1chb5).get_attribute('class')
        if u1chb5ch == 'passed':
            print(f"{url} - test_url_checkbox5 - PASS")
            assert True
        else:
            allure.attach(driver.get_screenshot_as_png(), name="testLoginScreen", attachment_type=AttachmentType.PNG)
            logging.error(f'{url} - checkbox 5 - FAILED', exc_info=True)
            assert False

    @allure.severity(allure.severity_level.CRITICAL)
    def test_url_checkbox6(self, url):
        wait.until(EC.visibility_of_element_located((By.XPATH, u1chb6)))
        u1chb6ch = driver.find_element_by_xpath(u1chb6).get_attribute('class')
        if u1chb6ch == 'passed':
            print(f"{url} - test_url_checkbox6 - PASS")
            assert True
        else:
            allure.attach(driver.get_screenshot_as_png(), name="testLoginScreen", attachment_type=AttachmentType.PNG)
            logging.error(f'{url} - checkbox 6 - FAILED', exc_info=True)
            assert False


    @allure.severity(allure.severity_level.NORMAL)
    def test_url_expdate_w(self, url):
        expdatech = driver.find_element_by_id(expdate).text
        if int(expdatech) > 7:
            print(f"{url} - expdate more than 7 days - PASS")
            assert True
        else:
            allure.attach(driver.get_screenshot_as_png(), name="testLoginScreen", attachment_type=AttachmentType.PNG)
            print(expdatech)
            logging.warning(f"{url} SSL certificate will expire in less than 7 days, days left: {expdatech}", exc_info=True)
            assert False


def test_close_browser():
    try:
        logger.info("Testing finished")
        driver.close()
        assert True
    except (Exception, NameError, AssertionError):
        logging.error('Error - close_browser module', exc_info=True)
        assert False

Result of first test: empty suite with conftest in comments and I have not knowledge hot to run it separately (tried to learn, but too hard for me for now).

Answer 1

You can change the order of the items in the pytest_collection_modifyitems hook. If you put this in your conftest.py :

def pytest_collection_modifyitems(config, items):
    def param_part(item):
        # check for the wanted module and test class
        if item.nodeid.startswith("test_urls.py::TestSSL::"):
            # find the start of the parameter part in the nodeid
            index = item.nodeid.find('[')
            if index > 0:
                # sort by parameter name
                return item.name[item.nodeid.index('['):]

        # for all other cases, sort by node id as usual
        return item.nodeid

    # re-order the items using the param_part function as key
    items[:] = sorted(items, key=param_part)

Here is the output of pytest -vv for your example:

================================================= test session starts =================================================
...
collected 6 items

test_paramtrize_order.py::Test01::test01[url0001] PASSED                                                         [ 16%]
test_paramtrize_order.py::Test01::test02[url0001] PASSED                                                         [ 33%]
test_paramtrize_order.py::Test01::test01[url0002] PASSED                                                         [ 50%]
test_paramtrize_order.py::Test01::test02[url0002] PASSED                                                         [ 66%]
test_paramtrize_order.py::Test01::test01[url0003] PASSED                                                         [ 83%]
test_paramtrize_order.py::Test01::test02[url0003] PASSED                                                         [100%]

================================================== 6 passed in 0.09s ==================================================

This will change the order only for the parametrized tests in the module test_urls.py and the class TestSSL . If you want to have this for more or all parametrized tests, you can adapt or remove the check in param_part .