I want to collect data about test to an external database, where I would collect all assert and parse the expected value versus the actual result.

To describe what I want in code:

test_something.py:

def test_me_1(send_to_test):
    """
    Docstring to pad even more
    """
    x,y = send_to_test(3)
    assert x == 3**2
    assert y == 3**3
    assert y == 3**2 # I want this to fail intentionally


conftest.py:

@pytest.fixture(scope='function')
def send_to_test() -> Callable:
    return lambda x: (x ** 2, x ** 3)

The data I want to collect is (Any other way is fine as long as I get this data):

test_data = [{'func': 'test_me_1', 'variable': 'x', 'expected_value': 9, 'actual_value': 9},
             {'func': 'test_me_1', 'variable': 'y', 'expected_value': 27, 'actual_value': 27},
             {'func': 'test_me_1', 'variable': 'y', 'expected_value': 27, 'actual_value': 9}]

My current idea is a wrapper, that can easily be implemented to existing test and will collect the assert statements and their value.
Collecting the number of assert statements is straightforward if I use AST, but these are good only for static analysis. I've also tried to use the inspect module, but have hard time figuring out how to use it for this purpose

My poor attempt thus far (print statements are temporary since I want to collect the data)

def get_this(func: Callable):
    @functools.wraps(func)
    def wrapper(*arg, **kwargs):
        print(func.__name__)
        func_content = inspect.getsource(func)
        func_root = ast.parse(func_content)
        for node in func_root.body:
            for content in node.body:
                if isinstance(content, ast.Assert):
                    print(content)
        return func(*arg, **kwargs)

    return wrapper

CodePudding user response：

You could try this:

import inspect
import json
from typing import Callable

import pytest


@pytest.fixture(scope="function")
def send_to_test() -> Callable:
    return lambda x: (x ** 2, x ** 3)

# Helper function to run and monitor a test
def run_test(test_func, variable, expected_value, actual_value, data):
    try:
        with open("./data.json", "r") as f:
            data = json.load(f)
    except FileNotFoundError:
        data = []

    data.append(
        {
            "func": test_func,
            "variable": variable,
            "expected_value": expected_value,
            "actual_value": actual_value,
        }
    )
    with open("./data.json", "w", encoding="utf-8") as f:
        json.dump(data, f, ensure_ascii=False, indent=4)

    assert actual_value == expected_value


# Reformatted tests
def test_me_1(send_to_test):
    # setup
    data = []
    x, y = send_to_test(3)

    # tests
    run_test(
        test_func=inspect.stack()[0][3],
        variable="x",
        expected_value=3 ** 2,
        actual_value=x,
        data=data,
    )
    run_test(
        test_func=inspect.stack()[0][3],
        variable="y",
        expected_value=3 ** 3,
        actual_value=y,
        data=data,
    )
    run_test(
        test_func=inspect.stack()[0][3],
        variable="y",
        expected_value=3 ** 2,
        actual_value=y,
        data=data,
    )

And so, when you run pytest, a new data.jsonfile is created with the expected content:

[
    {
        "func": "test_me_1",
        "variable": "x",
        "expected_value": 9,
        "actual_value": 9
    },
    {
        "func": "test_me_1",
        "variable": "y",
        "expected_value": 27,
        "actual_value": 27
    },
    {
        "func": "test_me_1",
        "variable": "y",
        "expected_value": 9,
        "actual_value": 27
    }
]

CodePudding user response：

@hoefling answered my question with quite an easy solution of using the following hooks in conftest:

def pytest_assertrepr_compare(op, left, right):...
def pytest_assertion_pass(item, lineno, orig, expl):...

According to the Pytest documentation, I have to add to pytest.ini enable_assertion_pass_hook=true and erase .pyc files, but other than that it works like a charm. Now I have left and right comparisons with the operator used (pytest_assertrepr_compare) and if the test passed (pytest_assertion_pass).