Commit a5e18ac9 authored by webmanue's avatar webmanue
Browse files

validate configuration files with schema

- ensures that configuration files of passing tests conform to schema
- validity of definition files is now checked in CI
parent e3711f59
Pipeline #138134 passed with stages
in 16 minutes and 38 seconds
......@@ -43,7 +43,8 @@ RUN pip3 install \
pylint==2.12.2 \
sympy==1.7.1 \
gcovr==5.0 \
h5py==3.6.0
h5py==3.6.0 \
jsonschema==4.6.0
RUN pip3 install vtk --extra-index-url https://gitlab.kitware.com/api/v4/projects/13/packages/pypi/simple
......
......@@ -30,6 +30,9 @@ import tempfile
import typing
import unittest
import re
import sys
import jsonschema
ROOT_DIRECTORY = pathlib.Path(__file__).resolve().parent.parent
......@@ -168,7 +171,7 @@ def run_executable_with_mpirun(
mpi_processes: int,
args: typing.List[str],
working_directory: pathlib.Path,
) -> subprocess.CompletedProcess:
) -> subprocess.CompletedProcess[str]:
"""
Runs the executable at `executable` with the provided `args`
from `working_directory` with `mpi_processes` processes.
......@@ -193,7 +196,7 @@ def run_executable_with_mpirun(
def run_process(
args: typing.List[str],
working_directory: pathlib.Path = pathlib.Path.cwd(),
) -> subprocess.CompletedProcess:
) -> subprocess.CompletedProcess[str]:
"""
Runs a process with the provided `args` from `working_directory`.
......@@ -220,7 +223,7 @@ def diff_text_files(
value: pathlib.Path,
reference: pathlib.Path,
comparison: ComparisonType,
):
) -> None:
"""
Compares the files at `value`, `reference` as specified by `comparison.
Results are reported to `case`.
......@@ -247,7 +250,7 @@ def diff_text_files(
def diff_vtu_files(
value: pathlib.Path,
reference: pathlib.Path,
):
) -> None:
"""
Compares the files at `value`, `reference`.
"""
......@@ -261,7 +264,7 @@ def diff_vtu_files(
def diff_text_string(
value: str, reference: str, case: unittest.TestCase = unittest.TestCase()
):
) -> None:
"""
Checks that the lines in the strings `value` and `reference` are equal.
......@@ -332,7 +335,7 @@ def extract_numbers(text: typing.Iterable[str]) -> typing.Iterable[float]:
def diff_numeric_string(
value: str, reference: str, case: unittest.TestCase = unittest.TestCase()
):
) -> None:
"""
Checks that the lines in the strings `value` and `reference` are almost equal
when interpreted as floats. Non-float lines are interpreted as NaNs.
......@@ -377,21 +380,46 @@ def load_tests(
for path in paths:
group_name, test_name = path.parent.parts[-2:]
to_method_name = (
lambda processes, name=test_name: f"test_{name}_with_{processes}_mpi_processes"
)
def to_method_name(name: str, processes: int) -> str:
"""
Generates a test name given the number of MPI processes.
"""
return f"test_{name}_with_{processes}_mpi_processes"
with open(path, "r", encoding="utf-8") as file:
test_case_definitions = as_test_case_definitions(
path.parent, json.load(file)
)
instance = json.load(file)
try:
with open(
ROOT_DIRECTORY / "tests" / "schema.json", "r", encoding="utf-8"
) as schema:
jsonschema.validate(instance=instance, schema=json.load(schema))
test_case_definitions = as_test_case_definitions(path.parent, instance)
except jsonschema.ValidationError as error:
print(
f"Warning: Test definition '{path}' is invalid. {error.message}.",
file=sys.stderr,
)
test_case_definitions = (
definition
for definition in [
TestCaseDefinition(
executable=pathlib.Path(),
references=pathlib.Path(),
args=[],
mpi_processes=1,
compare_stdout=ComparisonType.NONE,
ae108_output=[],
)
]
)
testcase = type(
group_name,
(unittest.TestCase,),
{
to_method_name(
definition.mpi_processes
test_name, definition.mpi_processes
): lambda case, definition=definition: run_testcase(
definition, case
)
......@@ -405,7 +433,7 @@ def load_tests(
def run_testcase(
definition: TestCaseDefinition, case: unittest.TestCase = unittest.TestCase()
):
) -> None:
"""
Runs the test defined by `definition` and reports the issues to `case`.
Nonexisting references are automatically created.
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment