From a5e18ac98475965f1ab06dae1622e365de8d8ee3 Mon Sep 17 00:00:00 2001
From: Manuel Weberndorfer <manuel.weberndorfer@id.ethz.ch>
Date: Thu, 23 Jun 2022 13:11:49 +0000
Subject: [PATCH] validate configuration files with schema
- ensures that configuration files of passing tests conform to schema
- validity of definition files is now checked in CI
---
docker/Dockerfile | 3 ++-
tests/run.py | 56 +++++++++++++++++++++++++++++++++++------------
2 files changed, 44 insertions(+), 15 deletions(-)
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 36108805..5f727197 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -43,7 +43,8 @@ RUN pip3 install \
pylint==2.12.2 \
sympy==1.7.1 \
gcovr==5.0 \
- h5py==3.6.0
+ h5py==3.6.0 \
+ jsonschema==4.6.0
RUN pip3 install vtk --extra-index-url https://gitlab.kitware.com/api/v4/projects/13/packages/pypi/simple
diff --git a/tests/run.py b/tests/run.py
index 2d0b6401..3944fbd2 100755
--- a/tests/run.py
+++ b/tests/run.py
@@ -30,6 +30,9 @@ import tempfile
import typing
import unittest
import re
+import sys
+
+import jsonschema
ROOT_DIRECTORY = pathlib.Path(__file__).resolve().parent.parent
@@ -168,7 +171,7 @@ def run_executable_with_mpirun(
mpi_processes: int,
args: typing.List[str],
working_directory: pathlib.Path,
-) -> subprocess.CompletedProcess:
+) -> subprocess.CompletedProcess[str]:
"""
Runs the executable at `executable` with the provided `args`
from `working_directory` with `mpi_processes` processes.
@@ -193,7 +196,7 @@ def run_executable_with_mpirun(
def run_process(
args: typing.List[str],
working_directory: pathlib.Path = pathlib.Path.cwd(),
-) -> subprocess.CompletedProcess:
+) -> subprocess.CompletedProcess[str]:
"""
Runs a process with the provided `args` from `working_directory`.
@@ -220,7 +223,7 @@ def diff_text_files(
value: pathlib.Path,
reference: pathlib.Path,
comparison: ComparisonType,
-):
+) -> None:
"""
Compares the files at `value`, `reference` as specified by `comparison.
Results are reported to `case`.
@@ -247,7 +250,7 @@ def diff_text_files(
def diff_vtu_files(
value: pathlib.Path,
reference: pathlib.Path,
-):
+) -> None:
"""
Compares the files at `value`, `reference`.
"""
@@ -261,7 +264,7 @@ def diff_vtu_files(
def diff_text_string(
value: str, reference: str, case: unittest.TestCase = unittest.TestCase()
-):
+) -> None:
"""
Checks that the lines in the strings `value` and `reference` are equal.
@@ -332,7 +335,7 @@ def extract_numbers(text: typing.Iterable[str]) -> typing.Iterable[float]:
def diff_numeric_string(
value: str, reference: str, case: unittest.TestCase = unittest.TestCase()
-):
+) -> None:
"""
Checks that the lines in the strings `value` and `reference` are almost equal
when interpreted as floats. Non-float lines are interpreted as NaNs.
@@ -377,21 +380,46 @@ def load_tests(
for path in paths:
group_name, test_name = path.parent.parts[-2:]
- to_method_name = (
- lambda processes, name=test_name: f"test_{name}_with_{processes}_mpi_processes"
- )
+
+ def to_method_name(name: str, processes: int) -> str:
+ """
+ Generates a test name given the number of MPI processes.
+ """
+ return f"test_{name}_with_{processes}_mpi_processes"
with open(path, "r", encoding="utf-8") as file:
- test_case_definitions = as_test_case_definitions(
- path.parent, json.load(file)
- )
+ instance = json.load(file)
+ try:
+ with open(
+ ROOT_DIRECTORY / "tests" / "schema.json", "r", encoding="utf-8"
+ ) as schema:
+ jsonschema.validate(instance=instance, schema=json.load(schema))
+ test_case_definitions = as_test_case_definitions(path.parent, instance)
+ except jsonschema.ValidationError as error:
+ print(
+ f"Warning: Test definition '{path}' is invalid. {error.message}.",
+ file=sys.stderr,
+ )
+ test_case_definitions = (
+ definition
+ for definition in [
+ TestCaseDefinition(
+ executable=pathlib.Path(),
+ references=pathlib.Path(),
+ args=[],
+ mpi_processes=1,
+ compare_stdout=ComparisonType.NONE,
+ ae108_output=[],
+ )
+ ]
+ )
testcase = type(
group_name,
(unittest.TestCase,),
{
to_method_name(
- definition.mpi_processes
+ test_name, definition.mpi_processes
): lambda case, definition=definition: run_testcase(
definition, case
)
@@ -405,7 +433,7 @@ def load_tests(
def run_testcase(
definition: TestCaseDefinition, case: unittest.TestCase = unittest.TestCase()
-):
+) -> None:
"""
Runs the test defined by `definition` and reports the issues to `case`.
Nonexisting references are automatically created.
--
GitLab