validate configuration files with schema

- ensures that configuration files of passing tests conform to schema
- validity of definition files is now checked in CI

docker/Dockerfile

@@ -43,7 +43,8 @@ RUN pip3 install \
   pylint==2.12.2 \
   sympy==1.7.1 \
   gcovr==5.0 \
-  h5py==3.6.0
+  h5py==3.6.0 \
+  jsonschema==4.6.0
 
 RUN pip3 install vtk --extra-index-url https://gitlab.kitware.com/api/v4/projects/13/packages/pypi/simple
 

tests/run.py

@@ -30,6 +30,9 @@ import tempfile
 import typing
 import unittest
 import re
+import sys
+
+import jsonschema
 
 
 ROOT_DIRECTORY = pathlib.Path(__file__).resolve().parent.parent
@@ -168,7 +171,7 @@ def run_executable_with_mpirun(
     mpi_processes: int,
     args: typing.List[str],
     working_directory: pathlib.Path,
-) -> subprocess.CompletedProcess:
+) -> subprocess.CompletedProcess[str]:
     """
     Runs the executable at `executable` with the provided `args`
     from `working_directory` with `mpi_processes` processes.
@@ -193,7 +196,7 @@ def run_executable_with_mpirun(
 def run_process(
     args: typing.List[str],
     working_directory: pathlib.Path = pathlib.Path.cwd(),
-) -> subprocess.CompletedProcess:
+) -> subprocess.CompletedProcess[str]:
     """
     Runs a process with the provided `args` from `working_directory`.
 
@@ -220,7 +223,7 @@ def diff_text_files(
     value: pathlib.Path,
     reference: pathlib.Path,
     comparison: ComparisonType,
-):
+) -> None:
     """
     Compares the files at `value`, `reference` as specified by `comparison.
     Results are reported to `case`.
@@ -247,7 +250,7 @@ def diff_text_files(
 def diff_vtu_files(
     value: pathlib.Path,
     reference: pathlib.Path,
-):
+) -> None:
     """
     Compares the files at `value`, `reference`.
     """
@@ -261,7 +264,7 @@ def diff_vtu_files(
 
 def diff_text_string(
     value: str, reference: str, case: unittest.TestCase = unittest.TestCase()
-):
+) -> None:
     """
     Checks that the lines in the strings `value` and `reference` are equal.
 
@@ -332,7 +335,7 @@ def extract_numbers(text: typing.Iterable[str]) -> typing.Iterable[float]:
 
 def diff_numeric_string(
     value: str, reference: str, case: unittest.TestCase = unittest.TestCase()
-):
+) -> None:
     """
     Checks that the lines in the strings `value` and `reference` are almost equal
     when interpreted as floats. Non-float lines are interpreted as NaNs.
@@ -377,21 +380,46 @@ def load_tests(
 
     for path in paths:
         group_name, test_name = path.parent.parts[-2:]
-        to_method_name = (
-            lambda processes, name=test_name: f"test_{name}_with_{processes}_mpi_processes"
-        )
+
+        def to_method_name(name: str, processes: int) -> str:
+            """
+            Generates a test name given the number of MPI processes.
+            """
+            return f"test_{name}_with_{processes}_mpi_processes"
 
         with open(path, "r", encoding="utf-8") as file:
-            test_case_definitions = as_test_case_definitions(
-                path.parent, json.load(file)
-            )
+            instance = json.load(file)
+            try:
+                with open(
+                    ROOT_DIRECTORY / "tests" / "schema.json", "r", encoding="utf-8"
+                ) as schema:
+                    jsonschema.validate(instance=instance, schema=json.load(schema))
+                test_case_definitions = as_test_case_definitions(path.parent, instance)
+            except jsonschema.ValidationError as error:
+                print(
+                    f"Warning: Test definition '{path}' is invalid. {error.message}.",
+                    file=sys.stderr,
+                )
+                test_case_definitions = (
+                    definition
+                    for definition in [
+                        TestCaseDefinition(
+                            executable=pathlib.Path(),
+                            references=pathlib.Path(),
+                            args=[],
+                            mpi_processes=1,
+                            compare_stdout=ComparisonType.NONE,
+                            ae108_output=[],
+                        )
+                    ]
+                )
 
             testcase = type(
                 group_name,
                 (unittest.TestCase,),
                 {
                     to_method_name(
-                        definition.mpi_processes
+                        test_name, definition.mpi_processes
                     ): lambda case, definition=definition: run_testcase(
                         definition, case
                     )
@@ -405,7 +433,7 @@ def load_tests(
 
 def run_testcase(
     definition: TestCaseDefinition, case: unittest.TestCase = unittest.TestCase()
-):
+) -> None:
     """
     Runs the test defined by `definition` and reports the issues to `case`.
     Nonexisting references are automatically created.