Commit 3203fc82 authored by webmanue's avatar webmanue
Browse files

Merge branch '116-add-json-schema-for-test-configuration-files' into 'master'

Resolve "Add JSON schema for test configuration files"

Closes #116

See merge request mechanics-and-materials/ae108!105
parents 37a189ab a5e18ac9
Pipeline #138286 canceled with stages
in 1 minute and 37 seconds
......@@ -43,7 +43,8 @@ RUN pip3 install \
pylint==2.12.2 \
sympy==1.7.1 \
gcovr==5.0 \
h5py==3.6.0
h5py==3.6.0 \
jsonschema==4.6.0
RUN pip3 install vtk --extra-index-url https://gitlab.kitware.com/api/v4/projects/13/packages/pypi/simple
......
{
"executable": [
"examples",
"ae108-examples-Basic"
],
"compare_stdout": "numeric",
"mpi_processes": [
1,
3
]
"$schema": "../../schema.json",
"executable": [
"examples",
"ae108-examples-Basic"
],
"compare_stdout": "numeric",
"mpi_processes": [
1,
3
]
}
\ No newline at end of file
{
"$schema": "../../schema.json",
"executable": [
"examples",
"ae108-examples-CantileverBeam"
......
{
"executable": [
"examples",
"ae108-examples-Cmdline"
],
"args": [
"--enable_greeting",
"true"
],
"compare_stdout": "text"
"$schema": "../../schema.json",
"executable": [
"examples",
"ae108-examples-Cmdline"
],
"args": [
"--enable_greeting",
"true"
],
"compare_stdout": "text"
}
\ No newline at end of file
{
"executable": [
"examples",
"ae108-examples-CuboidMesh"
],
"args": [
"--stdout-output",
"true"
],
"compare_stdout": "numeric",
"mpi_processes": [
1,
3
]
"$schema": "../../schema.json",
"executable": [
"examples",
"ae108-examples-CuboidMesh"
],
"args": [
"--stdout-output",
"true"
],
"compare_stdout": "numeric",
"mpi_processes": [
1,
3
]
}
\ No newline at end of file
{
"executable": [
"examples",
"ae108-examples-DynamicSolver"
],
"compare_stdout": "numeric",
"mpi_processes": [
1,
3
]
"$schema": "../../schema.json",
"executable": [
"examples",
"ae108-examples-DynamicSolver"
],
"compare_stdout": "numeric",
"mpi_processes": [
1,
3
]
}
\ No newline at end of file
{
"$schema": "../../schema.json",
"executable": [
"examples",
"ae108-examples-ForceElement"
......
{
"$schema": "../../schema.json",
"executable": [
"examples",
"ae108-examples-GeneralizedNonlinearSolver"
......
{
"executable": [
"examples",
"ae108-examples-Input"
],
"compare_stdout": "numeric",
"mpi_processes": [
1
]
"$schema": "../../schema.json",
"executable": [
"examples",
"ae108-examples-Input"
],
"compare_stdout": "numeric",
"mpi_processes": [
1
]
}
\ No newline at end of file
{
"executable": [
"examples",
"ae108-examples-MeshGeneration"
],
"compare_stdout": "numeric",
"mpi_processes": [
1
]
"$schema": "../../schema.json",
"executable": [
"examples",
"ae108-examples-MeshGeneration"
],
"compare_stdout": "numeric",
"mpi_processes": [
1
]
}
\ No newline at end of file
{
"$schema": "../../schema.json",
"executable": [
"examples",
"ae108-examples-Output"
......
{
"executable": [
"examples",
"ae108-examples-PeriodicBC"
],
"compare_stdout": "numeric",
"mpi_processes": [
1,
3
]
"$schema": "../../schema.json",
"executable": [
"examples",
"ae108-examples-PeriodicBC"
],
"compare_stdout": "numeric",
"mpi_processes": [
1,
3
]
}
\ No newline at end of file
{
"$schema": "../../schema.json",
"executable": [
"examples",
"ae108-examples-QuadraticTetrahedra"
......
{
"$schema": "../../schema.json",
"executable": [
"examples",
"ae108-examples-QuadraticTriangles"
......
{
"executable": [
"examples",
"ae108-examples-SurfaceForce"
],
"compare_stdout": "numeric",
"mpi_processes": [
1,
3
]
"$schema": "../../schema.json",
"executable": [
"examples",
"ae108-examples-SurfaceForce"
],
"compare_stdout": "numeric",
"mpi_processes": [
1,
3
]
}
\ No newline at end of file
{
"executable": [
"examples",
"ae108-examples-TimoshenkoBeam"
],
"compare_stdout": "numeric",
"mpi_processes": [
1,
3
]
"$schema": "../../schema.json",
"executable": [
"examples",
"ae108-examples-TimoshenkoBeam"
],
"compare_stdout": "numeric",
"mpi_processes": [
1,
3
]
}
\ No newline at end of file
......@@ -30,6 +30,9 @@ import tempfile
import typing
import unittest
import re
import sys
import jsonschema
ROOT_DIRECTORY = pathlib.Path(__file__).resolve().parent.parent
......@@ -168,7 +171,7 @@ def run_executable_with_mpirun(
mpi_processes: int,
args: typing.List[str],
working_directory: pathlib.Path,
) -> subprocess.CompletedProcess:
) -> subprocess.CompletedProcess[str]:
"""
Runs the executable at `executable` with the provided `args`
from `working_directory` with `mpi_processes` processes.
......@@ -193,7 +196,7 @@ def run_executable_with_mpirun(
def run_process(
args: typing.List[str],
working_directory: pathlib.Path = pathlib.Path.cwd(),
) -> subprocess.CompletedProcess:
) -> subprocess.CompletedProcess[str]:
"""
Runs a process with the provided `args` from `working_directory`.
......@@ -220,7 +223,7 @@ def diff_text_files(
value: pathlib.Path,
reference: pathlib.Path,
comparison: ComparisonType,
):
) -> None:
"""
Compares the files at `value`, `reference` as specified by `comparison.
Results are reported to `case`.
......@@ -247,7 +250,7 @@ def diff_text_files(
def diff_vtu_files(
value: pathlib.Path,
reference: pathlib.Path,
):
) -> None:
"""
Compares the files at `value`, `reference`.
"""
......@@ -261,7 +264,7 @@ def diff_vtu_files(
def diff_text_string(
value: str, reference: str, case: unittest.TestCase = unittest.TestCase()
):
) -> None:
"""
Checks that the lines in the strings `value` and `reference` are equal.
......@@ -332,7 +335,7 @@ def extract_numbers(text: typing.Iterable[str]) -> typing.Iterable[float]:
def diff_numeric_string(
value: str, reference: str, case: unittest.TestCase = unittest.TestCase()
):
) -> None:
"""
Checks that the lines in the strings `value` and `reference` are almost equal
when interpreted as floats. Non-float lines are interpreted as NaNs.
......@@ -377,21 +380,46 @@ def load_tests(
for path in paths:
group_name, test_name = path.parent.parts[-2:]
to_method_name = (
lambda processes, name=test_name: f"test_{name}_with_{processes}_mpi_processes"
)
def to_method_name(name: str, processes: int) -> str:
"""
Generates a test name given the number of MPI processes.
"""
return f"test_{name}_with_{processes}_mpi_processes"
with open(path, "r", encoding="utf-8") as file:
test_case_definitions = as_test_case_definitions(
path.parent, json.load(file)
)
instance = json.load(file)
try:
with open(
ROOT_DIRECTORY / "tests" / "schema.json", "r", encoding="utf-8"
) as schema:
jsonschema.validate(instance=instance, schema=json.load(schema))
test_case_definitions = as_test_case_definitions(path.parent, instance)
except jsonschema.ValidationError as error:
print(
f"Warning: Test definition '{path}' is invalid. {error.message}.",
file=sys.stderr,
)
test_case_definitions = (
definition
for definition in [
TestCaseDefinition(
executable=pathlib.Path(),
references=pathlib.Path(),
args=[],
mpi_processes=1,
compare_stdout=ComparisonType.NONE,
ae108_output=[],
)
]
)
testcase = type(
group_name,
(unittest.TestCase,),
{
to_method_name(
definition.mpi_processes
test_name, definition.mpi_processes
): lambda case, definition=definition: run_testcase(
definition, case
)
......@@ -405,7 +433,7 @@ def load_tests(
def run_testcase(
definition: TestCaseDefinition, case: unittest.TestCase = unittest.TestCase()
):
) -> None:
"""
Runs the test defined by `definition` and reports the issues to `case`.
Nonexisting references are automatically created.
......
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": {
"$schema": {
"type": "string"
},
"executable": {
"type": "array",
"items": {
"type": "string"
},
"minItems": 1
},
"compare_stdout": {
"type": "string",
"enum": [
"numeric",
"text",
"none"
],
"default": "none"
},
"mpi_processes": {
"type": "array",
"default": "[1]",
"items": {
"type": "integer"
}
},
"ae108_output": {
"type": "array",
"items": {
"type": "object",
"required": [
"filename"
],
"properties": {
"filename": {
"type": "string"
},
"xdmf_generator_flags": {
"type": "array",
"items": {
"type": "string"
},
"default": []
}
},
"additionalProperties": false
}
},
"args": {
"type": "array",
"items": {
"type": "string"
},
"default": []
}
},
"required": [
"executable"
],
"additionalProperties": false
}
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment