diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ebef366c647c9f5ddcbaa48fe7b4fdcd153aa72e..10cb32776a4fcd45e9ffc4cd2aebdfc30db132aa 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -73,6 +73,8 @@ check-license-header: ! grep \ --recursive \ --exclude-dir='.git' \ + --exclude='*.json' \ + --exclude='*.txt' \ --files-without-match "Apache License" \ . @@ -82,7 +84,7 @@ check-script: script: - find . -name "*.py" -print0 | xargs -0 python3 -m black --check - find . -name "*.py" -print0 | xargs -0 python3 -m pylint - - find . -name "*.py" -print0 | xargs -0 python3 -m mypy --python-version 3.5 --ignore-missing + - find . -name "*.py" -print0 | xargs -0 python3 -m mypy --python-version 3.7 --ignore-missing - find . -name "*.py" -print0 | xargs -0 python3 -m doctest build-library: diff --git a/CMakeLists.txt b/CMakeLists.txt index 3328aa26f30b02a53386b076a96108ec398712de..453e95c45778194ddaafe5bb34263367ae48727e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,6 +29,8 @@ foreach(AE108_LIBRARY elements cpppetsc assembly solve cmdline) endforeach() add_subdirectory(examples) +add_test(NAME ${PROJECT_NAME}-ExamplesTests COMMAND "${CMAKE_CURRENT_LIST_DIR}/tests/run.py") + include(CMakePackageConfigHelpers) include(GNUInstallDirs) write_basic_package_version_file(${PROJECT_NAME}-config-version.cmake diff --git a/examples/SurfaceForce.cc b/examples/SurfaceForce.cc index e60f5323aef894534667baaa0dad6a925cd50489..2cc3f5e4e24eab95662a7b9b885fea2ef142ffba 100644 --- a/examples/SurfaceForce.cc +++ b/examples/SurfaceForce.cc @@ -138,12 +138,12 @@ void print_force_at_x(const typename Geometry::Point::value_type x, return force; }(); - // We are done with the computation and print the results to stderr. + // We are done with the computation and print the results to stdout. if (Policy::isPrimaryRank()) { static_assert(dof_per_vertex == 3, "We assume 3 degrees of freedom."); - fprintf(stderr, "The force at x=%+f is [%+f, %+f, %+f].\n", x, force[0], - force[1], force[2]); + printf("The force at x=%+f is [%+f, %+f, %+f].\n", x, force[0], force[1], + force[2]); } } diff --git a/tests/examples/basic/definition.json b/tests/examples/basic/definition.json new file mode 100644 index 0000000000000000000000000000000000000000..faf67ae5039090c3905a93d2cdbc4856e5418bf8 --- /dev/null +++ b/tests/examples/basic/definition.json @@ -0,0 +1,11 @@ +{ + "executable": [ + "examples", + "ae108-examples-Basic" + ], + "compare_stdout": "numeric", + "mpi_processes": [ + 1, + 3 + ] +} \ No newline at end of file diff --git a/tests/examples/basic/references/stdout.txt b/tests/examples/basic/references/stdout.txt new file mode 100644 index 0000000000000000000000000000000000000000..12fd4b23bf6357481c6e6a4794e370a3c0abaeb9 --- /dev/null +++ b/tests/examples/basic/references/stdout.txt @@ -0,0 +1,14 @@ +Vec Object: 1 MPI processes + type: seq +0. +0. +0.25 +0. +0.25 +0. +0. +0. +0.5 +0. +0.5 +0. diff --git a/tests/examples/cmdline/definition.json b/tests/examples/cmdline/definition.json new file mode 100644 index 0000000000000000000000000000000000000000..a3139a435fadab53b2739ea22818948720bc9443 --- /dev/null +++ b/tests/examples/cmdline/definition.json @@ -0,0 +1,11 @@ +{ + "executable": [ + "examples", + "ae108-examples-Cmdline" + ], + "args": [ + "--enable_greeting", + "true" + ], + "compare_stdout": "text" +} \ No newline at end of file diff --git a/tests/examples/cmdline/references/stdout.txt b/tests/examples/cmdline/references/stdout.txt new file mode 100644 index 0000000000000000000000000000000000000000..cd0875583aabe89ee197ea133980a9085d08e497 --- /dev/null +++ b/tests/examples/cmdline/references/stdout.txt @@ -0,0 +1 @@ +Hello world! diff --git a/tests/examples/cuboid_mesh_stdout/definition.json b/tests/examples/cuboid_mesh_stdout/definition.json new file mode 100644 index 0000000000000000000000000000000000000000..62e3e2400a3f6c7040b22f753dbcbd80fffc76fc --- /dev/null +++ b/tests/examples/cuboid_mesh_stdout/definition.json @@ -0,0 +1,15 @@ +{ + "executable": [ + "examples", + "ae108-examples-CuboidMesh" + ], + "args": [ + "--stdout-output", + "true" + ], + "compare_stdout": "numeric", + "mpi_processes": [ + 1, + 3 + ] +} \ No newline at end of file diff --git a/tests/examples/cuboid_mesh_stdout/references/stdout.txt b/tests/examples/cuboid_mesh_stdout/references/stdout.txt new file mode 100644 index 0000000000000000000000000000000000000000..f4b4af6af83bef419f52aaf36ef49ba6b0840fe4 --- /dev/null +++ b/tests/examples/cuboid_mesh_stdout/references/stdout.txt @@ -0,0 +1,83 @@ +Vec Object: 1 MPI processes + type: seq +0. +0. +0. +0.25 +0.0267857 +0.0267857 +0.5 +0. +0. +0. +0. +0. +0.25 +-3.31171e-13 +0.0267857 +0.5 +0. +0. +0. +0. +0. +0.25 +-0.0267857 +0.0267857 +0.5 +0. +0. +0. +0. +0. +0.25 +0.0267857 +1.48821e-13 +0.5 +0. +0. +0. +0. +0. +0.25 +-4.7345e-13 +1.53613e-13 +0.5 +0. +0. +0. +0. +0. +0.25 +-0.0267857 +-4.0519e-13 +0.5 +0. +0. +0. +0. +0. +0.25 +0.0267857 +-0.0267857 +0.5 +0. +0. +0. +0. +0. +0.25 +-4.63614e-13 +-0.0267857 +0.5 +0. +0. +0. +0. +0. +0.25 +-0.0267857 +-0.0267857 +0.5 +0. +0. diff --git a/tests/examples/input/definition.json b/tests/examples/input/definition.json new file mode 100644 index 0000000000000000000000000000000000000000..4a6103602cd78245244e93b55ba03c3605445dd1 --- /dev/null +++ b/tests/examples/input/definition.json @@ -0,0 +1,10 @@ +{ + "executable": [ + "examples", + "ae108-examples-Input" + ], + "compare_stdout": "numeric", + "mpi_processes": [ + 1 + ] +} \ No newline at end of file diff --git a/tests/examples/input/references/stdout.txt b/tests/examples/input/references/stdout.txt new file mode 100644 index 0000000000000000000000000000000000000000..bc9a9f75894131d2acf9c1939cf3fa6a279cc80b --- /dev/null +++ b/tests/examples/input/references/stdout.txt @@ -0,0 +1,16 @@ +Vec Object: vertex_index_data 1 MPI processes + type: seq +0. +1. +2. +3. +4. +5. +Vec Object: vertex_index_data 1 MPI processes + type: seq +0. +1. +2. +3. +4. +5. diff --git a/tests/examples/mesh_generation/definition.json b/tests/examples/mesh_generation/definition.json new file mode 100644 index 0000000000000000000000000000000000000000..7516c4a48a06a0a2905d087b086fc4a264345557 --- /dev/null +++ b/tests/examples/mesh_generation/definition.json @@ -0,0 +1,10 @@ +{ + "executable": [ + "examples", + "ae108-examples-MeshGeneration" + ], + "compare_stdout": "numeric", + "mpi_processes": [ + 1 + ] +} \ No newline at end of file diff --git a/tests/examples/mesh_generation/references/stdout.txt b/tests/examples/mesh_generation/references/stdout.txt new file mode 100644 index 0000000000000000000000000000000000000000..4c162d29152704e0a1fc531fe8cb500c101f91f6 --- /dev/null +++ b/tests/examples/mesh_generation/references/stdout.txt @@ -0,0 +1,10 @@ +DM Object: 1 MPI processes + type: plex +DM_0x55d0c9a9c6d0_0 in 2 dimensions: + 0-cells: 10 + 2-cells: 8 +Labels: + depth: 2 strata with value/size (0 (10), 1 (8)) +Field Field_0: + adjacency FEM +The second vertex is at (0.000000, 2.000000). diff --git a/tests/examples/periodic_bc/definition.json b/tests/examples/periodic_bc/definition.json new file mode 100644 index 0000000000000000000000000000000000000000..0517bf364902bfafc6185ef7b8ada68021f9027c --- /dev/null +++ b/tests/examples/periodic_bc/definition.json @@ -0,0 +1,11 @@ +{ + "executable": [ + "examples", + "ae108-examples-PeriodicBC" + ], + "compare_stdout": "numeric", + "mpi_processes": [ + 1, + 3 + ] +} \ No newline at end of file diff --git a/tests/examples/periodic_bc/references/stdout.txt b/tests/examples/periodic_bc/references/stdout.txt new file mode 100644 index 0000000000000000000000000000000000000000..4934c0673f3f483bcfa2ee3352d6e8a66d2c7592 --- /dev/null +++ b/tests/examples/periodic_bc/references/stdout.txt @@ -0,0 +1,10 @@ +Vec Object: 1 MPI processes + type: seq +0. +0. +0.5 +0. +0.5 +-0.125 +0. +-0.125 diff --git a/tests/examples/surface_force/definition.json b/tests/examples/surface_force/definition.json new file mode 100644 index 0000000000000000000000000000000000000000..38353128e5cc47da95685031a2366476b48e6b12 --- /dev/null +++ b/tests/examples/surface_force/definition.json @@ -0,0 +1,11 @@ +{ + "executable": [ + "examples", + "ae108-examples-SurfaceForce" + ], + "compare_stdout": "numeric", + "mpi_processes": [ + 1, + 3 + ] +} \ No newline at end of file diff --git a/tests/examples/surface_force/references/stdout.txt b/tests/examples/surface_force/references/stdout.txt new file mode 100644 index 0000000000000000000000000000000000000000..a3a5bc3271b3e9d1b6f9bc01e4babd2d07e14338 --- /dev/null +++ b/tests/examples/surface_force/references/stdout.txt @@ -0,0 +1,3 @@ +The force at x=+0.000000 is [-0.500000, -0.000000, -0.000000]. +The force at x=+0.500000 is [-0.000000, +0.000000, +0.000000]. +The force at x=+1.000000 is [+0.500000, -0.000000, -0.000000]. diff --git a/tests/examples/timoshenko_beam/definition.json b/tests/examples/timoshenko_beam/definition.json new file mode 100644 index 0000000000000000000000000000000000000000..ef97ee160f65da0ee1f415ace4baecef6563a3d5 --- /dev/null +++ b/tests/examples/timoshenko_beam/definition.json @@ -0,0 +1,11 @@ +{ + "executable": [ + "examples", + "ae108-examples-TimoshenkoBeam" + ], + "compare_stdout": "numeric", + "mpi_processes": [ + 1, + 3 + ] +} \ No newline at end of file diff --git a/tests/examples/timoshenko_beam/references/stdout.txt b/tests/examples/timoshenko_beam/references/stdout.txt new file mode 100644 index 0000000000000000000000000000000000000000..e6e0817092d871f02ebbfc81696e5efc2acfa13b --- /dev/null +++ b/tests/examples/timoshenko_beam/references/stdout.txt @@ -0,0 +1,17 @@ +Vec Object: 1 MPI processes + type: seq +0. +0. +-0.0394841 +-0.0159154 +-0.0382547 +-0.0357375 +-0.0237921 +-0.1 +-0.0632511 +0.00797989 +-0.0460029 +-0.0333497 +0. +0. +-0.0399347 diff --git a/cpppetsc/tools/numeric_diff.py b/tests/numeric_diff.py similarity index 100% rename from cpppetsc/tools/numeric_diff.py rename to tests/numeric_diff.py diff --git a/tests/run.py b/tests/run.py new file mode 100755 index 0000000000000000000000000000000000000000..693eda6f2ff8271a3afbb414513340ab78a8aa51 --- /dev/null +++ b/tests/run.py @@ -0,0 +1,387 @@ +#!/usr/bin/env python3 + +# © 2021 ETH Zurich, Mechanics and Materials Lab +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Runs the tests defined in definition.json files. +""" + +import dataclasses +import enum +import itertools +import json +import math +import pathlib +import shutil +import subprocess +import tempfile +import typing +import unittest +import re + + +ROOT_DIRECTORY = pathlib.Path(__file__).parent.parent + + +class ComparisonType(enum.Enum): + """ + Types of file comparisons. + """ + + NONE = 0 + TEXT = 1 + NUMERIC = 2 + + +@dataclasses.dataclass(frozen=True) +class TestCaseDefinition: + """ + Contains the parameters necessary to execute a test. + """ + + executable: pathlib.Path + args: typing.List[str] + references: pathlib.Path + compare_stdout: ComparisonType + mpi_processes: int + + +def as_test_case_definitions( + path: pathlib.Path, definition: typing.Dict[str, typing.Any] +) -> typing.Generator[TestCaseDefinition, None, None]: + r""" + Generates test case definitions from `definition` for a test at `path`. + + >>> empty_path = pathlib.Path() + >>> cwd_path = pathlib.Path.cwd() + + >>> next( + ... as_test_case_definitions( + ... empty_path, + ... {"executable": ["a", "b"]} + ... ) + ... ).executable.relative_to(cwd_path) + PosixPath('a/b') + + >>> next( + ... as_test_case_definitions( + ... empty_path, + ... {"executable": []} + ... ) + ... ).args + [] + >>> next( + ... as_test_case_definitions( + ... empty_path, + ... {"executable": [], "args": ["b"]} + ... ) + ... ).args + ['b'] + + >>> next( + ... as_test_case_definitions( + ... pathlib.Path("a"), + ... {"executable": []} + ... ) + ... ).references + PosixPath('a/references') + + >>> next( + ... as_test_case_definitions( + ... empty_path, + ... {"executable": []} + ... ) + ... ).compare_stdout + <ComparisonType.NONE: 0> + >>> next( + ... as_test_case_definitions( + ... empty_path, + ... {"executable": [], "compare_stdout": "text"} + ... ) + ... ).compare_stdout + <ComparisonType.TEXT: 1> + + >>> next( + ... as_test_case_definitions( + ... empty_path, + ... {"executable": []} + ... ) + ... ).mpi_processes + 1 + >>> generator = as_test_case_definitions( + ... empty_path, + ... {"executable": [], "mpi_processes": [1, 2]} + ... ) + >>> list(definitions.mpi_processes for definitions in generator) + [1, 2] + """ + string_to_comparison_type = { + "none": ComparisonType.NONE, + "text": ComparisonType.TEXT, + "numeric": ComparisonType.NUMERIC, + } + + for mpi_processes in definition.get("mpi_processes", [1]): + yield TestCaseDefinition( + executable=pathlib.Path.cwd() / pathlib.Path(*definition["executable"]), + args=definition.get("args", []), + references=path / "references", + compare_stdout=string_to_comparison_type[ + definition.get("compare_stdout", "none") + ], + mpi_processes=mpi_processes, + ) + + +def run_executable_with_mpirun( + executable: pathlib.Path, + mpi_processes: int, + args: typing.List[str], + working_directory: pathlib.Path, +) -> subprocess.CompletedProcess: + """ + Runs the executable at `path` with the provided `args` + from `working_directory` with `mpi_processes` processes. + + >>> empty_path = pathlib.Path() + >>> run_executable_with_mpirun( + ... empty_path, + ... mpi_processes = 2, + ... args=["-v"], + ... working_directory=empty_path + ... ) # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + subprocess.CalledProcessError: Command '['mpirun', '-n', '2', '.', '-v']' ... + """ + return subprocess.run( + args=["mpirun", "-n", str(mpi_processes), str(executable)] + args, + cwd=working_directory, + capture_output=True, + check=True, + text=True, + ) + + +def diff_files( + case: unittest.TestCase, + value: pathlib.Path, + reference: pathlib.Path, + comparison: ComparisonType, +): + """ + Compares the files at `value`, `reference` as specified by `comparison. + Results are reported to `case`. + Nonexisting references are automatically created. + + >>> path = pathlib.Path(__file__) + >>> diff_files(unittest.TestCase(), path, path, ComparisonType.TEXT) + """ + if not reference.exists(): + shutil.copy(value, reference) + + comparison_to_function = { + ComparisonType.TEXT: diff_text_string, + ComparisonType.NUMERIC: diff_numeric_string, + } + + with open(value, "r") as value_file: + with open(reference, "r") as reference_file: + comparison_to_function.get(comparison, lambda _0, _1, _2: None)( + value_file.read(), reference_file.read(), case + ) + + +def diff_text_string( + value: str, reference: str, case: unittest.TestCase = unittest.TestCase() +): + """ + Checks that the lines in the strings `value` and `reference` are equal. + + >>> diff_text_string("a", "a") + >>> diff_text_string("a", "b") # doctest: +ELLIPSIS + Traceback (most recent call last): + ... + AssertionError: 'a' != 'b' + ... + """ + case.assertEqual(value, reference) + + +def float_or_nan(value: str) -> float: + """ + Converts the `value` to float. If this fails then returns NaN. + + >>> float_or_nan("1") + 1.0 + >>> float_or_nan("1.123") + 1.123 + >>> float_or_nan(" 1.123 ") + 1.123 + >>> float_or_nan("ab") + nan + >>> float_or_nan(" ") + nan + """ + + try: + return float(value) + except ValueError: + return math.nan + + +def extract_numbers(text: typing.Iterable[str]) -> typing.Iterable[float]: + """ + Extracts the numbers (separated by ',', '(', ')', ']', '[', '=', or whitespace) from text + and returns an iterator. + + >>> list(extract_numbers(["1.0, 2.0"])) + [1.0, 2.0] + >>> list(extract_numbers(["x=1.0"])) + [1.0] + >>> list(extract_numbers(["[1.0]"])) + [1.0] + >>> list(extract_numbers(["1.e1"])) + [10.0] + >>> list(extract_numbers(["NaN"])) + [] + >>> list(extract_numbers(["1.0 (2.0)"])) + [1.0, 2.0] + >>> list(extract_numbers(["1.0 a 2.0"])) + [1.0, 2.0] + >>> list(extract_numbers(["1.0 a 2.0", "b 3.0"])) + [1.0, 2.0, 3.0] + """ + return filter( + lambda x: not math.isnan(x), + map( + float_or_nan, + itertools.chain.from_iterable( + map(lambda x: re.split(r"[,\(\)\[\]=\s]", x), text) + ), + ), + ) + + +def diff_numeric_string( + value: str, reference: str, case: unittest.TestCase = unittest.TestCase() +): + """ + Checks that the lines in the strings `value` and `reference` are almost equal + when interpreted as floats. Non-float lines are interpreted as NaNs. + + >>> diff_numeric_string("a", "a") + >>> diff_numeric_string("1", "1") + >>> diff_numeric_string("1 2", "1 2") + >>> diff_numeric_string("1", "2") + Traceback (most recent call last): + ... + AssertionError: 1.0 != 2.0 within 7 places (1.0 difference) + >>> diff_numeric_string("a 1", "b 2") + Traceback (most recent call last): + ... + AssertionError: 1.0 != 2.0 within 7 places (1.0 difference) + >>> diff_numeric_string("a", "1") + Traceback (most recent call last): + ... + AssertionError: nan != 1.0 within 7 places (nan difference) + >>> diff_numeric_string("1", "a") + Traceback (most recent call last): + ... + AssertionError: 1.0 != nan within 7 places (nan difference) + """ + for float_value, float_reference in itertools.zip_longest( + extract_numbers(iter(value.splitlines())), + extract_numbers(iter(reference.splitlines())), + fillvalue=math.nan, + ): + case.assertAlmostEqual(float_value, float_reference) + + +def load_tests( + loader: unittest.TestLoader, + standard_tests: unittest.TestSuite, + _: typing.Any, +) -> unittest.TestSuite: + """ + Uses the provided definitions in tests/ to create a TestSuite of tests. + """ + paths = (ROOT_DIRECTORY / "tests").glob("*/*/definition.json") + + for path in paths: + group_name, test_name = path.parent.parts[-2:] + to_method_name = ( + lambda processes, name=test_name: f"test_{name}_with_{processes}_mpi_processes" + ) + + with open(path, "r") as file: + test_case_definitions = as_test_case_definitions( + path.parent, json.load(file) + ) + + testcase = type( + group_name, + (unittest.TestCase,), + { + to_method_name( + definition.mpi_processes + ): lambda case, definition=definition: run_testcase( + definition, case + ) + for definition in test_case_definitions + }, + ) + standard_tests.addTests(loader.loadTestsFromTestCase(testcase)) + + return standard_tests + + +def run_testcase( + definition: TestCaseDefinition, case: unittest.TestCase = unittest.TestCase() +): + """ + Runs the test defined by `definition` and reports the issues to `case`. + Nonexisting references are automatically created. + """ + with tempfile.TemporaryDirectory() as directory: + directory_path = pathlib.Path(directory) + + result = run_executable_with_mpirun( + executable=definition.executable, + args=definition.args, + working_directory=directory_path, + mpi_processes=definition.mpi_processes, + ) + + with open(directory_path / "stdout.txt", "w") as file: + file.write(result.stdout) + + diff_files( + case, + directory_path / "stdout.txt", + definition.references / "stdout.txt", + definition.compare_stdout, + ) + + +def main() -> None: + """ + Runs the tests defined in "definition.json"s. + """ + unittest.main() + + +if __name__ == "__main__": + main()