diff --git a/changelog-entries/821.md b/changelog-entries/821.md new file mode 100644 index 000000000..8d4ed554f --- /dev/null +++ b/changelog-entries/821.md @@ -0,0 +1 @@ +- System tests now update the GitHub Actions job summary incrementally after each test finishes, with per-test status and timings, while keeping the final results table (fixes [#791](https://github.com/precice/tutorials/issues/791)). diff --git a/tools/tests/systemtests.py b/tools/tests/systemtests.py index 6debe18d0..f3f3c9aaa 100644 --- a/tools/tests/systemtests.py +++ b/tools/tests/systemtests.py @@ -2,7 +2,13 @@ import argparse from pathlib import Path from systemtests.SystemtestArguments import SystemtestArguments -from systemtests.Systemtest import Systemtest, GLOBAL_TIMEOUT, display_systemtestresults_as_table +from systemtests.Systemtest import ( + Systemtest, + GLOBAL_TIMEOUT, + append_systemtest_progress_to_github_summary, + display_systemtestresults_as_table, + initialize_systemtests_progress_in_github_summary, +) from systemtests.TestSuite import TestSuites from metadata_parser.metdata import Tutorials, Case import logging @@ -74,6 +80,7 @@ def main(): logging.info(f"About to run the following systemtest in the directory {run_directory}:\n {systemtests_to_run}") + initialize_systemtests_progress_in_github_summary(len(systemtests_to_run)) results = [] for number, systemtest in enumerate(systemtests_to_run, start=1): logging.info(f"Started running {systemtest}, {number}/{len(systemtests_to_run)}") @@ -82,6 +89,12 @@ def main(): elapsed_time = time.perf_counter() - t logging.info(f"Running {systemtest} took {elapsed_time:^.1f} seconds") results.append(result) + append_systemtest_progress_to_github_summary( + result, + number, + len(systemtests_to_run), + elapsed_time, + ) system_test_success = True for result in results: diff --git a/tools/tests/systemtests/Systemtest.py b/tools/tests/systemtests/Systemtest.py index f438eac03..abe1cf91a 100644 --- a/tools/tests/systemtests/Systemtest.py +++ b/tools/tests/systemtests/Systemtest.py @@ -23,6 +23,8 @@ SHORT_TIMEOUT = 10 DIFF_RESULTS_DIR = "diff-results" +GITHUB_SUMMARY_PROGRESS_HEADING = "## System tests progress" +GITHUB_SUMMARY_RESULTS_HEADING = "## Final system test results" def slugify(value, allow_unicode=False): @@ -76,6 +78,44 @@ class SystemtestResult: fieldcompare_time: float # in seconds +def _append_lines_to_github_summary(lines: List[str]): + summary_path = os.environ.get("GITHUB_STEP_SUMMARY") + if not summary_path: + return + + with open(summary_path, "a") as summary_file: + for line in lines: + print(line, file=summary_file) + + +def initialize_systemtests_progress_in_github_summary(total_systemtests: int): + if total_systemtests <= 0: + return + + _append_lines_to_github_summary([ + "", + GITHUB_SUMMARY_PROGRESS_HEADING, + "", + f"- Starting execution of `{total_systemtests}` system tests.", + ]) + + +def append_systemtest_progress_to_github_summary( + result: SystemtestResult, + number: int, + total_systemtests: int, + elapsed_time: float): + status_checkbox = "x" if result.success else " " + status_text = "finished successfully" if result.success else "failed" + progress_line = ( + f"- [{status_checkbox}] `{number}/{total_systemtests}` " + f"`{result.systemtest}` {status_text} in `{elapsed_time:.1f}s` " + f"(build `{result.build_time:.1f}s`, solver `{result.solver_time:.1f}s`, " + f"fieldcompare `{result.fieldcompare_time:.1f}s`)." + ) + _append_lines_to_github_summary([progress_line]) + + def display_systemtestresults_as_table(results: List[SystemtestResult]): """ Prints the result in a nice tabluated way to get an easy overview @@ -100,6 +140,9 @@ def _get_length_of_name(results: List[SystemtestResult]) -> int: if "GITHUB_STEP_SUMMARY" in os.environ: with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f: + print("", file=f) + print(GITHUB_SUMMARY_RESULTS_HEADING, file=f) + print("", file=f) print(header, file=f) print(separator_markdown, file=f)