ci: report build and test outcomes accurately
Build and Analyse / build-and-analyse (push) Successful in 1m37s
Build and Analyse / build-and-analyse (push) Successful in 1m37s
Co-authored-by: Junie <junie@jetbrains.com>
This commit is contained in:
1 parent
63713e30bd
commit
4c65cab3f6
3 files changed
+79
-3
No files matched your search
@@ -4,6 +4,9 @@ inputs:
|
|||||||
token:
|
token:
|
||||||
description: Gitea token with write access to the repository.
|
description: Gitea token with write access to the repository.
|
||||||
required: true
|
required: true
|
||||||
|
build-outcome:
|
||||||
|
description: Outcome of the build and test step.
|
||||||
|
required: true
|
||||||
|
|
||||||
runs:
|
runs:
|
||||||
using: composite
|
using: composite
|
||||||
@@ -17,6 +20,7 @@ runs:
|
|||||||
run: |
|
run: |
|
||||||
mkdir -p junit-short
|
mkdir -p junit-short
|
||||||
shopt -s globstar nullglob
|
shopt -s globstar nullglob
|
||||||
|
rm -f junit-short/*.xml
|
||||||
for f in **/target/*-reports/TEST-*.xml; do
|
for f in **/target/*-reports/TEST-*.xml; do
|
||||||
base="$(basename "$f")"
|
base="$(basename "$f")"
|
||||||
short="${base#TEST-}"
|
short="${base#TEST-}"
|
||||||
@@ -25,10 +29,12 @@ runs:
|
|||||||
|
|
||||||
- name: Generate Markdown Test Report
|
- name: Generate Markdown Test Report
|
||||||
shell: bash
|
shell: bash
|
||||||
|
env:
|
||||||
|
BUILD_OUTCOME: ${{ inputs.build-outcome }}
|
||||||
run: |
|
run: |
|
||||||
mkdir -p reports
|
mkdir -p reports
|
||||||
python3 - <<'EOF'
|
python3 - <<'EOF'
|
||||||
import glob, xml.etree.ElementTree as ET
|
import glob, os, xml.etree.ElementTree as ET
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
|
||||||
generated_at = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
|
generated_at = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S UTC")
|
||||||
@@ -57,7 +63,17 @@ runs:
|
|||||||
test_rows.append(f"| {status} | {name} | {tests} | {passed} | {failures + errors} | {skipped} |")
|
test_rows.append(f"| {status} | {name} | {tests} | {passed} | {failures + errors} | {skipped} |")
|
||||||
|
|
||||||
total_passed = total_tests - total_failures - total_errors - total_skipped
|
total_passed = total_tests - total_failures - total_errors - total_skipped
|
||||||
overall = "✅ All tests passed" if (total_failures + total_errors) == 0 else "❌ Some tests failed"
|
build_outcome = os.environ.get("BUILD_OUTCOME", "unknown")
|
||||||
|
if total_failures + total_errors:
|
||||||
|
overall = "❌ Some tests failed"
|
||||||
|
elif not files:
|
||||||
|
overall = "⚠️ No JUnit reports found"
|
||||||
|
elif total_passed == 0:
|
||||||
|
overall = "⚠️ No tests passed"
|
||||||
|
elif build_outcome != "success":
|
||||||
|
overall = "❌ Build did not succeed"
|
||||||
|
else:
|
||||||
|
overall = "✅ All tests passed"
|
||||||
|
|
||||||
# --- JaCoCo ---
|
# --- JaCoCo ---
|
||||||
def counter(el, type_):
|
def counter(el, type_):
|
||||||
@@ -87,6 +103,7 @@ runs:
|
|||||||
"# Test Report\n\n"
|
"# Test Report\n\n"
|
||||||
f"_Generated on {generated_at}_\n\n"
|
f"_Generated on {generated_at}_\n\n"
|
||||||
f"**{overall}**\n\n"
|
f"**{overall}**\n\n"
|
||||||
|
f"Build outcome: **{build_outcome}**\n\n"
|
||||||
"## Test Results\n\n"
|
"## Test Results\n\n"
|
||||||
"| | Tests | Passed | Failed | Skipped |\n"
|
"| | Tests | Passed | Failed | Skipped |\n"
|
||||||
"|---|---|---|---|---|\n"
|
"|---|---|---|---|---|\n"
|
||||||
|
|||||||
@@ -0,0 +1,57 @@
|
|||||||
|
import os
|
||||||
|
import pathlib
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
|
||||||
|
ACTION = pathlib.Path(__file__).resolve().parents[1] / "actions/publish-report/action.yml"
|
||||||
|
|
||||||
|
|
||||||
|
def generate_report(directory, outcome, xml=None):
|
||||||
|
action = ACTION.read_text()
|
||||||
|
script = action.split(" python3 - <<'EOF'\n", 1)[1].split(" EOF", 1)[0]
|
||||||
|
script = "\n".join(line[8:] for line in script.splitlines())
|
||||||
|
root = pathlib.Path(directory)
|
||||||
|
(root / "reports").mkdir()
|
||||||
|
(root / "junit-short").mkdir()
|
||||||
|
if xml is not None:
|
||||||
|
(root / "junit-short/test.xml").write_text(xml)
|
||||||
|
env = dict(os.environ, BUILD_OUTCOME=outcome)
|
||||||
|
subprocess.run(["python3", "-c", script], cwd=root, env=env, check=True, capture_output=True)
|
||||||
|
return (root / "reports/test.md").read_text()
|
||||||
|
|
||||||
|
|
||||||
|
class PublishReportTest(unittest.TestCase):
|
||||||
|
def test_missing_reports_are_not_successful(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
report = generate_report(directory, "failure")
|
||||||
|
self.assertNotIn("All tests passed", report)
|
||||||
|
self.assertIn("No JUnit", report)
|
||||||
|
self.assertIn("failure", report)
|
||||||
|
|
||||||
|
def test_failed_build_with_passing_tests_is_not_successful(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
report = generate_report(directory, "failure", '<testsuite name="example" tests="1" failures="0" errors="0" skipped="0"/>')
|
||||||
|
self.assertNotIn("All tests passed", report)
|
||||||
|
self.assertIn("failure", report)
|
||||||
|
|
||||||
|
def test_passing_build_and_tests(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
report = generate_report(directory, "success", '<testsuite name="example" tests="1" failures="0" errors="0" skipped="0"/>')
|
||||||
|
self.assertIn("All tests passed", report)
|
||||||
|
|
||||||
|
def test_all_skipped_is_not_successful(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
report = generate_report(directory, "success", '<testsuite name="example" tests="1" failures="0" errors="0" skipped="1"/>')
|
||||||
|
self.assertNotIn("All tests passed", report)
|
||||||
|
|
||||||
|
def test_failed_tests(self):
|
||||||
|
with tempfile.TemporaryDirectory() as directory:
|
||||||
|
report = generate_report(directory, "failure", '<testsuite name="example" tests="1" failures="1" errors="0" skipped="0"/>')
|
||||||
|
self.assertNotIn("All tests passed", report)
|
||||||
|
self.assertIn("Some tests failed", report)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -26,13 +26,15 @@ jobs:
|
|||||||
uses: ./.gitea/actions/setup-java-maven
|
uses: ./.gitea/actions/setup-java-maven
|
||||||
|
|
||||||
- name: Build and test
|
- name: Build and test
|
||||||
|
id: build
|
||||||
env:
|
env:
|
||||||
# required by the live tests (CfClientTest, CfClientPenTest), they are skipped if not set
|
# required by the live tests (CfClientTest, CfClientPenTest), they are skipped if not set
|
||||||
API_TOKEN: ${{ secrets.API_TOKEN }}
|
API_TOKEN: ${{ secrets.API_TOKEN }}
|
||||||
run: mvn -B verify
|
run: mvn -B clean verify
|
||||||
|
|
||||||
- name: Publish Test Report
|
- name: Publish Test Report
|
||||||
uses: ./.gitea/actions/publish-report
|
uses: ./.gitea/actions/publish-report
|
||||||
if: ${{ always() }}
|
if: ${{ always() }}
|
||||||
with:
|
with:
|
||||||
token: ${{ secrets.GITHUB_TOKEN }}
|
token: ${{ secrets.GITHUB_TOKEN }}
|
||||||
|
build-outcome: ${{ steps.build.outcome }}
|
||||||
Reference in new issue
Block a user