GItHub CI: Run linux amd64+aarch64 retagger weekly jobs and enable auto tests on PR

Summary:
What this does:
- Build python-svm and upload it so next jobs can use it.
- Run {RETAGGER_SPLIT} retaggers batches and upload each JSON reports using the {MX_REPORT_SUFFIX} suffix.
- Merge all reports, run the merge-tags-from-reports from runner.py

Uploaded artifacts:
- graalpy_standalone jvm and native => used by next jobs to speed up the process
- retagger-report.json files => used by the merger
- report_diff => git diff that contains all the tagges unittests changes

This commit only support Linux amd64?aarch64 atm, darwin and windows are WIP.

Author: Ariouz

.github/scripts/extract_matrix.py

@@ -18,6 +18,7 @@
     "GITHUB_CI": "true"
 }
 
+PR_JOBS='^(?=.*python)(?!.*(retagger|dsl)).*$'
 
 # If any of these terms are in the job json, they do not run in public
 # infrastructure
@@ -208,6 +209,7 @@ def download_artifact(self) -> Artifact | None:
             pattern = self.common_glob([a["name"] for a in artifacts])
             return Artifact(pattern, os.path.normpath(artifacts[0].get("dir", ".")))
         return None
+    
 
     @staticmethod
     def flatten_command(args: list[str | list[str]]) -> list[str]:
@@ -293,6 +295,8 @@ def get_tagged_jobs(buildspec, target, filter=None):
 
 
 def main(jsonnet_bin, ci_jsonnet, target, filter=None, indent=False):
+    if not filter: filter = PR_JOBS
+
     result = subprocess.check_output([jsonnet_bin, ci_jsonnet], text=True)
     buildspec = json.loads(result)
     tagged_jobs = get_tagged_jobs(buildspec, target, filter=filter)

.github/scripts/merge_retagger_results.py

@@ -0,0 +1,73 @@
+# ================================
+# 
+# This script is used by ci to merge several retagger report JSON files, which is then used
+# by running python3 runner.py merge-tags-from-reports reports-merged.json
+#
+# ================================
+
+import os
+import sys
+import json
+import glob
+import argparse
+from dataclasses import dataclass
+
+# status we want to focus on
+EXPORT_STATUS = ["FAILED"]
+
+@dataclass
+class Test:
+    name:       str
+    status:     str
+    duration:   str
+
+
+def read_report(path: str) -> list[Test]:
+    tests = []
+    with open(path) as f:
+        data = json.load(f)
+        for result in data:
+            name, status, duration = result.values()
+            if status in EXPORT_STATUS: tests.append(Test(f"{name}", status, duration))
+
+    return tests
+
+def merge_tests(report: list[Test], merged: dict[str, dict]):
+    for test in report:
+        if test.name not in merged:
+            merged[test.name] = test.__dict__
+
+def export_reports(merged: dict[str, dict], outfile: str):
+    with open(outfile, "w") as f:
+        json.dump(list(merged.values()), f)
+    print(f"=== Exported {len(merged)} ({EXPORT_STATUS}) tests to {f.name} ===")
+
+def merge_reports(reports: list[str], outfile: str):
+    merged_reports = {}
+    for report in reports:
+        report_tests = read_report(report)
+        merge_tests(report_tests, merged_reports)
+
+    export_reports(merged_reports, outfile)
+
+def main(outfile: str, source_dir: str, pattern: str):
+    path = f"{source_dir}/{pattern}"
+    files = glob.glob(path)
+    if len(files) == 0: raise FileNotFoundError("No report file found")
+
+    files = [file for file in files if file.endswith(".json")]
+    merge_reports(files, outfile)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Merge unittest retagger report JSON files")
+    parser.add_argument("--outfile", help="Output file name (optional)", default="reports-merged.json")
+    parser.add_argument("--dir", help="Reports files directory (optional)", default=".")
+    parser.add_argument("--pattern", default="*", help="Pattern matching for input files (optional)")
+
+    args = parser.parse_args()
+    main(
+        outfile=args.outfile,
+        source_dir=args.dir,
+        pattern=args.pattern
+    )

.github/workflows/ci-matrix-gen.yml

@@ -7,6 +7,9 @@ on:
         type: string
         description: Job selection (Python regex)
         required: false
+  pull_request:
+  schedule:
+    - cron: '0 0 * * 1'
 
 jobs:
   generate-tier1:
@@ -15,7 +18,7 @@ jobs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     env:
       TARGET: tier1
-      JOBS: ${{ inputs.jobs_to_run }}
+      JOBS: ${{ inputs.jobs_to_run || (github.event_name == 'schedule' && 'python-svm-build-gate-linux|python-unittest-retagger|python-unittest-retagger-merge') || ''}}
     steps: &generate_matrix
       - uses: actions/checkout@v4
       - name: Download sjsonnet
@@ -153,9 +156,10 @@ jobs:
         uses: actions/download-artifact@v5
         if: ${{ matrix.require_artifact }}
         with:
-          name: ${{ matrix.require_artifact[0] }}
+          pattern: ${{ matrix.require_artifact[0] }}
           merge-multiple: true
         continue-on-error: true
+      
       - name: Export artifact paths Linux
         if: ${{ matrix.require_artifact }}
         run: |
@@ -203,8 +207,8 @@ jobs:
         shell: bash
         working-directory: main
         run: |
-          tar cf ${{ matrix.provide_artifact[0] }}.tar ${{ matrix.provide_artifact[1] }}
           ls && pwd
+          tar cf ${{ matrix.provide_artifact[0] }}.tar ${{ matrix.provide_artifact[1] }}
       - name: Upload artifacts
         if: ${{ matrix.provide_artifact }}
         uses: actions/upload-artifact@v5

ci.jsonnet

@@ -101,7 +101,7 @@
     local GRAAL_JDK_LATEST          = "graal-jdk-latest",
     local TAGGED_UNITTESTS_SPLIT    = 8,
     local COVERAGE_SPLIT            = 3,
-    local RETAGGER_SPLIT            = 8,
+    local RETAGGER_SPLIT            = 12,
 
     // -----------------------------------------------------------------------------------------------------------------
     // gates
@@ -242,8 +242,8 @@
             "linux:amd64:jdk-latest"     : tier1,
         }),
         "python-unittest-retagger": ut_retagger + platform_spec(no_jobs) + batches(RETAGGER_SPLIT) + platform_spec({
-            "linux:amd64:jdk-latest"     : tier3,
-            "linux:aarch64:jdk-latest"   : weekly    + t("20:00:00"),
+            "linux:amd64:jdk-latest"     : tier2        + require(GPY_NATIVE_STANDALONE),
+            "linux:aarch64:jdk-latest"   : tier3        + require(GPY_NATIVE_STANDALONE),
             "darwin:aarch64:jdk-latest"  : weekly    + t("20:00:00"),
             "windows:amd64:jdk-latest"   : weekly    + t("20:00:00"),
         }),
@@ -548,6 +548,37 @@
                 ["git", "push", "--force", "origin", "published"],
             ]
         },
+        {
+            name: "python-unittest-retagger-merge",
+            targets: ["tier3"],
+            capabilities: ["linux", "amd64"],
+            packages: {
+                mx: "7.34.1",
+                python3: "==3.12.8",
+            },
+            requireArtifacts: [
+                {
+                    name: "python-unittest-retagger*",
+                    dir: ".",
+                }
+            ],
+            publishArtifacts: [
+                {
+                    name: "retagger.diff",
+                    dir: ".",
+                    patterns: ["diff_reports"]
+                }
+            ],
+            run: [
+                ["mkdir", "-p", "../retagger-reports"],
+                ["sh", "-c", "mv retagger-report*.json ../retagger-reports"],
+                ["cd", "../retagger-reports"],
+                ["python3", "../main/.github/scripts/merge_retagger_results.py"],
+                ["cd", "../main"],
+                ["python3", "./graalpython/com.oracle.graal.python.test/src/runner.py", "merge-tags-from-report", "../retagger-reports/reports-merged.json"],
+                ["sh", "-c", "git diff >> diff_reports"],
+            ]
+        },
     ],
 }
 

ci/python-gate.libsonnet

@@ -340,6 +340,8 @@
         task_spec({
             environment+: {
                 TAGGED_UNITTEST_PARTIAL: "%d/%d" % [i, num],
+                RETAGGER_BATCH_NO: i,
+                MX_REPORT_SUFFIX: "_batch_%d" % [i],
             },
             variations+::["batch" + i]
         }),
@@ -510,11 +512,11 @@
                 # The default timeout is very generous to allow for infrastructure flakiness,
                 # but we don't want to auto tag tests that take a long time
                 "--timeout-factor", "0.3",
-                "--mx-report", "report.json",
+                "--mx-report", "retagger-report.json",
                 "--exit-success-on-failures",
             ],
         ],
-        logs+: ["*/*/report.json"],
+        logs+: ["main/retagger-report*.json"],
     }),
 
     coverage_gate:: $.graalpy_gate + task_spec({

graalpython/com.oracle.graal.python.test/src/runner.py

@@ -446,6 +446,12 @@ def run_tests(self, tests: list['TestSuite']):
         self.display_summary()
 
     def generate_mx_report(self, path: str):
+        # Some reports may be split when ran on github, this sets different file names
+        report_suffix = os.environ.get("MX_REPORT_SUFFIX")
+        if report_suffix:
+            tmppath, ext = os.path.splitext(path)
+            path = f"{tmppath}{report_suffix}{ext}"
+
         report_data = []
         for result in self.results:
             # Skip synthetic results for failed class setups and such
@@ -496,6 +502,7 @@ def update_tags(test_file: 'TestFile', results: list[TestResult], tag_platform:
             tag_by_id[tag.test_id] = tag
 
     for test_id, status in status_by_id.items():
+
         if tag := tag_by_id.get(test_id):
             if status == TestStatus.SUCCESS:
                 tag_by_id[test_id] = tag.with_key(tag_platform)
@@ -1349,7 +1356,7 @@ def main_merge_tags(args):
             test_file,
             results,
             tag_platform=args.platform,
-            untag_failed=False,
+            untag_failed= os.environ.get("GITHUB_CI") is not None,
             untag_skipped=True,
             untag_missing=True,
         )
@@ -1509,7 +1516,7 @@ def main():
     # merge-tags-from-report command declaration
     merge_tags_parser = subparsers.add_parser('merge-tags-from-report', help="Merge tags from automated retagger")
     merge_tags_parser.set_defaults(main=main_merge_tags)
-    merge_tags_parser.add_argument('platform')
+    merge_tags_parser.add_argument('--platform', default=CURRENT_PLATFORM)
     merge_tags_parser.add_argument('report_path')
 
     # run the appropriate command