Add Weekly Retagger + GitHub unittest exclusion char

See #3

Author: Ariouz

.github/scripts/extract_matrix.py

@@ -18,8 +18,6 @@
     "GITHUB_CI": "true"
 }
 
-PR_JOBS='^(?=.*python)(?!.*(retagger|dsl)).*$'
-
 # If any of these terms are in the job json, they do not run in public
 # infrastructure
 JOB_EXCLUSION_TERMS = (
@@ -36,7 +34,7 @@
 
     # WIP
     "darwin",
-    "windows"
+    #"windows"
 )
 
 DOWNLOADS_LINKS = {
@@ -130,10 +128,12 @@ def get_download_steps(self, key: str, version: str) -> str:
         filename = download_link.split('/')[-1]
 
         if self.runs_on == "windows-latest":
-            return (f"""Invoke-WebRequest -Uri {download_link} -OutFile {filename}
+            return (f"""
+            Invoke-WebRequest -Uri {download_link} -OutFile {filename}
             $dirname = (& tar -tzf {filename} | Select-Object -First 1).Split('/')[0]
             tar -xzf {filename}
-            Add-Content $env:GITHUB_ENV "{key}=$(Resolve-Path $dirname)""")
+            Add-Content $env:GITHUB_ENV "{key}=$(Resolve-Path $dirname)"
+            """)
 
         return (f"wget -q {download_link} && "
             f"dirname=$(tar -tzf {filename} | head -1 | cut -f1 -d '/') && "
@@ -295,7 +295,6 @@ def get_tagged_jobs(buildspec, target, filter=None):
 
 
 def main(jsonnet_bin, ci_jsonnet, target, filter=None, indent=False):
-    if not filter: filter = PR_JOBS
 
     result = subprocess.check_output([jsonnet_bin, ci_jsonnet], text=True)
     buildspec = json.loads(result)

.github/scripts/merge_retagger_results.py

@@ -53,7 +53,6 @@ def merge_reports(reports: list[str], outfile: str):
 def main(outfile: str, source_dir: str, pattern: str):
     path = f"{source_dir}/{pattern}"
     files = glob.glob(path)
-    if len(files) == 0: raise FileNotFoundError("No report file found")
 
     files = [file for file in files if file.endswith(".json")]
     merge_reports(files, outfile)

.github/scripts/unpack-artifact.cmd

@@ -0,0 +1,18 @@
+@echo off
+setlocal
+
+if not "%1"=="" (
+    set "to_extract=%ARTIFACT_PATH_PREFIX%\%1.tar"
+)
+
+for %%i in (%to_extract%) do (
+    if exist "%%i" (
+        tar -xf "%%i" -C "%ARTIFACT_PATH_PREFIX%"
+        echo Unpacked %%i in %CD%\%ARTIFACT_PATH_PREFIX%
+    ) else (
+        echo file not found in %ARTIFACT_PATH_PREFIX%
+        dir "%ARTIFACT_PATH_PREFIX%"
+    )
+)
+
+endlocal

.github/workflows/ci-matrix-gen.yml

@@ -7,7 +7,11 @@ on:
         type: string
         description: Job selection (Python regex)
         required: false
-  pull_request:
+  workflow_call:
+    inputs:
+      jobs_to_run:
+        required: true
+        type: string
   schedule:
     - cron: '0 0 * * 1'
 
@@ -18,7 +22,7 @@ jobs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     env:
       TARGET: tier1
-      JOBS: ${{ inputs.jobs_to_run || (github.event_name == 'schedule' && 'python-svm-build-gate-linux|python-unittest-retagger|python-unittest-retagger-merge') || ''}}
+      JOBS: ${{ inputs.jobs_to_run }}
     steps: &generate_matrix
       - uses: actions/checkout@v4
       - name: Download sjsonnet
@@ -122,7 +126,7 @@ jobs:
 
       # Setup mx, buildtools and labsjdk
       - name: Get mx, buildtools, and labsjdk
-        if: ${{ runner.os != 'Windows' && matrix.mx_version }}
+        if: ${{ matrix.mx_version }}
         shell: bash
         run: |
           git clone https://github.com/graalvm/mx
@@ -186,8 +190,59 @@ jobs:
         run: |
           ${{ matrix.setup_steps }}
 
-      - name: Check disk space
-        run: df -h
+      - name: Check disk space and memory usage
+        if: ${{ runner.os == 'macOS' }}
+        run: |
+          echo "before purge"
+          df -h
+          pagesize=$(sysctl -n hw.pagesize)
+          vm_stat
+          vm_stat | awk -v ps=$pagesize '
+            /Pages free/      {free=$3}
+            /Pages active/    {active=$3}
+            /Pages inactive/  {inactive=$3}
+            /Pages speculative/ {speculative=$3}
+            /Pages wired/     {wired=$4}
+            END {
+              used=(active+inactive+wired+speculative)*ps/1024/1024;
+              free=free*ps/1024/1024;
+              printf "Used: %.2f MB\nFree: %.2f MB\n", used, free
+            }'
+          ps aux | sort -nrk 4 | head
+          sudo purge
+          echo "after purge"
+          pagesize=$(sysctl -n hw.pagesize)
+          vm_stat
+          vm_stat | awk -v ps=$pagesize '
+            /Pages free/      {free=$3}
+            /Pages active/    {active=$3}
+            /Pages inactive/  {inactive=$3}
+            /Pages speculative/ {speculative=$3}
+            /Pages wired/     {wired=$4}
+            END {
+              used=(active+inactive+wired+speculative)*ps/1024/1024;
+              free=free*ps/1024/1024;
+              printf "Used: %.2f MB\nFree: %.2f MB\n", used, free
+            }'
+
+
+      - name: Free resources on macOS
+        run: |
+          echo "Disabling Spotlight indexing..."
+          sudo mdutil -a -i off
+          sudo mdutil -a -d
+          sudo launchctl bootout system /System/Library/LaunchDaemons/com.apple.metadata.mds.plist
+          sudo pkill -f "mds|mds_stores|mdworker_shared|Spotlight|corespotlightd"
+
+          echo "Forcing disk cache to be purged..."
+          sudo purge
+
+          echo "Top processes sorted by memory usage:"
+          ps -emo pid,pmem,pcpu,comm | head 2>/dev/null
+          echo ""
+          memory_pressure
+          echo ""
+        if: ${{ runner.os == 'macOS' }}
 
       - name: Run on Unix
         working-directory: main
@@ -196,6 +251,7 @@ jobs:
           env
           ${{ matrix.run_steps }}
       - name: Run on Windows
+        working-directory: main
         if: ${{ matrix.run_steps && runner.os == 'Windows' }}
         shell: cmd
         run: |

.github/workflows/ci-unittest-retagger.yml

@@ -0,0 +1,67 @@
+name: Run Weekly unittest retagger
+on:
+  schedule:
+    - cron: '0 0 * * 1'
+  workflow_dispatch:
+
+jobs:
+  run-retagger:
+    uses: ./.github/workflows/ci-matrix-gen.yml
+    with:
+      jobs_to_run: python-svm-build-gate-linux-amd64|python-unittest-retagger
+
+  merge_all_reports:
+    runs-on: ubuntu-latest
+    needs: run-retagger
+    if: ${{ success() }}
+    env:
+      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps: 
+
+      - name: Actions/Checkout
+        uses: actions/checkout@main
+        with:
+          path: main
+
+      - name: Download reports
+        uses: actions/download-artifact@v5
+        with:
+          pattern: python-unittest-retagger*
+          merge-multiple: true
+        continue-on-error: true
+
+      - name: Init Github branch
+        working-directory: main
+        run: |
+          echo "Set new repo origin url"
+          git remote set-url origin https://$GH_TOKEN@github.com/${{ github.repository }}.git
+          git remote -v
+          git config --global user.name "Retagger Workflow" 
+          git config --global user.email "Retagger_Workflow@oracle.com"
+          git fetch origin
+          git checkout -b weekly_retagger_${{ github.run_id }} origin/master
+
+      - name: Merge retagger reports
+        env:
+          GITHUB_CI: true
+        working-directory: main
+        run: |
+          mkdir -p ../retagger-reports ../diffs
+          ls && ls ..
+          mv retagger-report*.json ../retagger-reports
+
+          declare -a os_list=("linux-x86_64" "linux-aarch64" "win32-AMD64")
+          for os in "${os_list[@]}"; do
+            echo "Merging tags for $os"
+            python3 .github/scripts/merge_retagger_results.py --dir ../retagger-reports --outfile "../retagger-reports/reports-merged-$os.json" --pattern "*$os*" || true
+            python3 graalpython/com.oracle.graal.python.test/src/runner.py merge-tags-from-report "../retagger-reports/reports-merged-$os.json" --platform "$os" || true
+            git add -A
+            git commit -m "Apply retags for $os" || true
+          done
+        
+      - name: Push updates to a new PR
+        working-directory: main
+        run: |
+          git push --set-upstream origin weekly_retagger_${{ github.run_id }}
+          gh label create "weekly-retagger" --color "BDDFDF" -d "[DO NOT ADD] Automatically set to weekly retagger PR" || echo "Label 'weekly-retagger' already exists"
+          gh pr create -B master --title "[WORKFLOW] Weekly Retagger: Update tags" --body "Applied weekly retags on $os_list" -l "weekly-retagger"

.github/workflows/ci-unittests.yml

@@ -0,0 +1,12 @@
+name: Run CI unittests
+on:
+  pull_request:
+    types: [review_requested, ready_for_review, synchronize]
+  workflow_dispatch:
+
+jobs:
+  run-tests:
+    if: github.event.pull_request.draft == false
+    uses: ./.github/workflows/ci-matrix-gen.yml
+    with:
+      jobs_to_run: ^(?=.*python)(?!.*(retagger|dsl|windows|darwin)).*$

ci.jsonnet

@@ -244,8 +244,8 @@
         "python-unittest-retagger": ut_retagger + platform_spec(no_jobs) + batches(RETAGGER_SPLIT) + platform_spec({
             "linux:amd64:jdk-latest"     : tier2        + require(GPY_NATIVE_STANDALONE),
             "linux:aarch64:jdk-latest"   : tier3        + require(GPY_NATIVE_STANDALONE),
-            "darwin:aarch64:jdk-latest"  : weekly    + t("20:00:00"),
-            "windows:amd64:jdk-latest"   : weekly    + t("20:00:00"),
+            "darwin:aarch64:jdk-latest"  : tier3        + require(GPY_NATIVE_STANDALONE),
+            "windows:amd64:jdk-latest"   : tier3        + require(GPY_NATIVE_STANDALONE),
         }),
         "python-coverage-jacoco-tagged": cov_jacoco_tagged + batches(COVERAGE_SPLIT) + platform_spec(no_jobs) + platform_spec({
             "linux:amd64:jdk21"          : weekly    + t("20:00:00"),
@@ -548,37 +548,6 @@
                 ["git", "push", "--force", "origin", "published"],
             ]
         },
-        {
-            name: "python-unittest-retagger-merge",
-            targets: ["tier3"],
-            capabilities: ["linux", "amd64"],
-            packages: {
-                mx: "7.34.1",
-                python3: "==3.12.8",
-            },
-            requireArtifacts: [
-                {
-                    name: "python-unittest-retagger*",
-                    dir: ".",
-                }
-            ],
-            publishArtifacts: [
-                {
-                    name: "retagger.diff",
-                    dir: ".",
-                    patterns: ["diff_reports"]
-                }
-            ],
-            run: [
-                ["mkdir", "-p", "../retagger-reports"],
-                ["sh", "-c", "mv retagger-report*.json ../retagger-reports"],
-                ["cd", "../retagger-reports"],
-                ["python3", "../main/.github/scripts/merge_retagger_results.py"],
-                ["cd", "../main"],
-                ["python3", "./graalpython/com.oracle.graal.python.test/src/runner.py", "merge-tags-from-report", "../retagger-reports/reports-merged.json"],
-                ["sh", "-c", "git diff >> diff_reports"],
-            ]
-        },
     ],
 }
 

ci/graal/ci/common.jsonnet

@@ -150,6 +150,7 @@ local common_json = import "../common.json";
       packages+: {
         python3: "==3.8.10",
         "pip:ninja_syntax": common_json.pip.ninja_syntax,
+        "pip:psutil": "==7.1.3",
         mx: common_json.mx_version,
       },
       python_version: "3", # To use the correct virtualenv

ci/graal/common.json

@@ -4,7 +4,7 @@
     "Jsonnet files should not include this file directly but use ci/common.jsonnet instead."
   ],
 
-  "mx_version": "7.67.0",
+  "mx_version": "7.68.4",
 
   "COMMENT.jdks": "When adding or removing JDKs keep in sync with JDKs in ci/common.jsonnet",
   "jdks": {
@@ -49,12 +49,12 @@
     "graalvm-ee-25-ea":   {"name": "graalvm-jdk", "version": "25.0.0", "ea": "36", "platformspecific": true },
 
     "oraclejdk-latest":       {"name": "jpg-jdk",   "version": "25",      "build_id": "jdk-25.0.1+8", "platformspecific": true, "extrabundles": ["static-libs"]},
-    "labsjdk-ce-latest":      {"name": "labsjdk",   "version": "ce-25.0.1+8-jvmci-25.1-b10", "platformspecific": true },
-    "labsjdk-ce-latestDebug": {"name": "labsjdk",   "version": "ce-25.0.1+8-jvmci-25.1-b10-debug", "platformspecific": true },
-    "labsjdk-ce-latest-llvm": {"name": "labsjdk",   "version": "ce-25.0.1+8-jvmci-25.1-b10-sulong", "platformspecific": true },
-    "labsjdk-ee-latest":      {"name": "labsjdk",   "version": "ee-25.0.1+8-jvmci-25.1-b10", "platformspecific": true },
-    "labsjdk-ee-latestDebug": {"name": "labsjdk",   "version": "ee-25.0.1+8-jvmci-25.1-b10-debug", "platformspecific": true },
-    "labsjdk-ee-latest-llvm": {"name": "labsjdk",   "version": "ee-25.0.1+8-jvmci-25.1-b10-sulong", "platformspecific": true }
+    "labsjdk-ce-latest":      {"name": "labsjdk",   "version": "ce-25.0.1+8-jvmci-25.1-b11", "platformspecific": true },
+    "labsjdk-ce-latestDebug": {"name": "labsjdk",   "version": "ce-25.0.1+8-jvmci-25.1-b11-debug", "platformspecific": true },
+    "labsjdk-ce-latest-llvm": {"name": "labsjdk",   "version": "ce-25.0.1+8-jvmci-25.1-b11-sulong", "platformspecific": true },
+    "labsjdk-ee-latest":      {"name": "labsjdk",   "version": "ee-25.0.1+8-jvmci-25.1-b11", "platformspecific": true },
+    "labsjdk-ee-latestDebug": {"name": "labsjdk",   "version": "ee-25.0.1+8-jvmci-25.1-b11-debug", "platformspecific": true },
+    "labsjdk-ee-latest-llvm": {"name": "labsjdk",   "version": "ee-25.0.1+8-jvmci-25.1-b11-sulong", "platformspecific": true }
   },
 
   "eclipse": {
@@ -69,4 +69,4 @@
     "pylint": "==2.4.4",
     "black": "==23.11.0"
   }
-}
+}

ci/python-gate.libsonnet

@@ -160,7 +160,7 @@
         },
         windows: {
             common: {
-                PATH: "$MAVEN_HOME\\bin;$PATH",
+                PATH: "$MAVEN_HOME\\bin;$PATH;C:\\hostedtoolcache\\windows\\Python\\3.12.3\\x64\\Scripts;C:\\hostedtoolcache\\windows\\Python\\3.12.3\\x64;C:\\Program Files\\PowerShell\\7;C:\\windows\\system32;C:\\windows;C:\\windows\\System32\\Wbem;C:\\windows\\System32\\WindowsPowerShell\\v1.0\\;C:\\windows\\System32\\OpenSSH\\;C:\\Program Files\\dotnet\\;C:\\Program Files\\Microsoft SQL Server\\150\\Tools\\Binn\\;C:\\Program Files\\Git\\cmd;C:\\Program Files\\Git\\usr\\bin;C:\\hostedtoolcache\\windows\\Ruby\\3.2.3\\x64\\bin;C:\\hostedtoolcache\\windows\\Go\\1.22.2\\x64\\bin;C:\\hostedtoolcache\\windows\\Node\\20.11.1\\x64\\;C:\\npm\\prefix;C:\\Program Files\\Amazon\\AWSCLIV2\\;C:\\Program Files\\Amazon\\SessionManagerPlugin\\bin\\;C:\\Program Files\\LLVM\\bin;C:\\Program Files\\CMake\\bin;C:\\Program Files\\OpenSSL\\bin;C:\\msys64\\usr\\bin;C:\\hostedtoolcache\\windows\\php\\8.2.15\\x64;C:\\Program Files\\Azure DevOps CLI\\bin;C:\\hostedtoolcache\\windows\\R\\4.3.3\\x64\\bin;C:\\Program Files (x86)\\Microsoft Visual Studio\\2022\\Enterprise\\Common7\\IDE\\Extensions\\Microsoft\\NuGet;",
             },
             amd64: {},
             aarch64: {},