update ci-restart script with code that works and is testable locally

.github/workflows/check_repo_ci_jobs.py

@@ -1,57 +1,103 @@
+import pathlib
+import shutil
+
 import requests
+import zipfile
+import os
+
+# To test this locally, set these environment variables
+REPO_OWNER = os.environ.get("REPO_OWNER", "IfcOpenShell/IfcOpenShell")
+MY_WORKFLOW = os.environ.get("MY_WORKFLOW", "ci-ifcopenshell-conda-daily")
+WORKFLOW_RUN_ID = os.environ.get("WORKFLOW_RUN_ID", None)
+TOKEN = os.environ.get("GITHUB_TOKEN", None)
+# Create a fine-grained personal access token for the repo in question with repo permissions "actions:read"
+# See https://github.com/settings/tokens?type=beta
+
+# This is a list of strings that indicate that a job has stopped abruptly.
+SIGNS_OF_STOPPAGE = [
+    "Error: The operation was canceled.",
+    "fatal error C1060: compiler is out of heap space",
+]
 
 
-def get_ci_jobs(repo_name):
-    url = f"https://api.github.com/repos/{repo_name}/actions/runs"
-    response = requests.get(url)
+def start_request_session():
+    s = requests.Session()
+    headers = {
+        "Accept": "application/vnd.github+json",
+        "X-GitHub-Api-Version": "2022-11-28",
+    }
+    if TOKEN:
+        headers["Authorization"] = f"Bearer {TOKEN}"
+
+    s.headers = headers
+    return s
+
+
+def get_ci_run(repo_name, run_id):
+    url = f"https://api.github.com/repos/{repo_name}/actions/runs/{run_id}"
+    s = start_request_session()
+    response = s.get(url)
     return response.json()
 
 
+def evaluate_log_file_for_abrupt_stop(log_file):
+    with open(log_file) as f:
+        for i, line in enumerate(f):
+            for sign in SIGNS_OF_STOPPAGE:
+                if sign in line:
+                    print(f"Found sign of abrupt stoppage in [line {i}]: '{sign}'")
+                    return True
+    return False
+
+
 def get_ci_specific_run_specific_failure_details(repo_name, run_id):
     url = f"https://api.github.com/repos/{repo_name}/actions/runs/{run_id}/attempts/1/logs"
-    response = requests.get(url)
-    return response.json()
+    s = start_request_session()
+    response = s.get(url)
+
+    # SAVE zip file
+    with open("logs.zip", "wb") as f:
+        f.write(response.content)
+
+    # unzip file
+    with zipfile.ZipFile("logs.zip", "r") as zip_ref:
+        zip_ref.extractall("logs")
+
+    failed_logs = []
+    for file in pathlib.Path("logs").iterdir():
+        if file.is_dir():
+            continue
+
+        if evaluate_log_file_for_abrupt_stop(file):
+            failed_logs.append(file)
+
+    return failed_logs
 
 
 def restart_job(repo_name, job_id):
-    url = f"https://api.github.com/repos/{repo_name}/actions/runs/{job_id}/rerun"
-    response = requests.post(url)
+    url = f"https://api.github.com/repos/{repo_name}/actions/runs/{job_id}/rerun-failed-jobs"
+    s = start_request_session()
+    response = s.post(url)
     return response.json()
 
 
-def logs_contain_signs_of_stoppage_due_to_lack_of_vm_resources(logs):
-    """Do something like this?"""
-    for log in logs["lines"]:
-        if log["name"] == "stderr":
-            if "Error: The operation was canceled." in log["text"]:
-                return True
-            if "fatal error C1060: compiler is out of heap space" in log["text"]:
-                return True
-    return False
-
-
 def eval_jobs():
-    jobs = get_ci_jobs("IfcOpenShell/IfcOpenShell")
-    for job in jobs["workflow_runs"]:
-        if job["name"] != "ci-ifcopenshell-conda-daily":
-            continue
+    run = get_ci_run(REPO_OWNER, WORKFLOW_RUN_ID)
+    if run["run_attempt"] > 1:
+        print("This is not the first attempt. Exiting")
+        return
+    failed_logs = get_ci_specific_run_specific_failure_details(REPO_OWNER, WORKFLOW_RUN_ID)
 
-        if job["status"] != "completed":
-            continue
-        if job["conclusion"] != "failure":
-            continue
+    if len(failed_logs) == 0:
+        print("No runs exhibit signs of abrupt stoppage")
+        return
 
-        if job["run_attempt"] > 1:
-            continue
+    print("restarting job", WORKFLOW_RUN_ID)
+    r = restart_job(REPO_OWNER, WORKFLOW_RUN_ID)
+    print(r)
 
-        logs = get_ci_specific_run_specific_failure_details("IfcOpenShell/IfcOpenShell", job["id"])
-
-        print(logs)
-        print("restarting job", job["id"])
-
-        if logs_contain_signs_of_stoppage_due_to_lack_of_vm_resources(logs):
-            r = restart_job("IfcOpenShell/IfcOpenShell", job["id"])
-            print(r)
+    shutil.rmtree("logs")
+    os.remove("logs.zip")
 
 
 if __name__ == "__main__":

.github/workflows/ci-restart-failed-daily-conda-builds.yml

@@ -5,12 +5,9 @@ on:
     workflows: [ ci-ifcopenshell-conda-daily ]
     types:
       - completed
-  push: # Remove this before merging
-    branches:
-    - add-conda-daily-restart-workflow
 
 env:
-  REPO_OWNER: IfcOpenShell
+  REPO_OWNER: IfcOpenShell/IfcOpenShell
   MY_WORKFLOW: ci-ifcopenshell-conda-daily
 
 jobs:
@@ -18,10 +15,20 @@ jobs:
     runs-on: ubuntu-latest
     if: ${{ github.event.workflow_run.conclusion == 'failure' }}
     steps:
-      - uses: actions/setup-python@v2
+      - name: checkout
+        uses: actions/checkout@v3
+      - name: Use python 3.11
+        uses: actions/setup-python@v2
         with:
-          python-version: '3.10'
+          python-version: 3.11
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install requests
       - name: Check for failed runs
-        id: check-failed-runs
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }}
         run: |
           python check_repo_ci_jobs.py
+        working-directory: .github/workflows