diff --git a/.github/workflows/check_repo_ci_jobs.py b/.github/workflows/check_repo_ci_jobs.py index d770ceb41d..85ecd7b38a 100644 --- a/.github/workflows/check_repo_ci_jobs.py +++ b/.github/workflows/check_repo_ci_jobs.py @@ -1,57 +1,103 @@ +import pathlib +import shutil + import requests +import zipfile +import os + +# To test this locally, set these environment variables +REPO_OWNER = os.environ.get("REPO_OWNER", "IfcOpenShell/IfcOpenShell") +MY_WORKFLOW = os.environ.get("MY_WORKFLOW", "ci-ifcopenshell-conda-daily") +WORKFLOW_RUN_ID = os.environ.get("WORKFLOW_RUN_ID", None) +TOKEN = os.environ.get("GITHUB_TOKEN", None) +# Create a fine-grained personal access token for the repo in question with repo permissions "actions:read" +# See https://github.com/settings/tokens?type=beta + +# This is a list of strings that indicate that a job has stopped abruptly. +SIGNS_OF_STOPPAGE = [ + "Error: The operation was canceled.", + "fatal error C1060: compiler is out of heap space", +] -def get_ci_jobs(repo_name): - url = f"https://api.github.com/repos/{repo_name}/actions/runs" - response = requests.get(url) +def start_request_session(): + s = requests.Session() + headers = { + "Accept": "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + } + if TOKEN: + headers["Authorization"] = f"Bearer {TOKEN}" + + s.headers = headers + return s + + +def get_ci_run(repo_name, run_id): + url = f"https://api.github.com/repos/{repo_name}/actions/runs/{run_id}" + s = start_request_session() + response = s.get(url) return response.json() +def evaluate_log_file_for_abrupt_stop(log_file): + with open(log_file) as f: + for i, line in enumerate(f): + for sign in SIGNS_OF_STOPPAGE: + if sign in line: + print(f"Found sign of abrupt stoppage in [line {i}]: '{sign}'") + return True + return False + + def get_ci_specific_run_specific_failure_details(repo_name, run_id): url = f"https://api.github.com/repos/{repo_name}/actions/runs/{run_id}/attempts/1/logs" - response = requests.get(url) - return response.json() + s = start_request_session() + response = s.get(url) + + # SAVE zip file + with open("logs.zip", "wb") as f: + f.write(response.content) + + # unzip file + with zipfile.ZipFile("logs.zip", "r") as zip_ref: + zip_ref.extractall("logs") + + failed_logs = [] + for file in pathlib.Path("logs").iterdir(): + if file.is_dir(): + continue + + if evaluate_log_file_for_abrupt_stop(file): + failed_logs.append(file) + + return failed_logs def restart_job(repo_name, job_id): - url = f"https://api.github.com/repos/{repo_name}/actions/runs/{job_id}/rerun" - response = requests.post(url) + url = f"https://api.github.com/repos/{repo_name}/actions/runs/{job_id}/rerun-failed-jobs" + s = start_request_session() + response = s.post(url) return response.json() -def logs_contain_signs_of_stoppage_due_to_lack_of_vm_resources(logs): - """Do something like this?""" - for log in logs["lines"]: - if log["name"] == "stderr": - if "Error: The operation was canceled." in log["text"]: - return True - if "fatal error C1060: compiler is out of heap space" in log["text"]: - return True - return False - - def eval_jobs(): - jobs = get_ci_jobs("IfcOpenShell/IfcOpenShell") - for job in jobs["workflow_runs"]: - if job["name"] != "ci-ifcopenshell-conda-daily": - continue + run = get_ci_run(REPO_OWNER, WORKFLOW_RUN_ID) + if run["run_attempt"] > 1: + print("This is not the first attempt. Exiting") + return + failed_logs = get_ci_specific_run_specific_failure_details(REPO_OWNER, WORKFLOW_RUN_ID) - if job["status"] != "completed": - continue - if job["conclusion"] != "failure": - continue + if len(failed_logs) == 0: + print("No runs exhibit signs of abrupt stoppage") + return - if job["run_attempt"] > 1: - continue + print("restarting job", WORKFLOW_RUN_ID) + r = restart_job(REPO_OWNER, WORKFLOW_RUN_ID) + print(r) - logs = get_ci_specific_run_specific_failure_details("IfcOpenShell/IfcOpenShell", job["id"]) - - print(logs) - print("restarting job", job["id"]) - - if logs_contain_signs_of_stoppage_due_to_lack_of_vm_resources(logs): - r = restart_job("IfcOpenShell/IfcOpenShell", job["id"]) - print(r) + shutil.rmtree("logs") + os.remove("logs.zip") if __name__ == "__main__": diff --git a/.github/workflows/ci-restart-failed-daily-conda-builds.yml b/.github/workflows/ci-restart-failed-daily-conda-builds.yml index 23814e6240..ba585fb041 100644 --- a/.github/workflows/ci-restart-failed-daily-conda-builds.yml +++ b/.github/workflows/ci-restart-failed-daily-conda-builds.yml @@ -5,12 +5,9 @@ on: workflows: [ ci-ifcopenshell-conda-daily ] types: - completed - push: # Remove this before merging - branches: - - add-conda-daily-restart-workflow env: - REPO_OWNER: IfcOpenShell + REPO_OWNER: IfcOpenShell/IfcOpenShell MY_WORKFLOW: ci-ifcopenshell-conda-daily jobs: @@ -18,10 +15,20 @@ jobs: runs-on: ubuntu-latest if: ${{ github.event.workflow_run.conclusion == 'failure' }} steps: - - uses: actions/setup-python@v2 + - name: checkout + uses: actions/checkout@v3 + - name: Use python 3.11 + uses: actions/setup-python@v2 with: - python-version: '3.10' + python-version: 3.11 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install requests - name: Check for failed runs - id: check-failed-runs + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }} run: | python check_repo_ci_jobs.py + working-directory: .github/workflows