diff --git a/.github/workflows/check_repo_ci_jobs.py b/.github/workflows/check_repo_ci_jobs.py new file mode 100644 index 0000000000..d770ceb41d --- /dev/null +++ b/.github/workflows/check_repo_ci_jobs.py @@ -0,0 +1,58 @@ +import requests + + +def get_ci_jobs(repo_name): + url = f"https://api.github.com/repos/{repo_name}/actions/runs" + response = requests.get(url) + return response.json() + + +def get_ci_specific_run_specific_failure_details(repo_name, run_id): + url = f"https://api.github.com/repos/{repo_name}/actions/runs/{run_id}/attempts/1/logs" + response = requests.get(url) + return response.json() + + +def restart_job(repo_name, job_id): + url = f"https://api.github.com/repos/{repo_name}/actions/runs/{job_id}/rerun" + response = requests.post(url) + return response.json() + + +def logs_contain_signs_of_stoppage_due_to_lack_of_vm_resources(logs): + """Do something like this?""" + for log in logs["lines"]: + if log["name"] == "stderr": + if "Error: The operation was canceled." in log["text"]: + return True + if "fatal error C1060: compiler is out of heap space" in log["text"]: + return True + return False + + +def eval_jobs(): + jobs = get_ci_jobs("IfcOpenShell/IfcOpenShell") + for job in jobs["workflow_runs"]: + if job["name"] != "ci-ifcopenshell-conda-daily": + continue + + if job["status"] != "completed": + continue + if job["conclusion"] != "failure": + continue + + if job["run_attempt"] > 1: + continue + + logs = get_ci_specific_run_specific_failure_details("IfcOpenShell/IfcOpenShell", job["id"]) + + print(logs) + print("restarting job", job["id"]) + + if logs_contain_signs_of_stoppage_due_to_lack_of_vm_resources(logs): + r = restart_job("IfcOpenShell/IfcOpenShell", job["id"]) + print(r) + + +if __name__ == "__main__": + eval_jobs() diff --git a/.github/workflows/ci-restart-failed-daily-conda-builds.yml b/.github/workflows/ci-restart-failed-daily-conda-builds.yml index b89a9c5078..23814e6240 100644 --- a/.github/workflows/ci-restart-failed-daily-conda-builds.yml +++ b/.github/workflows/ci-restart-failed-daily-conda-builds.yml @@ -2,9 +2,12 @@ name: Restart failed daily conda builds on: workflow_run: - workflows: [ci-ifcopenshell-conda-daily] + workflows: [ ci-ifcopenshell-conda-daily ] types: - completed + push: # Remove this before merging + branches: + - add-conda-daily-restart-workflow env: REPO_OWNER: IfcOpenShell @@ -15,19 +18,10 @@ jobs: runs-on: ubuntu-latest if: ${{ github.event.workflow_run.conclusion == 'failure' }} steps: - - name: Check for failed runs - id: check-failed-runs - run: | - echo "Checking for failed runs..." - # Replace "my-workflow" with the name of your workflow - failed_runs=$(curl -s https://api.github.com/repos/${{ env.REPO_OWNER }}/REPO/actions/runs?status=completed&conclusion=failure&workflow_id=${{ env.MY_WORKFLOW }} | jq '. | map(select(.status.code == 143))') - echo "Found $(echo $failed_runs | jq length) failed runs" - - name: Restart failed runs - if: steps.check-failed-runs.outputs.failed_runs != null - run: | - echo "Restarting failed runs..." - for run_id in $(echo $failed_runs | jq -r '.[].id'); do - echo "Restarting run $run_id..." - curl -s -X POST https://api.github.com/repos/${{ env.REPO_OWNER }}/REPO/actions/runs/$run_id/rerun \ - -H "Accept: application/vnd.github+json" - done \ No newline at end of file + - uses: actions/setup-python@v2 + with: + python-version: '3.10' + - name: Check for failed runs + id: check-failed-runs + run: | + python check_repo_ci_jobs.py