Retry build if failed

sasha0552 · Aug 7, 2024 · 9c3c012 · 9c3c012
1 parent 07e31ec
commit 9c3c012
Show file tree

Hide file tree

Showing 8 changed files with 280 additions and 16 deletions.
diff --git a/.ci/template/gh-build-iso.yml.jinja2 b/.ci/template/gh-build-iso.yml.jinja2
@@ -54,8 +54,41 @@ jobs:
               .ci/configure.py {{ platform }} {{ type }}
             popd
 
-            # Build image
-            mkarchiso -v -m iso -w /_work -o /workspace/out /workspace
+            # Number of retries
+            MAX_RETRIES=5
+
+            # Try to build image
+            for ((i=1; i<=MAX_RETRIES; i++)); do
+              # Do not exit on error
+              set +e
+
+              # Build image
+              mkarchiso -v -m iso -w /_work -o /workspace/out /workspace
+
+              # Capture exit code
+              STATUS=$?
+
+              # Exit on error
+              set -e
+
+              # Check if build succeeded
+              if [ $STATUS -eq 0 ]; then
+                echo "Build succeeded"
+                break
+              fi
+
+              # Print failure message and retry
+              echo "Build failed. Retry $i/$MAX_RETRIES"
+
+              # Remove build directory
+              rm -rf /_work
+            done
+
+            # Check if maximum retries were exhausted
+            if [ $STATUS -ne 0 ]; then
+              echo "Build failed after $MAX_RETRIES attempts"
+              exit 1
+            fi
 
       - name: Create summary
         run: |

diff --git a/.ci/template/sh-build-iso.yml.jinja2 b/.ci/template/sh-build-iso.yml.jinja2
@@ -72,8 +72,41 @@ jobs:
               .ci/configure.py {{ platform }} {{ type }}
             popd
 
-            # Build image
-            mkarchiso -v -m iso -w /_work -o /workspace/out /workspace
+            # Number of retries
+            MAX_RETRIES=5
+
+            # Try to build image
+            for ((i=1; i<=MAX_RETRIES; i++)); do
+              # Do not exit on error
+              set +e
+
+              # Build image
+              mkarchiso -v -m iso -w /_work -o /workspace/out /workspace
+
+              # Capture exit code
+              STATUS=$?
+
+              # Exit on error
+              set -e
+
+              # Check if build succeeded
+              if [ $STATUS -eq 0 ]; then
+                echo "Build succeeded"
+                break
+              fi
+
+              # Print failure message and retry
+              echo "Build failed. Retry $i/$MAX_RETRIES"
+
+              # Remove build directory
+              rm -rf /_work
+            done
+
+            # Check if maximum retries were exhausted
+            if [ $STATUS -ne 0 ]; then
+              echo "Build failed after $MAX_RETRIES attempts"
+              exit 1
+            fi
 
       - name: Create summary
         run: |

diff --git a/.github/workflows/gh-build-iso-cuda-empty.yml b/.github/workflows/gh-build-iso-cuda-empty.yml
@@ -45,8 +45,41 @@ jobs:
               .ci/configure.py cuda empty
             popd
 
-            # Build image
-            mkarchiso -v -m iso -w /_work -o /workspace/out /workspace
+            # Number of retries
+            MAX_RETRIES=5
+
+            # Try to build image
+            for ((i=1; i<=MAX_RETRIES; i++)); do
+              # Do not exit on error
+              set +e
+
+              # Build image
+              mkarchiso -v -m iso -w /_work -o /workspace/out /workspace
+
+              # Capture exit code
+              STATUS=$?
+
+              # Exit on error
+              set -e
+
+              # Check if build succeeded
+              if [ $STATUS -eq 0 ]; then
+                echo "Build succeeded"
+                break
+              fi
+
+              # Print failure message and retry
+              echo "Build failed. Retry $i/$MAX_RETRIES"
+                
+              # Remove build directory
+              rm -rf /_work
+            done
+
+            # Check if maximum retries were exhausted
+            if [ $STATUS -ne 0 ]; then
+              echo "Build failed after $MAX_RETRIES attempts"
+              exit 1
+            fi
 
       - name: Create summary
         run: |

diff --git a/.github/workflows/gh-build-iso-cuda-gui.yml b/.github/workflows/gh-build-iso-cuda-gui.yml
@@ -47,8 +47,41 @@ jobs:
               .ci/configure.py cuda gui
             popd
 
-            # Build image
-            mkarchiso -v -m iso -w /_work -o /workspace/out /workspace
+            # Number of retries
+            MAX_RETRIES=5
+
+            # Try to build image
+            for ((i=1; i<=MAX_RETRIES; i++)); do
+              # Do not exit on error
+              set +e
+
+              # Build image
+              mkarchiso -v -m iso -w /_work -o /workspace/out /workspace
+
+              # Capture exit code
+              STATUS=$?
+
+              # Exit on error
+              set -e
+
+              # Check if build succeeded
+              if [ $STATUS -eq 0 ]; then
+                echo "Build succeeded"
+                break
+              fi
+
+              # Print failure message and retry
+              echo "Build failed. Retry $i/$MAX_RETRIES"
+                
+              # Remove build directory
+              rm -rf /_work
+            done
+
+            # Check if maximum retries were exhausted
+            if [ $STATUS -ne 0 ]; then
+              echo "Build failed after $MAX_RETRIES attempts"
+              exit 1
+            fi
 
       - name: Create summary
         run: |

diff --git a/.github/workflows/gh-build-iso-cuda.yml b/.github/workflows/gh-build-iso-cuda.yml
@@ -47,8 +47,41 @@ jobs:
               .ci/configure.py cuda normal
             popd
 
-            # Build image
-            mkarchiso -v -m iso -w /_work -o /workspace/out /workspace
+            # Number of retries
+            MAX_RETRIES=5
+
+            # Try to build image
+            for ((i=1; i<=MAX_RETRIES; i++)); do
+              # Do not exit on error
+              set +e
+
+              # Build image
+              mkarchiso -v -m iso -w /_work -o /workspace/out /workspace
+
+              # Capture exit code
+              STATUS=$?
+
+              # Exit on error
+              set -e
+
+              # Check if build succeeded
+              if [ $STATUS -eq 0 ]; then
+                echo "Build succeeded"
+                break
+              fi
+
+              # Print failure message and retry
+              echo "Build failed. Retry $i/$MAX_RETRIES"
+                
+              # Remove build directory
+              rm -rf /_work
+            done
+
+            # Check if maximum retries were exhausted
+            if [ $STATUS -ne 0 ]; then
+              echo "Build failed after $MAX_RETRIES attempts"
+              exit 1
+            fi
 
       - name: Create summary
         run: |

diff --git a/.github/workflows/gh-build-iso-rocm-empty.yml b/.github/workflows/gh-build-iso-rocm-empty.yml
@@ -49,8 +49,41 @@ jobs:
               .ci/configure.py rocm empty
             popd
 
-            # Build image
-            mkarchiso -v -m iso -w /_work -o /workspace/out /workspace
+            # Number of retries
+            MAX_RETRIES=5
+
+            # Try to build image
+            for ((i=1; i<=MAX_RETRIES; i++)); do
+              # Do not exit on error
+              set +e
+
+              # Build image
+              mkarchiso -v -m iso -w /_work -o /workspace/out /workspace
+
+              # Capture exit code
+              STATUS=$?
+
+              # Exit on error
+              set -e
+
+              # Check if build succeeded
+              if [ $STATUS -eq 0 ]; then
+                echo "Build succeeded"
+                break
+              fi
+
+              # Print failure message and retry
+              echo "Build failed. Retry $i/$MAX_RETRIES"
+                
+              # Remove build directory
+              rm -rf /_work
+            done
+
+            # Check if maximum retries were exhausted
+            if [ $STATUS -ne 0 ]; then
+              echo "Build failed after $MAX_RETRIES attempts"
+              exit 1
+            fi
 
       - name: Create summary
         run: |

diff --git a/.github/workflows/sh-build-iso-rocm-gui.yml b/.github/workflows/sh-build-iso-rocm-gui.yml
@@ -66,8 +66,41 @@ jobs:
               .ci/configure.py rocm gui
             popd
 
-            # Build image
-            mkarchiso -v -m iso -w /_work -o /workspace/out /workspace
+            # Number of retries
+            MAX_RETRIES=5
+
+            # Try to build image
+            for ((i=1; i<=MAX_RETRIES; i++)); do
+              # Do not exit on error
+              set +e
+
+              # Build image
+              mkarchiso -v -m iso -w /_work -o /workspace/out /workspace
+
+              # Capture exit code
+              STATUS=$?
+
+              # Exit on error
+              set -e
+
+              # Check if build succeeded
+              if [ $STATUS -eq 0 ]; then
+                echo "Build succeeded"
+                break
+              fi
+
+              # Print failure message and retry
+              echo "Build failed. Retry $i/$MAX_RETRIES"
+                
+              # Remove build directory
+              rm -rf /_work
+            done
+
+            # Check if maximum retries were exhausted
+            if [ $STATUS -ne 0 ]; then
+              echo "Build failed after $MAX_RETRIES attempts"
+              exit 1
+            fi
 
       - name: Create summary
         run: |

diff --git a/.github/workflows/sh-build-iso-rocm.yml b/.github/workflows/sh-build-iso-rocm.yml
@@ -66,8 +66,41 @@ jobs:
               .ci/configure.py rocm normal
             popd
 
-            # Build image
-            mkarchiso -v -m iso -w /_work -o /workspace/out /workspace
+            # Number of retries
+            MAX_RETRIES=5
+
+            # Try to build image
+            for ((i=1; i<=MAX_RETRIES; i++)); do
+              # Do not exit on error
+              set +e
+
+              # Build image
+              mkarchiso -v -m iso -w /_work -o /workspace/out /workspace
+
+              # Capture exit code
+              STATUS=$?
+
+              # Exit on error
+              set -e
+
+              # Check if build succeeded
+              if [ $STATUS -eq 0 ]; then
+                echo "Build succeeded"
+                break
+              fi
+
+              # Print failure message and retry
+              echo "Build failed. Retry $i/$MAX_RETRIES"
+                
+              # Remove build directory
+              rm -rf /_work
+            done
+
+            # Check if maximum retries were exhausted
+            if [ $STATUS -ne 0 ]; then
+              echo "Build failed after $MAX_RETRIES attempts"
+              exit 1
+            fi
 
       - name: Create summary
         run: |