checkpoint commit

automations/updates/main.py

@@ -15,12 +15,17 @@ def main():
         for _, host in enumerate(tqdm(fedora_hosts, desc="Running system updates")):
             log_file.write(f"Updating {host}\n")
             log_file.flush()
-            subprocess.run(
-                ["ssh", host, "dnf", "upgrade", "-y"],
-                stdout=log_file,
-                stderr=log_file,
-                check=True,
-            )
+            try:
+                subprocess.run(
+                    ["ssh", host, "dnf", "upgrade", "-y"],
+                    stdout=log_file,
+                    stderr=log_file,
+                    check=True,
+                )
+            except Exception as e:
+                log_file.write(f"Couldn't connect to {host}. Skipping...\n")
+                continue
+            log_file.flush()
             log_file.write(f"Rebooting {host}\n")
             log_file.flush()
             subprocess.run(
@@ -31,9 +36,9 @@ def main():
             )
             time.sleep(5)  # wait for reboot to take effect
             booted = False
-            max_attempts = 5  # seconds
-            cur_attempts = 0  # seconds
-            while cur_attempts > max_attempts or not booted:
+            max_attempts = 10
+            cur_attempts = 0
+            while max_attempts > cur_attempts and not booted:
                 try:
                     subprocess.run(
                         ["ssh", host, "echo"],
@@ -42,6 +47,8 @@ def main():
                         check=True,
                         timeout=2,
                     )
+                    log_file.write(f"{host} booted!\n")
+                    log_file.flush()
                     booted = True
                 except Exception as e:
                     cur_attempts += 1