Merge branch 'master' into master

2025-11-04 16:35:02 -08:00
parent 1e89af9862 12d995f58c
commit 3a2ae631c4
11 changed files with 45 additions and 36 deletions
@@ -71,8 +71,10 @@ def download_file_with_lock(url, filename, postprocess_fn=None):
        return file_path

    with FileLock(lock_path):
+        # Only a single rank can acquire this lock
+        # All other ranks block until it is released

-        # Recheck after acquiring lock (another process may have downloaded it)
+        # Recheck after acquiring lock
        if os.path.exists(file_path):
            return file_path