Increase model size to 500MB for cache models (#1837)

* print csche size

* update script

* trigger ci

* trigger tests

* fix

* trigger

* fix

* set 500mb limit

* remove weekly schedule

Author: echarlaix

.github/workflows/create_model_cache.yml

@@ -1,8 +1,6 @@
 name: Create Model Cache
 
 on:
-  schedule:
-    - cron: "0 0 * * 1"  # every monday at midnight UTC
   workflow_dispatch:
     inputs:
       force_update:
@@ -20,7 +18,7 @@ env:
 
 jobs:
   create-cache:
-    if: github.event_name == 'workflow_dispatch' || github.event_name == 'schedule'
+    if: github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-22.04
 
     steps:
@@ -38,6 +36,7 @@ jobs:
           uv pip install .[tests]
 
       - name: Restore existing cache
+        if: ${{ !inputs.force_update }}
         uses: actions/cache/restore@v4
         with:
           path: ~/.cache/huggingface/hub

tests/scripts/download_cached_models.py

@@ -25,8 +25,8 @@
 
 # github actions repo cache limit is 10 GB so using 9 GB
 CACHE_LIMIT = 9 * 10**9
-# exclude model larger than 200 MB
-MODEL_SIZE_THRESHOLD = 200 * 10**6
+# exclude model larger than 500 MB
+MODEL_SIZE_THRESHOLD = 500 * 10**6
 
 
 def hub_size(api: HfApi, repo_id: str) -> int:
@@ -41,7 +41,6 @@ def hub_size(api: HfApi, repo_id: str) -> int:
 def main() -> None:
     api = HfApi()
     candidates = {}
-
     for name, repo_id in MODEL_NAMES.items():
         if not isinstance(repo_id, str) or not repo_id or repo_id.startswith("/"):
             continue
@@ -89,6 +88,25 @@ def main() -> None:
     print(f"Skipped (too large) : {len(skipped_too_large)}  ({', '.join(skipped_too_large)})")
     print(f"Skipped (cache full): {len(skipped_cache_full)}  ({', '.join(skipped_cache_full)})")
     print(f"Loading failed      : {len(failed)}  ({', '.join(failed)})")
+    final_cache_info = scan_cache_dir()
+    actual_size = final_cache_info.size_on_disk / 10**9
+    print(f"Actual cache        : {actual_size:.2f} GB, sum_repo_size {sum_repo_size / 10**9:.0f} GB")
+
+    final_cached_repos = {repo.repo_id for repo in final_cache_info.repos}
+    name_of = {repo_id: name for name, repo_id in MODEL_NAMES.items()}
+    downloaded_repo_ids = {repo_id for name, repo_id in MODEL_NAMES.items() if name in set(downloaded)}
+
+    in_cache_not_downloaded = final_cached_repos - downloaded_repo_ids
+    in_downloaded_not_in_cache = downloaded_repo_ids - final_cached_repos
+
+    print(
+        f"In cache but not in downloaded ({len(in_cache_not_downloaded)}): "
+        f"{', '.join(name_of.get(r, r) for r in sorted(in_cache_not_downloaded))}"
+    )
+    print(
+        f"In downloaded but not in cache ({len(in_downloaded_not_in_cache)}): "
+        f"{', '.join(name_of.get(r, r) for r in sorted(in_downloaded_not_in_cache))}"
+    )
 
 
 if __name__ == "__main__":