From b155e2a5a0b019c945b1b2d47bd25c6027ab8011 Mon Sep 17 00:00:00 2001 From: Thorsten Klein Date: Sat, 22 Nov 2025 16:29:11 +0100 Subject: [PATCH 1/4] scripts: west_commands: format blobs.py blobs.py is formatted with `ruff format` Signed-off-by: Thorsten Klein --- scripts/west_commands/blobs.py | 64 ++++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 22 deletions(-) diff --git a/scripts/west_commands/blobs.py b/scripts/west_commands/blobs.py index d65bd01fb8383..8b2dbb26697ac 100644 --- a/scripts/west_commands/blobs.py +++ b/scripts/west_commands/blobs.py @@ -18,7 +18,6 @@ class Blobs(WestCommand): - DEFAULT_LIST_FMT = '{module} {status} {path} {type} {abspath}' def __init__(self): @@ -27,7 +26,8 @@ def __init__(self): # Keep this in sync with the string in west-commands.yml. 'work with binary blobs', 'Work with binary blobs', - accepts_unknown_args=False) + accepts_unknown_args=False, + ) def do_add_parser(self, parser_adder): parser = parser_adder.add_parser( @@ -61,22 +61,30 @@ def do_add_parser(self, parser_adder): - uri: URI to the remote location of the blob - description: blob text description - doc-url: URL to the documentation for this blob - ''')) + '''), + ) # Remember to update west-completion.bash if you add or remove # flags - parser.add_argument('subcmd', nargs=1, - choices=['list', 'fetch', 'clean'], - help='sub-command to execute') + parser.add_argument( + 'subcmd', nargs=1, choices=['list', 'fetch', 'clean'], help='sub-command to execute' + ) - parser.add_argument('modules', metavar='MODULE', nargs='*', - help='''zephyr modules to operate on; - all modules will be used if not given''') + parser.add_argument( + 'modules', + metavar='MODULE', + nargs='*', + help='''zephyr modules to operate on; + all modules will be used if not given''', + ) group = parser.add_argument_group('west blob list options') - group.add_argument('-f', '--format', - help='''format string to use to list each blob; - see FORMAT STRINGS below''') + group.add_argument( + '-f', + '--format', + help='''format string to use to list each blob; + see FORMAT STRINGS below''', + ) group = parser.add_argument_group('west blobs fetch options') group.add_argument( @@ -86,8 +94,12 @@ def do_add_parser(self, parser_adder): Only local paths matching this regex will be fetched. Note that local paths are relative to the module directory''', ) - group.add_argument('-a', '--auto-accept', action='store_true', - help='''auto accept license if the fetching needs click-through''') + group.add_argument( + '-a', + '--auto-accept', + action='store_true', + help='''auto accept license if the fetching needs click-through''', + ) return parser @@ -125,6 +137,7 @@ def fetch_blob(self, url, path): scheme = urlparse(url).scheme self.dbg(f'Fetching {path} with {scheme}') import fetchers + fetcher = fetchers.get_fetcher_cls(scheme) self.dbg(f'Found fetcher: {fetcher}') @@ -139,8 +152,9 @@ def verify_blob(self, blob) -> bool: status = zephyr_module.get_blob_status(blob['abspath'], blob['sha256']) if status == zephyr_module.BLOB_OUTDATED: - self.err(textwrap.dedent( - f'''\ + self.err( + textwrap.dedent( + f'''\ The checksum of the downloaded file does not match that in the blob metadata: - if it is not certain that the download was successful, @@ -153,7 +167,9 @@ def verify_blob(self, blob) -> bool: Module: {blob['module']} Blob: {blob['path']} URL: {blob['url']} - Info: {blob['description']}''')) + Info: {blob['description']}''' + ) + ) return False return True @@ -176,9 +192,11 @@ def fetch(self, args): if blob['click-through'] and not args.auto_accept: while True: - user_input = input("For this blob, need to read and accept " - "license to continue. Read it?\n" - "Please type 'y' or 'n' and press enter to confirm: ") + user_input = input( + "For this blob, need to read and accept " + "license to continue. Read it?\n" + "Please type 'y' or 'n' and press enter to confirm: " + ) if user_input.upper() == "Y" or user_input.upper() == "N": break @@ -191,8 +209,10 @@ def fetch(self, args): print(license_content) while True: - user_input = input("Accept license to continue?\n" - "Please type 'y' or 'n' and press enter to confirm: ") + user_input = input( + "Accept license to continue?\n" + "Please type 'y' or 'n' and press enter to confirm: " + ) if user_input.upper() == "Y" or user_input.upper() == "N": break From 237268d138ccec802e744b42c7bcf7bcb2550a5b Mon Sep 17 00:00:00 2001 From: Thorsten Klein Date: Sat, 22 Nov 2025 16:59:40 +0100 Subject: [PATCH 2/4] .ruff-excludes: remove scripts/west_commands/blobs.py Regarding documentation, when changing an excluded file, contributors are encouraged to remove it from the list and format it in a separate commit. Signed-off-by: Thorsten Klein --- .ruff-excludes.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/.ruff-excludes.toml b/.ruff-excludes.toml index 29c43427278a3..9ef22d04cdc7d 100644 --- a/.ruff-excludes.toml +++ b/.ruff-excludes.toml @@ -1284,7 +1284,6 @@ exclude = [ "./scripts/utils/pinctrl_nrf_migrate.py", "./scripts/utils/twister_to_list.py", "./scripts/west_commands/bindesc.py", - "./scripts/west_commands/blobs.py", "./scripts/west_commands/boards.py", "./scripts/west_commands/build.py", "./scripts/west_commands/build_helpers.py", From 6ba90c13b9e0b76d587e5fe41d3defbdfb8b50fe Mon Sep 17 00:00:00 2001 From: Thorsten Klein Date: Sat, 22 Nov 2025 16:38:03 +0100 Subject: [PATCH 3/4] scripts: west_commands: blobs: support blobs cache An auto-cache directory can be provided via the `west blobs fetch --auto-cache` argument or the `blobs.auto-cache-dir` config option. When enabled, the auto-cache is automatically filled whenever a blob is missing and must be downloaded. One or more additional cache directories can be specified via argument `west blobs fetch --cache-dirs` or the `blobs.cache-dir` config option (multiple paths separated by `;`). `west blobs fetch` searches all configured cache directories (including the auto-cache) for a matching blob filename. Cached files may be stored either under their original filename or with a SHA-256 suffix (`.`). If found, the blob is copied from the cache to the blob path; otherwise it is downloaded from its url to the blob path. Signed-off-by: Thorsten Klein --- scripts/west_commands/blobs.py | 105 +++++++++++++++++++++++++++++++-- 1 file changed, 101 insertions(+), 4 deletions(-) diff --git a/scripts/west_commands/blobs.py b/scripts/west_commands/blobs.py index 8b2dbb26697ac..81eb1cc11c35d 100644 --- a/scripts/west_commands/blobs.py +++ b/scripts/west_commands/blobs.py @@ -5,6 +5,7 @@ import argparse import os import re +import shutil import sys import textwrap from pathlib import Path @@ -100,6 +101,18 @@ def do_add_parser(self, parser_adder): action='store_true', help='''auto accept license if the fetching needs click-through''', ) + group.add_argument( + '--cache-dirs', + help='''Semicolon-separated list of directories to search for cached + blobs before downloading. Cache files may use the original + filename or be suffixed with `.`.''', + ) + group.add_argument( + '--auto-cache', + help='''Path to a directory that is automatically populated when a blob + is downloaded. Cached blobs are stored using the original + filename suffixed with `.`.''', + ) return parser @@ -133,18 +146,102 @@ def list(self, args): def ensure_folder(self, path): path.parent.mkdir(parents=True, exist_ok=True) - def fetch_blob(self, url, path): + def handle_auto_cache(self, blob, auto_cache_dir) -> Path: + """ + This function guarantees that a given blob exists in the auto-cache. + It first checks whether the blob is already present. If so, it + returns the path of this cached blob. If the blob is not yet cached, + the blob is downloaded into the auto-cache directory and the path of + the freshly cached blob is returned. + """ + cached_blob = self.get_cached_blob(blob, [auto_cache_dir]) + if cached_blob: + return cached_blob + name = Path(blob['path']).name + sha256 = blob['sha256'] + self.download_blob(blob, auto_cache_dir / f'{name}.{sha256}') + cached_blob = self.get_cached_blob(blob, [auto_cache_dir]) + assert cached_blob, f'Blob {name} still not cached in auto-cache.' + return cached_blob + + def get_cached_blob(self, blob, cache_dirs: list) -> Path | None: + """ + Look for a cached blob in the provided cache directories. + A blob may be stored using either its original name or suffixed with + its SHA256 hash (e.g. "."). + Return the first matching path, or None if not found. + """ + name = Path(blob['path']).name + sha256 = blob["sha256"] + candidate_names = [ + f"{name}.{sha256}", # suffixed version + name, # original blob name + ] + + for cache_dir in cache_dirs: + if not cache_dir.exists(): + continue + for name in candidate_names: + candidate_path = cache_dir / name + if ( + zephyr_module.get_blob_status(candidate_path, sha256) + == zephyr_module.BLOB_PRESENT + ): + return candidate_path + return None + + def download_blob(self, blob, path): + '''Download a blob from its url to a given path.''' + url = blob['url'] scheme = urlparse(url).scheme - self.dbg(f'Fetching {path} with {scheme}') + self.dbg(f'Fetching blob from url {url} with {scheme} to path: {path}') import fetchers fetcher = fetchers.get_fetcher_cls(scheme) - self.dbg(f'Found fetcher: {fetcher}') inst = fetcher() self.ensure_folder(path) inst.fetch(url, path) + def fetch_blob(self, args, blob): + """ + Ensures that the specified blob is available at its path. + If caching is enabled and the blob exists in the cache, it is copied + from there. Otherwise, the blob is downloaded from its URL and placed + at the target path. + """ + path = Path(blob['abspath']) + + # collect existing cache dirs specified as args, otherwise from west config + cache_dirs = args.cache_dirs + auto_cache_dir = args.auto_cache + if self.has_config: + if cache_dirs is None: + cache_dirs = self.config.get('blobs.cache-dirs') + if auto_cache_dir is None: + auto_cache_dir = self.config.get('blobs.auto-cache') + + # expand user home for each cache directory + if auto_cache_dir is not None: + auto_cache_dir = Path(auto_cache_dir).expanduser() + if cache_dirs is not None: + cache_dirs = [Path(p).expanduser() for p in cache_dirs.split(';') if p] + + # search for cached blob in the cache directories + cached_blob = self.get_cached_blob(blob, cache_dirs or []) + + # If blob is not found in cache directories: Use auto-cache if enabled + if not cached_blob and auto_cache_dir: + cached_blob = self.handle_auto_cache(blob, auto_cache_dir) + + # Copy blob if it is cached, otherwise download it + if cached_blob: + self.dbg(f'Copy cached blob: {cached_blob}') + self.ensure_folder(path) + shutil.copy(cached_blob, path) + else: + self.download_blob(blob, path) + # Compare the checksum of a file we've just downloaded # to the digest in blob metadata, warn user if they differ. def verify_blob(self, blob) -> bool: @@ -220,7 +317,7 @@ def fetch(self, args): self.wrn('Skip fetching this blob.') continue - self.fetch_blob(blob['url'], blob['abspath']) + self.fetch_blob(args, blob) if not self.verify_blob(blob): bad_checksum_count += 1 From 5ac5442a988d28f4189b7c6e03ff54f75d7bd473 Mon Sep 17 00:00:00 2001 From: Thorsten Klein Date: Sat, 22 Nov 2025 16:52:31 +0100 Subject: [PATCH 4/4] doc: develop: west: add documentation for west blobs caches Documentation for west blobs `--cache-dirs` and `--auto-cache`, respectively config options `blobs.cache-dirs` and `blobs.auto-cache`. Signed-off-by: Thorsten Klein --- doc/develop/west/zephyr-cmds.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/doc/develop/west/zephyr-cmds.rst b/doc/develop/west/zephyr-cmds.rst index 47e1739051ff7..df211965ec19c 100644 --- a/doc/develop/west/zephyr-cmds.rst +++ b/doc/develop/west/zephyr-cmds.rst @@ -220,6 +220,19 @@ the specific blobs that are fetched, by passing a regular expression:: # For example, only download esp32 blobs, skip the other variants west blobs fetch hal_espressif --allow-regex 'lib/esp32/.*' +An auto-cache directory can be provided via the ``--auto-cache`` cli argument +or via the ``blobs.auto-cache`` config option. When enabled, the auto-cache +directory is automatically populated whenever a blob is missing and downloaded. + +One or more additional cache directories (separated by ``;``) can be provided +in ``--cache-dirs`` cli argument or ``blobs.cache-dirs`` config option. + +``west blobs fetch`` searches all configured cache directories (including the +auto-cache) for a matching blob filename. Cached files may be stored either +under their original filename or with a SHA-256 suffix (``.``). +If found, the blob is copied from the cache to the blob path; otherwise +it is downloaded from its url to the blob path. + .. _west-twister: Twister wrapper: ``west twister``