feat: add ability to cache json responses in-repo

Besides a performance improvement, it allows cross-platform builds to work (e.g. Linux host building Apple code).

Signed-off-by: Brentley Jones <[email protected]>

Author: brentleyjones

swiftpkg/bzlmod/swift_deps.bzl

@@ -1,6 +1,7 @@
 """Implementation for `swift_deps` bzlmod extension."""
 
 load("@bazel_skylib//lib:dicts.bzl", "dicts")
+load("@bazel_skylib//lib:paths.bzl", "paths")
 load("//swiftpkg/internal:bazel_repo_names.bzl", "bazel_repo_names")
 load("//swiftpkg/internal:local_swift_package.bzl", "local_swift_package")
 load("//swiftpkg/internal:pkginfos.bzl", "pkginfos")
@@ -59,11 +60,21 @@ def _declare_pkgs_from_package(module_ctx, from_package, config_pkgs, config_swi
     # Get the package info.
     pkg_swift = module_ctx.path(from_package.swift)
     debug_path = module_ctx.path(".")
+    workspace_root = str(pkg_swift.dirname)
+
+    root_cached_json_directory = None
+    if from_package.cached_json_directory:
+        root_cached_json_directory = paths.join(
+            workspace_root,
+            from_package.cached_json_directory,
+        )
+
     pkg_info = pkginfos.get(
         module_ctx,
-        directory = str(pkg_swift.dirname),
+        directory = workspace_root,
         env = env,
         debug_path = str(debug_path),
+        cached_json_directory = root_cached_json_directory,
         resolved_pkg_map = resolved_pkg_map,
         collect_src_info = False,
         registries_directory = registries_directory,
@@ -157,10 +168,18 @@ the Swift package to make it available.\
             processing = to_process
             to_process = []
             for dep in processing:
+                dep_cached_json_directory = None
+
+                if from_package.cached_json_directory:
+                    dep_cached_json_directory = paths.join(
+                        dep.file_system.path,
+                        from_package.cached_json_directory,
+                    )
                 dep_pkg_info = pkginfos.get(
                     module_ctx,
                     directory = dep.file_system.path,
                     debug_path = None,
+                    cached_json_directory = dep_cached_json_directory,
                     resolved_pkg_map = None,
                     collect_src_info = False,
                 )
@@ -185,7 +204,13 @@ the Swift package to make it available.\
             config_pkg = config_pkgs.get(
                 bazel_repo_names.from_identity(dep.identity),
             )
-        _declare_pkg_from_dependency(dep, config_pkg, from_package, config_swift_package)
+        _declare_pkg_from_dependency(
+            dep,
+            config_pkg,
+            from_package,
+            config_swift_package,
+            from_package.cached_json_directory,
+        )
 
     # Add all transitive dependencies to direct_dep_repo_names if `publicly_expose_all_targets` flag is set.
     for dep in all_deps_by_id.values():
@@ -199,7 +224,15 @@ the Swift package to make it available.\
 
     return direct_dep_repo_names
 
-def _declare_pkg_from_dependency(dep, config_pkg, from_package, config_swift_package):
+def _declare_pkg_from_dependency(
+        dep,
+        config_pkg,
+        from_package,
+        config_swift_package,
+        cached_json_directory):
+    if cached_json_directory:
+        cached_json_directory = paths.join(cached_json_directory, dep.name)
+
     name = bazel_repo_names.from_identity(dep.identity)
     if dep.source_control:
         init_submodules = None
@@ -236,7 +269,7 @@ def _declare_pkg_from_dependency(dep, config_pkg, from_package, config_swift_pac
             dependencies_index = None,
             env = from_package.env,
             env_inherit = from_package.env_inherit,
-            init_submodules = init_submodules,
+            cached_json_directory = cached_json_directory,            init_submodules = init_submodules,
             recursive_init_submodules = recursive_init_submodules,
             patch_args = patch_args,
             patch_cmds = patch_cmds,
@@ -256,6 +289,7 @@ def _declare_pkg_from_dependency(dep, config_pkg, from_package, config_swift_pac
             env_inherit = from_package.env_inherit,
             path = dep.file_system.path,
             dependencies_index = None,
+            cached_json_directory = cached_json_directory,
         )
 
     elif dep.registry:
@@ -350,6 +384,7 @@ bazel run @swift_package//:resolve
 ```
 """,
             ),
+            "cached_json_directory": attr.string(),
             "env": attr.string_dict(
                 doc = """\
 Environment variables that will be passed to the execution environments for \

swiftpkg/internal/local_swift_package.bzl

@@ -62,7 +62,19 @@ def _local_swift_package_impl(repository_ctx):
     repo_rules.write_workspace_file(repository_ctx, repo_dir)
 
     # Generate the build file
-    pkg_ctx = pkg_ctxs.read(repository_ctx, repo_dir, env)
+    cached_json_directory = None
+    if repository_ctx.attr.cached_json_directory:
+        cached_json_directory = paths.join(
+            str(repository_ctx.workspace_root),
+            repository_ctx.attr.cached_json_directory,
+        )
+
+    pkg_ctx = pkg_ctxs.read(
+        repository_ctx,
+        repo_dir,
+        env,
+        cached_json_directory,
+    )
     repo_rules.gen_build_files(repository_ctx, pkg_ctx)
 
     return update_attrs(repository_ctx.attr, _ALL_ATTRS.keys(), {})
@@ -78,6 +90,7 @@ _ALL_ATTRS = dicts.add(
     repo_rules.env_attrs,
     repo_rules.swift_attrs,
     _PATH_ATTRS,
+    {"cached_json_directory": attr.string()},
 )
 
 local_swift_package = repository_rule(

swiftpkg/internal/pkg_ctxs.bzl

@@ -7,13 +7,15 @@ def _read(
         repository_ctx,
         repo_dir,
         env,
+        cached_json_directory,
         resolved_pkg_map = None,
         registries_directory = None,
         replace_scm_with_registry = False):
     pkg_info = pkginfos.get(
         repository_ctx = repository_ctx,
         directory = repo_dir,
         env = env,
+        cached_json_directory = cached_json_directory,
         resolved_pkg_map = resolved_pkg_map,
         registries_directory = registries_directory,
         replace_scm_with_registry = replace_scm_with_registry,

swiftpkg/internal/pkginfos.bzl

@@ -28,6 +28,7 @@ def _get_dump_manifest(
         env = {},
         working_directory = "",
         debug_path = None,
+        cache_path = None,
         registries_directory = None,
         replace_scm_with_registry = False):
     """Returns a dict representing the package dump for an SPM package.
@@ -52,6 +53,10 @@ def _get_dump_manifest(
         debug_path = str(repository_ctx.path("."))
     debug_json_path = paths.join(debug_path, "dump.json")
 
+    cached_json_path = None
+    if cache_path:
+        cached_json_path = paths.join(cache_path, "dump.json")
+
     args = ["swift", "package"]
 
     if registries_directory:
@@ -66,13 +71,15 @@ def _get_dump_manifest(
         env = env,
         working_directory = working_directory,
         debug_json_path = debug_json_path,
+        cached_json_path = cached_json_path,
     )
 
 def _get_desc_manifest(
         repository_ctx,
         env = {},
         working_directory = "",
         debug_path = None,
+        cache_path = None,
         registries_directory = None,
         replace_scm_with_registry = False):
     """Returns a dict representing the package description for an SPM package.
@@ -97,6 +104,10 @@ def _get_desc_manifest(
         debug_path = str(repository_ctx.path("."))
     debug_json_path = paths.join(debug_path, "desc.json")
 
+    cached_json_path = None
+    if cache_path:
+        cached_json_path = paths.join(cache_path, "desc.json")
+
     args = ["swift", "package"]
 
     if registries_directory:
@@ -112,13 +123,15 @@ def _get_desc_manifest(
         env = env,
         working_directory = working_directory,
         debug_json_path = debug_json_path,
+        cached_json_path = cached_json_path,
     )
 
 def _get(
         repository_ctx,
         directory,
         env = {},
         debug_path = None,
+        cached_json_directory = None,
         resolved_pkg_map = None,
         collect_src_info = True,
         registries_directory = None,
@@ -150,11 +163,13 @@ def _get(
         if not paths.is_absolute(debug_path):
             # For backwards compatibility, resolve relative to the working directory.
             debug_path = paths.join(directory, debug_path)
+
     dump_manifest = _get_dump_manifest(
         repository_ctx,
         env = env,
         working_directory = directory,
         debug_path = debug_path,
+        cache_path = cached_json_directory,
         registries_directory = registries_directory,
         replace_scm_with_registry = replace_scm_with_registry,
     )
@@ -163,6 +178,7 @@ def _get(
         env = env,
         working_directory = directory,
         debug_path = debug_path,
+        cache_path = cached_json_directory,
         registries_directory = registries_directory,
         replace_scm_with_registry = replace_scm_with_registry,
     )
@@ -1648,7 +1664,9 @@ def _new_resource_rule_process(localization = None):
 
 def _new_resource_from_desc_map(desc_map, pkg_path):
     path = desc_map["path"]
-    if paths.is_absolute(path):
+    if path.startswith("./"):
+        path = path[2:]
+    elif paths.is_absolute(path):
         path = paths.relativize(path, pkg_path)
     return _new_resource(
         path = path,

swiftpkg/internal/registry_swift_package.bzl

@@ -235,10 +235,18 @@ def _registry_swift_package_impl(repository_ctx):
     else:
         resolved_pkg_map = dict()
 
+    cached_json_directory = None
+    if repository_ctx.attr.cached_json_directory:
+        cached_json_directory = paths.join(
+            str(repository_ctx.workspace_root),
+            repository_ctx.attr.cached_json_directory,
+        )
+
     pkg_ctx = pkg_ctxs.read(
         repository_ctx,
         directory,
         env,
+        cached_json_directory,
         resolved_pkg_map = resolved_pkg_map,
         registries_directory = registries_directory,
         replace_scm_with_registry = attr.replace_scm_with_registry,

swiftpkg/internal/repository_utils.bzl

@@ -78,17 +78,33 @@ def _parsed_json_from_spm_command(
         arguments,
         env = {},
         working_directory = "",
-        debug_json_path = None):
+        debug_json_path = None,
+        cached_json_path = None):
+    if cached_json_path:
+        cached_json_path_path = repository_ctx.path(cached_json_path)
+        if cached_json_path_path.exists:
+            return json.decode(repository_ctx.read(cached_json_path_path))
+
     json_str = repository_utils.exec_spm_command(
         repository_ctx,
         arguments,
         env = env,
         working_directory = working_directory,
-    )
+    ).replace(working_directory, ".")
+
     if debug_json_path:
         if not paths.is_absolute(debug_json_path):
             debug_json_path = paths.join(working_directory, debug_json_path)
         repository_ctx.file(debug_json_path, content = json_str, executable = False)
+    if cached_json_path:
+        if debug_json_path:
+            tmp_path = debug_json_path
+        else:
+            tmp_path = paths.join(str(repository_ctx.path(".")), paths.basename(cached_json_path))
+            repository_ctx.file(tmp_path, content = json_str, executable = False)
+        repository_ctx.execute(["mkdir", "-p", paths.dirname(cached_json_path)])
+        repository_ctx.execute(["cp", tmp_path, cached_json_path])
+
     return json.decode(json_str)
 
 def _package_name(repository_ctx):

swiftpkg/internal/swift_package.bzl

@@ -1,6 +1,7 @@
 """Implementation for `swift_package`."""
 
 load("@bazel_skylib//lib:dicts.bzl", "dicts")
+load("@bazel_skylib//lib:paths.bzl", "paths")
 load("@bazel_tools//tools/build_defs/repo:git_worker.bzl", "git_repo")
 load(
     "@bazel_tools//tools/build_defs/repo:utils.bzl",
@@ -68,10 +69,18 @@ def _swift_package_impl(repository_ctx):
         registries_directory = None
 
     # Generate the build file
+    cached_json_directory = None
+    if repository_ctx.attr.cached_json_directory:
+        cached_json_directory = paths.join(
+            str(repository_ctx.workspace_root),
+            repository_ctx.attr.cached_json_directory,
+        )
+
     pkg_ctx = pkg_ctxs.read(
         repository_ctx,
         directory,
         env,
+        cached_json_directory,
         registries_directory = registries_directory,
         replace_scm_with_registry = replace_scm_with_registry,
     )
@@ -197,7 +206,10 @@ _ALL_ATTRS = dicts.add(
     _GIT_ATTRS,
     repo_rules.env_attrs,
     repo_rules.swift_attrs,
-    {"version": attr.string(doc = "The resolved version of the package.")},
+    {
+        "cached_json_directory": attr.string(),
+        "version": attr.string(doc = "The resolved version of the package."),
+    },
 )
 
 swift_package = repository_rule(