From c08cbd0067df543b16a984dbf52ab63be71c992c Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 18 Nov 2025 20:24:40 +0100 Subject: [PATCH] allow building a "fat sysimage" with all the stdlibs in it --- base/sysimg_stdlibs.jl | 59 ++++++++++++++++++++++++++++++++++++++++++ pkgimage.mk | 17 ++++++++++++ sysimage.mk | 58 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 134 insertions(+) create mode 100644 base/sysimg_stdlibs.jl diff --git a/base/sysimg_stdlibs.jl b/base/sysimg_stdlibs.jl new file mode 100644 index 0000000000000..c7151a9534b52 --- /dev/null +++ b/base/sysimg_stdlibs.jl @@ -0,0 +1,59 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license + +# Second-stage sysimage builder that loads all stdlibs from precompiled cache files +# This is used when building a "fat" sysimage with all stdlibs included +# This file follows the same structure as sysimg.jl but loads all available stdlibs + +Base.reinit_stdio() +@eval Sys BINDIR = ccall(:jl_get_julia_bindir, Any, ())::String +@eval Sys STDLIB = abspath(Sys.BINDIR, Base.DATAROOTDIR, "julia", "stdlib", string('v', VERSION.major, '.', VERSION.minor)) +@eval Base _atexit_hooks_finished = false + +# Set up depot & load paths to be able to find stdlib packages +Base.init_depot_path() +Base.init_load_path() + +if Base.is_primary_base_module +# Load all stdlib packages (similar to sysimg.jl but loading all stdlibs) +let + # Collect all stdlibs from the stdlib directory + stdlibs = Symbol[] + for entry in readdir(Sys.STDLIB) + stdlib_path = joinpath(Sys.STDLIB, entry) + # Check if it's a directory with a Project.toml (indicates a stdlib) + if isdir(stdlib_path) && isfile(joinpath(stdlib_path, "Project.toml")) + push!(stdlibs, Symbol(entry)) + end + end + + + maxlen = maximum(textwidth.(string.(stdlibs)); init=0) + + m = Core.Module() + GC.@preserve m begin + print_time = @eval m (mod, t) -> (print(rpad(string(mod) * " ", $maxlen + 3, "─")); + Base.time_print(stdout, t * 10^9); println()) + + tot_time_stdlib = @elapsed for stdlib in stdlibs + tt = @elapsed Base.require(Base, stdlib) + print_time(stdlib, tt) + end + + print_time("Stdlibs total", tot_time_stdlib) + end + + # Clear global state + empty!(Core.ARGS) + empty!(Base.ARGS) + empty!(LOAD_PATH) + empty!(DEPOT_PATH) +end + +empty!(Base.TOML_CACHE.d) +Base.TOML.reinit!(Base.TOML_CACHE.p, "") +@eval Base BUILDROOT = "" +@eval Sys begin + BINDIR = "" + STDLIB = "" +end +end diff --git a/pkgimage.mk b/pkgimage.mk index ed5e1095c0229..6d399da7ec42b 100644 --- a/pkgimage.mk +++ b/pkgimage.mk @@ -4,6 +4,9 @@ JULIAHOME := $(SRCDIR) include $(JULIAHOME)/Make.inc include $(JULIAHOME)/stdlib/stdlib.mk +# Import the fat sysimage setting from sysimage.mk +JULIA_BUILD_FAT_SYSIMAGE ?= 0 + DEPOTDIR := $(build_prefix)/share/julia # set some influential environment variables @@ -25,6 +28,19 @@ $(DEPOTDIR)/compiled: print-depot-path: @$(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no -e '@show Base.DEPOT_PATH') +ifeq ($(JULIA_BUILD_FAT_SYSIMAGE),1) +# In fat mode, use sysbase and add stdlib to LOAD_PATH +$(BUILDDIR)/stdlib/release.image: $(build_private_libdir)/sysbase.$(SHLIB_EXT) $(JULIAHOME)/stdlib/Project.toml $(JULIAHOME)/stdlib/Manifest.toml $(INDEPENDENT_STDLIBS_SRCS) $(DEPOTDIR)/compiled + @$(call PRINT_JULIA, JULIA_CPU_TARGET="sysimage" $(call spawn,$(JULIA_EXECUTABLE)) --sysimage=$(build_private_libdir)/sysbase.$(SHLIB_EXT) --startup-file=no -e \ + 'Base.Precompilation.precompilepkgs(configs=[``=>Base.CacheFlags(debug_level=2, opt_level=3), ``=>Base.CacheFlags(check_bounds=1, debug_level=2, opt_level=3)])') + touch $@ + +$(BUILDDIR)/stdlib/debug.image: $(build_private_libdir)/sysbase-debug.$(SHLIB_EXT) $(JULIAHOME)/stdlib/Project.toml $(JULIAHOME)/stdlib/Manifest.toml $(INDEPENDENT_STDLIBS_SRCS) $(DEPOTDIR)/compiled + @$(call PRINT_JULIA, JULIA_CPU_TARGET="sysimage" $(call spawn,$(JULIA_EXECUTABLE)) --sysimage=$(build_private_libdir)/sysbase-debug.$(SHLIB_EXT) --startup-file=no -e \ + 'Base.Precompilation.precompilepkgs(configs=[``=>Base.CacheFlags(debug_level=2, opt_level=3), ``=>Base.CacheFlags(check_bounds=1, debug_level=2, opt_level=3)])') + touch $@ +else +# In normal mode, use default sysimage $(BUILDDIR)/stdlib/%.image: $(JULIAHOME)/stdlib/Project.toml $(JULIAHOME)/stdlib/Manifest.toml $(INDEPENDENT_STDLIBS_SRCS) $(DEPOTDIR)/compiled @$(call PRINT_JULIA, JULIA_CPU_TARGET="sysimage" $(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no -e \ 'Base.Precompilation.precompilepkgs(configs=[``=>Base.CacheFlags(debug_level=2, opt_level=3), ``=>Base.CacheFlags(check_bounds=1, debug_level=2, opt_level=3)])') @@ -32,6 +48,7 @@ $(BUILDDIR)/stdlib/%.image: $(JULIAHOME)/stdlib/Project.toml $(JULIAHOME)/stdlib $(BUILDDIR)/stdlib/release.image: $(build_private_libdir)/sys.$(SHLIB_EXT) $(BUILDDIR)/stdlib/debug.image: $(build_private_libdir)/sys-debug.$(SHLIB_EXT) +endif clean: rm -rf $(DEPOTDIR)/compiled diff --git a/sysimage.mk b/sysimage.mk index e7917875e0ef2..4b301d6e09785 100644 --- a/sysimage.mk +++ b/sysimage.mk @@ -4,6 +4,17 @@ JULIAHOME := $(SRCDIR) include $(JULIAHOME)/Make.inc include $(JULIAHOME)/stdlib/stdlib.mk +# Allow building a "fat" sysimage with all stdlibs included +# When enabled, this creates a two-stage build: +# 1. Build the normal sysimage (with default stdlibs) +# 2. Precompile all remaining stdlibs using the normal sysimage +# 3. Build the final fat sysimage by loading all precompiled stdlibs on top +# This allows stdlibs to run their __init__() during precompilation, +# which is necessary for stdlibs with complex precompile workloads. +# Enable by setting JULIA_BUILD_FAT_SYSIMAGE=1 in Make.user or environment +# TODO: Default to false +JULIA_BUILD_FAT_SYSIMAGE ?= 1 + default: sysimg-$(JULIA_BUILD_MODE) # contains either "debug" or "release" all: sysimg-release sysimg-debug basecompiler-ji: $(build_private_libdir)/basecompiler.ji @@ -147,7 +158,54 @@ $$(build_private_libdir)/sys$1-o.a $$(build_private_libdir)/sys$1-bc.a : $$(buil .SECONDARY: $$(build_private_libdir)/sys$1-o.a $(build_private_libdir)/sys$1-bc.a # request Make to keep these files around .SECONDARY: $$(build_private_libdir)/sysbase$1-o.a $(build_private_libdir)/sysbase$1-bc.a # request Make to keep these files around endef + +# Fat sysimage builder - builds on top of normal sysimage with all stdlibs +# Parameters: $1=suffix (e.g., -debug), $2=image name (release/debug), $3=opt flags (e.g., -O3), $4=executable +define sysimg_builder_fat +build_sysbase_$1 := $$(or $$(CROSS_BOOTSTRAP_SYSBASE),$$(build_private_libdir)/sysbase$1.$$(SHLIB_EXT)) +$$(build_private_libdir)/sys$1-o.a : $$(build_sysbase_$1) $$(BUILDROOT)/stdlib/$2.image $$(JULIAHOME)/base/sysimg_stdlibs.jl + @$$(call PRINT_JULIA, cd $$(JULIAHOME)/base && \ + if ! JULIA_BINDIR=$$(call cygpath_w,$$(build_bindir)) \ + WINEPATH="$$(call cygpath_w,$$(build_bindir));$$$$WINEPATH" \ + JULIA_LOAD_PATH='@stdlib' \ + JULIA_PROJECT= \ + JULIA_DEPOT_PATH=$$(call cygpath_w,$$(build_prefix)/share/julia) \ + JULIA_NUM_THREADS=1 \ + $$(call spawn, $4) $3 -C "$$(JULIA_CPU_TARGET)" $$(HEAPLIM) --output-o $$(call cygpath_w,$$@).tmp $$(JULIA_SYSIMG_BUILD_FLAGS) \ + --startup-file=no --warn-overwrite=yes --depwarn=error --sysimage $$(call cygpath_w,$$<) sysimg_stdlibs.jl; then \ + echo '*** This error is usually fixed by running `make clean`. If the error persists$$(COMMA) try `make cleanall`. ***'; \ + false; \ + fi ) + @mv $$@.tmp $$@ +.SECONDARY: $$(build_private_libdir)/sys$1-o.a # request Make to keep these files around +endef + +ifeq ($(JULIA_BUILD_FAT_SYSIMAGE),1) +# For fat sysimage, build normal sysbase first, then precompile all stdlibs, then build fat sys +# Stage 1: Build normal sysbase (with default stdlibs) +# Stage 2: Precompile all stdlibs (handled by pkgimage.mk) +# Stage 3: Build final fat sys with all precompiled stdlibs + +# Delegate to pkgimage.mk for building the precompilation stamps +$(BUILDROOT)/stdlib/release.image: $(build_private_libdir)/sysbase.$(SHLIB_EXT) + @$(MAKE) -C $(BUILDROOT) -f $(JULIAHOME)/pkgimage.mk release JULIA_BUILD_FAT_SYSIMAGE=$(JULIA_BUILD_FAT_SYSIMAGE) + +$(BUILDROOT)/stdlib/debug.image: $(build_private_libdir)/sysbase-debug.$(SHLIB_EXT) + @$(MAKE) -C $(BUILDROOT) -f $(JULIAHOME)/pkgimage.mk debug JULIA_BUILD_FAT_SYSIMAGE=$(JULIA_BUILD_FAT_SYSIMAGE) + +$(eval $(call base_builder,,-O1,$(JULIA_EXECUTABLE_release))) +$(eval $(call base_builder,-debug,-O0,$(JULIA_EXECUTABLE_debug))) +$(eval $(call sysimg_builder,,-O3,$(JULIA_EXECUTABLE_release))) +$(eval $(call sysimg_builder,-debug,-O0,$(JULIA_EXECUTABLE_debug))) +$(eval $(call sysimg_builder_fat,,release,-O3,$(JULIA_EXECUTABLE_release))) +$(eval $(call sysimg_builder_fat,-debug,debug,-O0,$(JULIA_EXECUTABLE_debug))) + +# In fat mode, the normal sysimg targets build the fat sysimage +# The dependency on stdlib/.image is implicit through sys-o.a dependencies + +else $(eval $(call base_builder,,-O1,$(JULIA_EXECUTABLE_release))) $(eval $(call base_builder,-debug,-O0,$(JULIA_EXECUTABLE_debug))) $(eval $(call sysimg_builder,,-O3,$(JULIA_EXECUTABLE_release))) $(eval $(call sysimg_builder,-debug,-O0,$(JULIA_EXECUTABLE_debug))) +endif