Merge pull request #4037 from JuliaLang/backports-release-1.11

[release-1.11] 1.11 backports

Author: IanButterworth

docs/src/creating-packages.md

@@ -416,7 +416,7 @@ Contour = "d38c429a-6771-53c6-b99e-75d170b6e991"
 # name of extension to the left
 # extension dependencies required to load the extension to the right
 # use a list for multiple extension dependencies
-PlottingContourExt = "Contour"
+ContourExt = "Contour"
 
 [compat]
 Contour = "0.6.2"
@@ -433,9 +433,9 @@ end
 end # module
 ```
 
-`ext/PlottingContourExt.jl` (can also be in `ext/PlottingContourExt/PlottingContourExt.jl`):
+`ext/ContourExt.jl` (can also be in `ext/ContourExt/ContourExt.jl`):
 ```julia
-module PlottingContourExt # Should be same name as the file (just like a normal package)
+module ContourExt # Should be same name as the file (just like a normal package)
 
 using Plotting, Contour
 
@@ -446,8 +446,8 @@ end
 end # module
 ```
 
-Extensions can have any arbitrary name (here `PlottingContourExt`), but using something similar to the format of
-this example that makes the extended functionality and dependency of the extension clear is likely a good idea.
+Extensions can have arbitrary names (here `ContourExt`), following the format of this example is likely a good idea for extensions with a single dependency.
+In `Pkg` output, extension names are always shown together with their parent package name.
 
 !!! compat
     Often you will put the extension dependencies into the `test` target so they are loaded when running e.g. `Pkg.test()`. On earlier Julia versions
@@ -461,8 +461,8 @@ this example that makes the extended functionality and dependency of the extensi
 
 ### Behavior of extensions
 
-A user that depends only on `Plotting` will not pay the cost of the "extension" inside the `PlottingContourExt` module.
-It is only when the `Contour` package actually gets loaded that the `PlottingContourExt` extension is loaded too
+A user that depends only on `Plotting` will not pay the cost of the "extension" inside the `ContourExt` module.
+It is only when the `Contour` package actually gets loaded that the `ContourExt` extension is loaded too
 and provides the new functionality.
 
 In our example, the new functionality is an additional _method_, which we add to an existing _function_ from the parent package `Plotting`.
@@ -474,17 +474,17 @@ function plot end
 ```
 
 !!! note
-    If one considers `PlottingContourExt` as a completely separate package, it could be argued that defining `Plotting.plot(c::Contour.ContourCollection)` is
-    [type piracy](https://docs.julialang.org/en/v1/manual/style-guide/#Avoid-type-piracy) since `PlottingContourExt` _owns_ neither the function `Plotting.plot` nor the type `Contour.ContourCollection`.
+    If one considers `ContourExt` as a completely separate package, it could be argued that defining `Plotting.plot(c::Contour.ContourCollection)` is
+    [type piracy](https://docs.julialang.org/en/v1/manual/style-guide/#Avoid-type-piracy) since `ContourExt` _owns_ neither the function `Plotting.plot` nor the type `Contour.ContourCollection`.
     However, for extensions, it is ok to assume that the extension owns the functions in its parent package.
 
 In other situations, one may need to define new symbols in the extension (types, structs, functions, etc.) instead of reusing those from the parent package.
-Such symbols are created in a separate module corresponding to the extension, namely `PlottingContourExt`, and thus not in `Plotting` itself.
+Such symbols are created in a separate module corresponding to the extension, namely `ContourExt`, and thus not in `Plotting` itself.
 If extension symbols are needed in the parent package, one must call `Base.get_extension` to retrieve them.
-Here is an example showing how a custom type defined in `PlottingContourExt` can be accessed in `Plotting`:
+Here is an example showing how a custom type defined in `ContourExt` can be accessed in `Plotting`:
 
 ```julia
-ext = Base.get_extension(@__MODULE__, :PlottingContourExt)
+ext = Base.get_extension(@__MODULE__, :ContourExt)
 if !isnothing(ext)
     ContourPlotType = ext.ContourPlotType
 end
@@ -512,7 +512,7 @@ This is done by making the following changes (using the example above):
 
   @static if !isdefined(Base, :get_extension)
   function __init__()
-      @require Contour = "d38c429a-6771-53c6-b99e-75d170b6e991" include("../ext/PlottingContourExt.jl")
+      @require Contour = "d38c429a-6771-53c6-b99e-75d170b6e991" include("../ext/ContourExt.jl")
   end
   end
   ```
@@ -526,11 +526,11 @@ This is done by making the following changes (using the example above):
       # Other init functionality here
 
       @static if !isdefined(Base, :get_extension)
-          @require Contour = "d38c429a-6771-53c6-b99e-75d170b6e991" include("../ext/PlottingContourExt.jl")
+          @require Contour = "d38c429a-6771-53c6-b99e-75d170b6e991" include("../ext/ContourExt.jl")
       end
   end
   ```
-- Make the following change in the conditionally-loaded code:
+- Make the following change in the conditionally-loaded code in `ContourExt.jl`:
   ```julia
   isdefined(Base, :get_extension) ? (using Contour) : (using ..Contour)
   ```
@@ -549,7 +549,7 @@ This is done by making the following changes (using the example above):
 - Add the following to your main package file (typically at the bottom):
   ```julia
   if !isdefined(Base, :get_extension)
-    include("../ext/PlottingContourExt.jl")
+    include("../ext/ContourExt.jl")
   end
   ```
 

docs/src/getting-started.md

@@ -154,7 +154,7 @@ We can see that the `tutorial` environment now contains `Example` and `JSON`.
     If you have the same
     package (at the same version) installed in multiple environments, the package
     will only be downloaded and stored on the hard drive once. This makes environments
-    very lightweight and effectively free to create. Only using the default
+    very lightweight and effectively free to create. Using only the default
     environment with a huge number of packages in it is a common beginners mistake in
     Julia. Learning how to use environments effectively will improve your experience with
     Julia packages.

ext/REPLExt/REPLExt.jl

@@ -7,7 +7,7 @@ import .REPL: LineEdit, REPLCompletions, TerminalMenus
 
 import Pkg
 import .Pkg: linewrap, pathrepr, compat, can_fancyprint, printpkgstyle, PKGMODE_PROJECT
-using .Pkg: Types, Operations, API, Registry, Resolve, REPLMode
+using .Pkg: Types, Operations, API, Registry, Resolve, REPLMode, safe_realpath
 
 using .REPLMode: Statement, CommandSpec, Command, prepare_cmd, tokenize, core_parse, SPECS, api_options, parse_option, api_options, is_opt, wrap_option
 
@@ -47,7 +47,7 @@ function projname(project_file::String)
     end
     for depot in Base.DEPOT_PATH
         envdir = joinpath(depot, "environments")
-        if startswith(abspath(project_file), abspath(envdir))
+        if startswith(safe_realpath(project_file), safe_realpath(envdir))
             return "@" * name
         end
     end

src/Operations.jl

@@ -246,17 +246,14 @@ function reset_all_compat!(proj::Project)
     return nothing
 end
 
-function collect_project(pkg::PackageSpec, path::String)
+function collect_project(pkg::Union{PackageSpec, Nothing}, path::String)
     deps = PackageSpec[]
     weakdeps = Set{UUID}()
     project_file = projectfile_path(path; strict=true)
-    if project_file === nothing
-        pkgerror("could not find project file for package $(err_rep(pkg)) at `$path`")
-    end
-    project = read_package(project_file)
+    project = project_file === nothing ?  Project() : read_project(project_file)
     julia_compat = get_compat(project, "julia")
     if !isnothing(julia_compat) && !(VERSION in julia_compat)
-        pkgerror("julia version requirement from Project.toml's compat section not satisfied for package $(err_rep(pkg)) at `$path`")
+        pkgerror("julia version requirement from Project.toml's compat section not satisfied for package at `$path`")
     end
     for (name, uuid) in project.deps
         path, repo = get_path_repo(project, name)
@@ -268,11 +265,13 @@ function collect_project(pkg::PackageSpec, path::String)
         push!(deps, PackageSpec(name, uuid, vspec))
         push!(weakdeps, uuid)
     end
-    if project.version !== nothing
-        pkg.version = project.version
-    else
-        # @warn("project file for $(pkg.name) is missing a `version` entry")
-        pkg.version = VersionNumber(0)
+    if pkg !== nothing
+        if project.version !== nothing
+            pkg.version = project.version
+        else
+            # @warn("project file for $(pkg.name) is missing a `version` entry")
+            pkg.version = VersionNumber(0)
+        end
     end
     return deps, weakdeps
 end
@@ -310,13 +309,13 @@ end
 function collect_fixed!(env::EnvCache, pkgs::Vector{PackageSpec}, names::Dict{UUID, String})
     deps_map = Dict{UUID,Vector{PackageSpec}}()
     weak_map = Dict{UUID,Set{UUID}}()
-    if env.pkg !== nothing
-        pkg = env.pkg
-        deps, weakdeps = collect_project(pkg, dirname(env.project_file))
-        deps_map[pkg.uuid] = deps
-        weak_map[pkg.uuid] = weakdeps
-        names[pkg.uuid] = pkg.name
-    end
+
+    uuid = Types.project_uuid(env)
+    deps, weakdeps = collect_project(env.pkg, dirname(env.project_file))
+    deps_map[uuid] = deps
+    weak_map[uuid] = weakdeps
+    names[uuid] = env.pkg === nothing ? "project" : env.pkg.name
+
     for pkg in pkgs
         # add repo package if necessary
         if (pkg.repo.rev !== nothing || pkg.repo.source !== nothing) && pkg.tree_hash === nothing
@@ -346,7 +345,8 @@ function collect_fixed!(env::EnvCache, pkgs::Vector{PackageSpec}, names::Dict{UU
             idx = findfirst(pkg -> pkg.uuid == uuid, pkgs)
             fix_pkg = pkgs[idx]
         end
-        fixed[uuid] = Resolve.Fixed(fix_pkg.version, q, weak_map[uuid])
+        fixpkgversion = fix_pkg === nothing ? v"0.0.0" : fix_pkg.version
+        fixed[uuid] = Resolve.Fixed(fixpkgversion, q, weak_map[uuid])
     end
     return fixed
 end
@@ -1403,6 +1403,7 @@ function add(ctx::Context, pkgs::Vector{PackageSpec}, new_git=Set{UUID}();
     assert_can_add(ctx, pkgs)
     # load manifest data
     for (i, pkg) in pairs(pkgs)
+        delete!(ctx.env.project.weakdeps, pkg.name)
         entry = manifest_info(ctx.env.manifest, pkg.uuid)
         is_dep = any(uuid -> uuid == pkg.uuid, [uuid for (name, uuid) in ctx.env.project.deps])
         source_path, source_repo = get_path_repo(ctx.env.project, pkg.name)
@@ -1467,6 +1468,7 @@ function develop(ctx::Context, pkgs::Vector{PackageSpec}, new_git::Set{UUID};
     assert_can_add(ctx, pkgs)
     # no need to look at manifest.. dev will just nuke whatever is there before
     for pkg in pkgs
+        delete!(ctx.env.project.weakdeps, pkg.name)
         ctx.env.project.deps[pkg.name] = pkg.uuid
     end
     # resolve & apply package versions
@@ -2288,6 +2290,7 @@ function status_ext_info(pkg::PackageSpec, env::EnvCache)
     manifest = env.manifest
     manifest_info = get(manifest, pkg.uuid, nothing)
     manifest_info === nothing && return nothing
+    depses = manifest_info.deps
     weakdepses = manifest_info.weakdeps
     exts = manifest_info.exts
     if !isempty(weakdepses) && !isempty(exts)
@@ -2299,10 +2302,14 @@ function status_ext_info(pkg::PackageSpec, env::EnvCache)
             # Check if deps are loaded
             extdeps_info= Tuple{String, Bool}[]
             for extdep in extdeps
-                haskey(weakdepses, extdep) ||
-                    pkgerror(isnothing(pkg.name) ? "M" : "$(pkg.name) has a m",
-                             "alformed Project.toml, the extension package $extdep is not listed in [weakdeps]")
-                uuid = weakdepses[extdep]
+                if !(haskey(weakdepses, extdep) || haskey(depses, extdep))
+                    pkgerror(isnothing(pkg.name) ? "M" : "$(pkg.name) has a malformed Project.toml, ",
+                             "the extension package $extdep is not listed in [weakdeps] or [deps]")
+                end
+                uuid = get(weakdepses, extdep, nothing)
+                if uuid === nothing
+                    uuid = depses[extdep]
+                end
                 loaded = haskey(Base.loaded_modules, Base.PkgId(uuid, extdep))
                 push!(extdeps_info, (extdep, loaded))
             end

src/Registry/Registry.jl

@@ -447,6 +447,15 @@ function update(regs::Vector{RegistrySpec}; io::IO=stderr_f(), force::Bool=true,
                                     registry_update_log[string(reg.uuid)] = now()
                                     @label done_tarball_read
                                 else
+                                    if reg.name == "General" && Base.get_bool_env("JULIA_PKG_GEN_REG_FMT_CHECK", true)
+                                        @info """
+                                            The General registry is installed via unpacked tarball.
+                                            Consider reinstalling it via the newer faster direct from
+                                            tarball format by running:
+                                              pkg> registry rm General; registry add General
+
+                                            """ maxlog=1
+                                    end
                                     mktempdir() do tmp
                                         try
                                             download_verify_unpack(url, nothing, tmp, ignore_existence = true, io=io)
@@ -465,6 +474,14 @@ function update(regs::Vector{RegistrySpec}; io::IO=stderr_f(), force::Bool=true,
                         end
                     elseif isdir(joinpath(reg.path, ".git"))
                         printpkgstyle(io, :Updating, "registry at " * regpath)
+                        if reg.name == "General" && Base.get_bool_env("JULIA_PKG_GEN_REG_FMT_CHECK", true)
+                            @info """
+                                The General registry is installed via git. Consider reinstalling it via
+                                the newer faster direct from tarball format by running:
+                                  pkg> registry rm General; registry add General
+
+                                """ maxlog=1
+                        end
                         LibGit2.with(LibGit2.GitRepo(reg.path)) do repo
                             if LibGit2.isdirty(repo)
                                 push!(errors, (regpath, "registry dirty"))

src/Types.jl

@@ -412,7 +412,7 @@ Base.@kwdef mutable struct Context
     julia_version::Union{VersionNumber,Nothing} = VERSION
 end
 
-project_uuid(env::EnvCache) = env.pkg === nothing ? nothing : env.pkg.uuid
+project_uuid(env::EnvCache) = env.pkg === nothing ? Base.dummy_uuid(env.project_file) : env.pkg.uuid
 collides_with_project(env::EnvCache, pkg::PackageSpec) =
     is_project_name(env, pkg.name) || is_project_uuid(env, pkg.uuid)
 is_project(env::EnvCache, pkg::PackageSpec) = is_project_uuid(env, pkg.uuid)

test/extensions.jl

@@ -1,5 +1,6 @@
 using  .Utils
 using Test
+using UUIDs
 
 @testset "weak deps" begin
     he_root = joinpath(@__DIR__, "test_packages", "ExtensionExamples", "HasExtensions.jl")
@@ -93,4 +94,24 @@ using Test
             @test proj.extras == Dict{String, Base.UUID}("Example" => Base.UUID("7876af07-990d-54b4-ab0e-23690620f79a"))
         end
     end
+
+    isolate(loaded_depot=false) do
+        mktempdir() do dir
+            Pkg.Registry.add("General")
+            path = joinpath(@__DIR__, "test_packages", "TestWeakDepProject")
+            cp(path, joinpath(dir, "TestWeakDepProject"))
+            Pkg.activate(joinpath(dir, "TestWeakDepProject"))
+            Pkg.resolve()
+            @test Pkg.dependencies()[UUID("2ab3a3ac-af41-5b50-aa03-7779005ae688")].version == v"0.3.26"
+
+            # Check that explicitly adding a package that is a weak dep removes it from the set of weak deps
+            ctx = Pkg.Types.Context()
+            @test "LogExpFunctions" in keys(ctx.env.project.weakdeps)
+            @test !("LogExpFunctions" in keys(ctx.env.project.deps))
+            Pkg.add("LogExpFunctions")
+            ctx = Pkg.Types.Context()
+            @test "LogExpFunctions" in keys(ctx.env.project.deps)
+            @test !("LogExpFunctions" in keys(ctx.env.project.weakdeps))
+        end
+    end
 end

test/repl.jl

@@ -734,4 +734,14 @@ end
     end
 end
 
+@testset "JuliaLang/julia #55850" begin
+    tmp_55850 = mktempdir()
+    tmp_sym_link = joinpath(tmp_55850, "sym")
+    symlink(tmp_55850, tmp_sym_link; dir_target=true)
+    withenv("JULIA_DEPOT_PATH" => tmp_sym_link * (Sys.iswindows() ? ";" : ":"), "JULIA_LOAD_PATH" => nothing) do
+        prompt = readchomp(`$(Base.julia_cmd()[1]) --project=$(dirname(@__DIR__)) --startup-file=no -e "using Pkg, REPL; Pkg.activate(io=devnull); REPLExt = Base.get_extension(Pkg, :REPLExt); print(REPLExt.promptf())"`)
+        @test prompt == "(@v$(VERSION.major).$(VERSION.minor)) pkg> "
+    end
+end
+
 end # module

test/test_packages/TestWeakDepProject/Project.toml

@@ -0,0 +1,9 @@
+[deps]
+ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
+
+[weakdeps]
+LogExpFunctions = "2ab3a3ac-af41-5b50-aa03-7779005ae688"
+
+[compat]
+ForwardDiff = "=0.10.36"
+LogExpFunctions = "=0.3.26"