diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index a8a094d..9706e52 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -8,57 +8,60 @@ on: - master tags: '*' workflow_dispatch: + +concurrency: + # Skip intermediate builds: all builds except for builds on the `master` branch + # Cancel intermediate builds: only pull request builds + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.ref != 'refs/heads/master' || github.run_number }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} + jobs: test: - name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} + name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ github.event_name }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: version: - - '1.0' - - '1.6' + - 'min' + - 'lts' - '1' - - 'nightly' + - 'pre' os: - ubuntu-latest - arch: - - x64 + - windows-latest + - macOS-latest + exclude: + # For Julia 1.6 no aarch64 binary exists + - version: 'min' + os: macOS-latest + include: + - version: 'min' + os: macOS-13 # uses x64 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: julia-actions/setup-julia@v2 with: version: ${{ matrix.version }} - arch: ${{ matrix.arch }} - - uses: actions/cache@v4 - env: - cache-name: cache-artifacts - with: - path: ~/.julia/artifacts - key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} - restore-keys: | - ${{ runner.os }}-test-${{ env.cache-name }}- - ${{ runner.os }}-test- - ${{ runner.os }}- + - uses: julia-actions/cache@v2 - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-runtest@v1 - uses: julia-actions/julia-processcoverage@v1 - uses: codecov/codecov-action@v5 with: - file: lcov.info + files: lcov.info + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: true docs: name: Documentation runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - uses: julia-actions/setup-julia@v2 with: version: '1' - - run: | - julia --project=docs -e ' - using Pkg - Pkg.develop(PackageSpec(path=pwd())) - Pkg.instantiate()' + - uses: julia-actions/cache@v2 + - run: julia --project=docs -e 'import Pkg; Pkg.instantiate()' - run: | julia --project=docs -e ' using Documenter: doctest diff --git a/.github/workflows/CompatHelper.yml b/.github/workflows/CompatHelper.yml index 14fd628..5aa0054 100644 --- a/.github/workflows/CompatHelper.yml +++ b/.github/workflows/CompatHelper.yml @@ -24,9 +24,8 @@ jobs: - name: "Run CompatHelper" run: | import CompatHelper - CompatHelper.main() + CompatHelper.main(; dirs = ["", "docs"]) shell: julia --color=yes {0} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -# COMPATHELPER_PRIV: ${{ secrets.DOCUMENTER_KEY }} -# COMPATHELPER_PRIV: ${{ secrets.COMPATHELPER_PRIV }} + COMPATHELPER_PRIV: ${{ secrets.DOCUMENTER_KEY }} diff --git a/.gitignore b/.gitignore index 8fe2f16..25e1e9d 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ Manifest.toml *.jl.cov *.jl.mem .DS_Store +.vscode/ \ No newline at end of file diff --git a/Project.toml b/Project.toml index b076d4b..99a2b9e 100644 --- a/Project.toml +++ b/Project.toml @@ -12,13 +12,22 @@ SparseArrays = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" [compat] +Aqua = "0.8.12" +Distributed = "<0.0.1, 1" +LinearAlgebra = "<0.0.1, 1" Primes = "0.4, 0.5" +Random = "<0.0.1, 1" +Serialization = "<0.0.1, 1" +SparseArrays = "<0.0.1, 1" +SpecialFunctions = "0.8, 1, 2" Statistics = "<0.0.1, 1" +Test = "<0.0.1, 1" julia = "1" [extras] +Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595" SpecialFunctions = "276daf66-3868-5448-9aa4-cd146d93841b" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Test", "SpecialFunctions"] +test = ["Aqua", "Test", "SpecialFunctions"] diff --git a/docs/Project.toml b/docs/Project.toml index aa96972..c271cfe 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,3 +1,10 @@ [deps] DistributedArrays = "aaf54ef3-cdf8-58ed-94cc-d582ad619b94" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" + +[compat] +DistributedArrays = "0.6" +Documenter = "1" + +[sources.DistributedArrays] +path = ".." diff --git a/src/DistributedArrays.jl b/src/DistributedArrays.jl index 916cce4..d7d43d0 100644 --- a/src/DistributedArrays.jl +++ b/src/DistributedArrays.jl @@ -14,6 +14,8 @@ import LinearAlgebra: axpy!, dot, norm, mul! import Primes import Primes: factor +import SparseArrays + # DArray exports export DArray, SubDArray, SubOrDArray, @DArray export dzeros, dones, dfill, drand, drandn, distribute, localpart, localindices, ppeval diff --git a/src/broadcast.jl b/src/broadcast.jl index 4026a54..030f689 100644 --- a/src/broadcast.jl +++ b/src/broadcast.jl @@ -63,7 +63,7 @@ end # - Q: How do decide on the cuts # - then localise arguments on each node ## -@inline function Base.copyto!(dest::DDestArray, bc::Broadcasted) +@inline function Base.copyto!(dest::DDestArray, bc::Broadcasted{Nothing}) axes(dest) == axes(bc) || Broadcast.throwdm(axes(dest), axes(bc)) # Distribute Broadcasted diff --git a/src/darray.jl b/src/darray.jl index b55ccbc..0d1311b 100644 --- a/src/darray.jl +++ b/src/darray.jl @@ -384,7 +384,9 @@ function makelocal(A::DArray{<:Any, <:Any, AT}, I::Vararg{Any, N}) where {N, AT} end # shortcut to set/get localparts of a distributed object -function Base.getindex(d::DArray, s::Symbol) +Base.getindex(d::DArray, s::Symbol) = _getindex(d, s) +Base.getindex(d::DistributedArrays.DArray{<:Any, 1}, s::Symbol) = _getindex(d, s) +function _getindex(d::DArray, s::Symbol) @assert s in [:L, :l, :LP, :lp] return localpart(d) end @@ -454,6 +456,14 @@ function Base.:(==)(d1::SubDArray, d2::SubDArray) return t end +# Fix method ambiguities +for T in (:DArray, :SubDArray) + @eval begin + Base.:(==)(d1::$T{<:Any,1}, d2::SparseArrays.ReadOnly) = d1 == parent(d2) + Base.:(==)(d1::SparseArrays.ReadOnly, d2::$T{<:Any,1}) = parent(d1) == d2 + end +end + """ locate(d::DArray, I::Int...) @@ -513,10 +523,28 @@ dfill(v, d1::Integer, drest::Integer...) = dfill(v, convert(Dims, tuple(d1, dres Construct a distributed uniform random array. Trailing arguments are the same as those accepted by `DArray`. """ -drand(r, dims::Dims, args...) = DArray(I -> rand(r, map(length,I)), dims, args...) -drand(r, d1::Integer, drest::Integer...) = drand(r, convert(Dims, tuple(d1, drest...))) -drand(d1::Integer, drest::Integer...) = drand(Float64, convert(Dims, tuple(d1, drest...))) -drand(d::Dims, args...) = drand(Float64, d, args...) +drand(::Type{T}, dims::Dims) where {T} = DArray(I -> rand(T, map(length, I)), dims) +drand(X, dims::Dims) = DArray(I -> rand(X, map(length, I)), dims) +drand(dims::Dims) = drand(Float64, dims) + +drand(::Type{T}, d1::Integer, drest::Integer...) where {T} = drand(T, Dims((d1, drest...))) +drand(X, d1::Integer, drest::Integer...) = drand(X, Dims((d1, drest...))) +drand(d1::Integer, drest::Integer...) = drand(Float64, Dims((d1, drest...))) + +# With optional process IDs and number of chunks +for N in (1, 2) + @eval begin + drand(::Type{T}, dims::Dims, args::Vararg{Any,$N}) where {T} = DArray(I -> rand(T, map(length, I)), dims, args...) + drand(X, dims::Dims, args::Vararg{Any,$N}) = DArray(I -> rand(X, map(length, I)), dims, args...) + drand(dims::Dims, args::Vararg{Any,$N}) = drand(Float64, dims, args...) + end +end + +# Fix method ambiguities +drand(dims::Dims, procs::Tuple{Vararg{Int}}) = drand(Float64, dims, procs) +drand(dims::Dims, procs::Tuple{Vararg{Int}}, dist) = drand(Float64, dims, procs, dist) +drand(X::Tuple{Vararg{Int}}, dim::Integer) = drand(X, Dims((dim,))) +drand(X::Tuple{Vararg{Int}}, d1::Integer, d2::Integer) = drand(X, Dims((d1, d2))) """ drandn(dims, ...) @@ -643,7 +671,7 @@ allowscalar(flag = true) = (_allowscalar[] = flag) _scalarindexingallowed() = _allowscalar[] || throw(ErrorException("scalar indexing disabled")) getlocalindex(d::DArray, idx...) = localpart(d)[idx...] -function getindex_tuple(d::DArray{T}, I::Tuple{Vararg{Int}}) where T +function getindex_tuple(d::DArray{T,N}, I::NTuple{N,Int}) where {T,N} chidx = locate(d, I...) idxs = d.indices[chidx...] localidx = ntuple(i -> (I[i] - first(idxs[i]) + 1), ndims(d)) @@ -655,16 +683,16 @@ function Base.getindex(d::DArray, i::Int) _scalarindexingallowed() return getindex_tuple(d, Tuple(CartesianIndices(d)[i])) end -function Base.getindex(d::DArray, i::Int...) +function Base.getindex(d::DArray{<:Any,N}, i::Vararg{Int,N}) where {N} _scalarindexingallowed() return getindex_tuple(d, i) end -Base.getindex(d::DArray) = d[1] -if VERSION > v"1.1-" -Base.getindex(d::SubDArray, I::Int...) = invoke(getindex, Tuple{SubArray{<:Any,N},Vararg{Int,N}} where N, d, I...) +function Base.getindex(d::DArray{<:Any,N}, I::Vararg{Any,N}) where {N} + return view(d, I...) end -Base.getindex(d::SubOrDArray, I::Union{Int,UnitRange{Int},Colon,Vector{Int},StepRange{Int,Int}}...) = view(d, I...) + +Base.getindex(d::DArray) = d[1] function Base.isassigned(D::DArray, i::Integer...) try @@ -692,6 +720,15 @@ function Base.copyto!(dest::SubOrDArray, src::AbstractArray) return dest end +# Fix method ambiguities +# TODO: Improve efficiency? +Base.copyto!(dest::SubOrDArray{<:Any,2}, src::SparseArrays.AbstractSparseMatrixCSC) = copyto!(dest, Matrix(src)) +@static if isdefined(SparseArrays, :CHOLMOD) + Base.copyto!(dest::SubOrDArray, src::SparseArrays.CHOLMOD.Dense) = copyto!(dest, Array(src)) + Base.copyto!(dest::SubOrDArray{T}, src::SparseArrays.CHOLMOD.Dense{T}) where {T<:Union{Float32,Float64,ComplexF32,ComplexF64}} = copyto!(dest, Array(src)) + Base.copyto!(dest::SubOrDArray{T,2}, src::SparseArrays.CHOLMOD.Dense{T}) where {T<:Union{Float32,Float64,ComplexF32,ComplexF64}} = copyto!(dest, Array(src)) +end + function Base.deepcopy(src::DArray) dest = similar(src) asyncmap(procs(src)) do p diff --git a/src/mapreduce.jl b/src/mapreduce.jl index 9403652..c529d42 100644 --- a/src/mapreduce.jl +++ b/src/mapreduce.jl @@ -15,7 +15,13 @@ function Base.map!(f::F, dest::DArray, src::DArray{<:Any,<:Any,A}) where {F,A} return dest end -function Base.reduce(f, d::DArray) +# Only defining `reduce(f, ::DArray)` causes method ambiguity issues with +# - `reduce(hcat, ::AbstractVector{<:AbstractVecOrMat})` +# - `reduce(vcat, ::AbstractVector{<:AbstractVecOrMat})` +Base.reduce(f, d::DArray) = _reduce(f, d) +Base.reduce(::typeof(hcat), d::DArray{<:AbstractVecOrMat, 1}) = _reduce(hcat, d) +Base.reduce(::typeof(vcat), d::DArray{<:AbstractVecOrMat, 1}) = _reduce(vcat, d) +function _reduce(f, d::DArray) results = asyncmap(procs(d)) do p remotecall_fetch(p) do return reduce(f, localpart(d)) diff --git a/test/REQUIRE b/test/REQUIRE deleted file mode 100644 index b2a39ac..0000000 --- a/test/REQUIRE +++ /dev/null @@ -1 +0,0 @@ -SpecialFunctions diff --git a/test/aqua.jl b/test/aqua.jl new file mode 100644 index 0000000..ec7a70b --- /dev/null +++ b/test/aqua.jl @@ -0,0 +1,6 @@ +using DistributedArrays, Test +import Aqua + +@testset "Aqua" begin + Aqua.test_all(DistributedArrays) +end diff --git a/test/runtests.jl b/test/runtests.jl index eb353ca..2bd5ca8 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -32,6 +32,7 @@ function check_leaks() end end +include("aqua.jl") include("darray.jl") include("spmd.jl")