Calculate reducedependencies in a slightly less stupid way, as the difference of the loopdependencies of the parent and of itself.

Author: chriselrod

src/LoopVectorization.jl

@@ -6,8 +6,8 @@ using VectorizationBase: REGISTER_SIZE, REGISTER_COUNT, extract_data, num_vector
     maybestaticlength, maybestaticsize, staticm1, subsetview, vzero,
     Static, StaticUnitRange, StaticLowerUnitRange, StaticUpperUnitRange,
     PackedStridedPointer, SparseStridedPointer, RowMajorStridedPointer, StaticStridedPointer, StaticStridedStruct
-using SIMDPirates: VECTOR_SYMBOLS, evadd, evmul, vrange, reduced_add, reduced_prod, reduce_to_add, reduce_to_prod,
-    vmullog2, vmullog10, vdivlog2, vdivlog2add, vdivlog10, vdivlog10add, vfmaddaddone
+using SIMDPirates: VECTOR_SYMBOLS, evadd, evmul, vrange, reduced_add, reduced_prod, reduce_to_add, reduce_to_prod#,
+#    vmullog2, vmullog10, vdivlog2, vdivlog2add, vdivlog10, vdivlog10add, vfmaddaddone
 using Base.Broadcast: Broadcasted, DefaultArrayStyle
 using LinearAlgebra: Adjoint, Transpose
 using MacroTools: prewalk, postwalk
@@ -22,6 +22,7 @@ include("map.jl")
 include("costs.jl")
 include("operations.jl")
 include("graphs.jl")
+include("operation_evaluation_order.jl")
 include("memory_ops_common.jl")
 include("add_loads.jl")
 include("add_stores.jl")
@@ -40,6 +41,7 @@ include("condense_loopset.jl")
 include("reconstruct_loopset.jl")
 include("constructors.jl")
 
+
 include("precompile.jl")
 _precompile_()
 

src/add_compute.jl

@@ -21,16 +21,23 @@ function mergesetdiffv!(
     end
     nothing
 end
+# Everything in arg2 (s1) that isn't in arg3 (s2) is added to arg1 (s3)
 function setdiffv!(s3::AbstractVector{T}, s1::AbstractVector{T}, s2::AbstractVector{T}) where {T}
     for s ∈ s1
         (s ∈ s2) || (s ∉ s3 && push!(s3, s))
     end
 end
+function setdiffv!(s4::AbstractVector{T}, s3::AbstractVector{T}, s1::AbstractVector{T}, s2::AbstractVector{T}) where {T}
+    for s ∈ s1
+        (s ∈ s2) ? (s ∉ s4 && push!(s4, s)) : (s ∉ s3 && push!(s3, s))
+    end
+end
 function update_deps!(deps::Vector{Symbol}, reduceddeps::Vector{Symbol}, parent::Operation)
-    mergesetdiffv!(deps, loopdependencies(parent), reduceddependencies(parent))
+    mergesetv!(deps, loopdependencies(parent))#, reduceddependencies(parent))        
     if !(isload(parent) || isconstant(parent)) && parent.instruction.instr ∉ (:reduced_add, :reduced_prod, :reduce_to_add, :reduce_to_prod)
         mergesetv!(reduceddeps, reduceddependencies(parent))
     end
+    # 
     nothing
 end
 
@@ -42,19 +49,19 @@ function pushparent!(mpref::ArrayReferenceMetaPosition, parent::Operation)
     pushparent!(mpref.parents, mpref.loopdependencies, mpref.reduceddeps, parent)
 end
 function add_parent!(
-    parents::Vector{Operation}, deps::Vector{Symbol}, reduceddeps::Vector{Symbol}, ls::LoopSet, var, elementbytes::Int = 8
+    parents::Vector{Operation}, deps::Vector{Symbol}, reduceddeps::Vector{Symbol}, ls::LoopSet, var, elementbytes::Int, position::Int
 )
     parent = if var isa Symbol
         getop(ls, var, elementbytes)
     elseif var isa Expr #CSE candidate
-        add_operation!(ls, gensym(:temporary), var, elementbytes)
+        add_operation!(ls, gensym(:temporary), var, elementbytes, position)
     else # assumed constant
         add_constant!(ls, var, elementbytes)
     end
     pushparent!(parents, deps, reduceddeps, parent)
 end
 function add_reduction!(
-    parents::Vector{Operation}, deps::Vector{Symbol}, reduceddeps::Vector{Symbol}, ls::LoopSet, var::Symbol, elementbytes::Int = 8
+    parents::Vector{Operation}, deps::Vector{Symbol}, reduceddeps::Vector{Symbol}, ls::LoopSet, var::Symbol, elementbytes::Int
 )
     get!(ls.opdict, var) do
         add_constant!(ls, var, elementbytes)
@@ -80,10 +87,10 @@ function update_reduction_status!(parentvec::Vector{Operation}, deps::Vector{Sym
     end
 end
 function add_reduction_update_parent!(
-    parents::Vector{Operation}, deps::Vector{Symbol}, reduceddeps::Vector{Symbol}, ls::LoopSet,
-    var::Symbol, instr::Symbol, directdependency::Bool, elementbytes::Int = 8
+    vparents::Vector{Operation}, deps::Vector{Symbol}, reduceddeps::Vector{Symbol}, ls::LoopSet,
+    parent::Operation, instr::Symbol, directdependency::Bool, elementbytes::Int
 )
-    parent = getop(ls, var, elementbytes)
+    var = name(parent)
     isouterreduction = parent.instruction === LOOPCONSTANT
     Instr = instruction(ls, instr)
     instrclass = reduction_instruction_class(Instr) # key allows for faster lookups
@@ -110,27 +117,27 @@ function add_reduction_update_parent!(
         reductsym = var
         reductcombine = Symbol("")
     end
-    setdiffv!(reduceddeps, deps, loopdependencies(reductinit))
     combineddeps = copy(deps); mergesetv!(combineddeps, reduceddeps)
-    directdependency && pushparent!(parents, deps, reduceddeps, reductinit)#parent) # deps and reduced deps will not be disjoint
-    update_reduction_status!(parents, combineddeps, name(reductinit))
+    directdependency && pushparent!(vparents, deps, reduceddeps, reductinit)#parent) # deps and reduced deps will not be disjoint
+    update_reduction_status!(vparents, combineddeps, name(reductinit))
     # this is the op added by add_compute
-    op = Operation(length(operations(ls)), reductsym, elementbytes, instr, compute, deps, reduceddeps, parents)
+    op = Operation(length(operations(ls)), reductsym, elementbytes, instr, compute, deps, reduceddeps, vparents)
     parent.instruction === LOOPCONSTANT && push!(ls.outer_reductions, identifier(op))
     opout = pushop!(ls, op, var) # note this overwrites the entry in the operations dict, but not the vector
+    # isouterreduction || iszero(length(reduceddeps)) && return opout
     isouterreduction && return opout
     # create child op, which is the reduction combination
-    childdeps = Symbol[]; childrdeps = Symbol[]; childparents = Operation[]
-    pushparent!(childparents, childdeps, childrdeps, op) # reduce op
-    pushparent!(childparents, childdeps, childrdeps, parent) # to
+    childrdeps = Symbol[]; childparents = Operation[ op, parent ]
+    childdeps = loopdependencies(reductinit)
+    setdiffv!(childrdeps, loopdependencies(op), childdeps)
     child = Operation(
         length(operations(ls)), name(parent), elementbytes, reductcombine, compute, childdeps, childrdeps, childparents
     )
     pushop!(ls, child, name(parent))
     opout
 end
 function add_compute!(
-    ls::LoopSet, var::Symbol, ex::Expr, elementbytes::Int = 8,
+    ls::LoopSet, var::Symbol, ex::Expr, elementbytes::Int, position::Int,
     mpref::Union{Nothing,ArrayReferenceMetaPosition} = nothing
 )
     @assert ex.head === :call
@@ -149,12 +156,12 @@ function add_compute!(
             if isref
                 if mpref == argref
                     reduction = true
-                    add_load!(ls, var, mpref, elementbytes)
+                    add_load!(ls, var, argref, elementbytes)
                 else
                     pushparent!(parents, deps, reduceddeps, add_load!(ls, gensym(:tempload), argref, elementbytes))
                 end
             else
-                add_parent!(parents, deps, reduceddeps, ls, arg, elementbytes)
+                add_parent!(parents, deps, reduceddeps, ls, arg, elementbytes, position)
             end
         elseif arg ∈ ls.loopsymbols
             loopsym = gensym(arg)
@@ -164,11 +171,30 @@ function add_compute!(
             push!(ls.refs_aliasing_syms, loopsymop.ref)
             pushparent!(parents, deps, reduceddeps, loopsymop)
         else
-            add_parent!(parents, deps, reduceddeps, ls, arg, elementbytes)
+            add_parent!(parents, deps, reduceddeps, ls, arg, elementbytes, position)
         end
     end
+    if iszero(length(deps)) && reduction
+        loopnestview = view(ls.loopsymbols, 1:position) 
+        append!(deps, loopnestview)
+        append!(reduceddeps, loopnestview)
+    else
+        loopnestview = view(ls.loopsymbols, 1:position) 
+        newloopdeps = Symbol[]; newreduceddeps = Symbol[];
+        setdiffv!(newloopdeps, newreduceddeps, deps, loopnestview)
+        mergesetv!(newreduceddeps, reduceddeps)
+        deps = newloopdeps; reduceddeps = newreduceddeps
+    end
     if reduction || search_tree(parents, var)
-        add_reduction_update_parent!(parents, deps, reduceddeps, ls, var, instr, reduction, elementbytes)
+        parent = getop(ls, var, elementbytes)
+        setdiffv!(reduceddeps, deps, loopdependencies(parent))
+        if length(reduceddeps) == 0
+            push!(parents, parent)
+            op = Operation(length(operations(ls)), var, elementbytes, instruction(ls,instr), compute, deps, reduceddeps, parents)
+            pushop!(ls, op, var)
+        else
+            add_reduction_update_parent!(parents, deps, reduceddeps, ls, parent, instr, reduction, elementbytes)
+        end
     else
         op = Operation(length(operations(ls)), var, elementbytes, instruction(ls,instr), compute, deps, reduceddeps, parents)
         pushop!(ls, op, var)

src/add_constants.jl

@@ -17,6 +17,8 @@ function add_constant!(ls::LoopSet, var::Symbol, mpref::ArrayReferenceMetaPositi
     pushop!(ls, op, temp)
 end
 # This version has loop dependencies. var gets assigned to sym when lowering.
+# value is what will get assigned within the loop.
+# assignedsym will be assigned to value within the preamble
 function add_constant!(
     ls::LoopSet, value::Symbol, deps::Vector{Symbol}, assignedsym::Symbol = gensym(:constant), elementbytes::Int = 8, f::Symbol = Symbol("")
 )

src/add_ifelse.jl

@@ -4,22 +4,22 @@
 ## although stores and return values will.
 
 
-function add_if!(ls::LoopSet, LHS::Symbol, RHS::Expr, elementbytes::Int = 8, mpref::Union{Nothing,ArrayReferenceMetaPosition} = nothing)
+function add_if!(ls::LoopSet, LHS::Symbol, RHS::Expr, elementbytes::Int, position::Int, mpref::Union{Nothing,ArrayReferenceMetaPosition} = nothing)
     # for now, just simple 1-liners
     @assert length(RHS.args) == 3 "if statements without an else cannot be assigned to a variable."
     condition = first(RHS.args)
-    condop = add_compute!(ls, gensym(:mask), condition, elementbytes, mpref)
+    condop = add_compute!(ls, gensym(:mask), condition, elementbytes, position, mpref)
     iftrue = RHS.args[2]
     (iftrue isa Expr && iftrue.head !== :call) && throw("Only calls or constant expressions are currently supported in if/else blocks.")
-    trueop = add_operation!(ls, Symbol(:iftrue), iftrue, elementbytes)
+    trueop = add_operation!(ls, Symbol(:iftrue), iftrue, elementbytes, position)
     iffalse = RHS.args[3]
     (iffalse isa Expr && iffalse.head !== :call) && throw("Only calls or constant expressions are currently supported in if/else blocks.")
-    falseop = add_operation!(ls, Symbol(:iffalse), iffalse, elementbytes)
+    falseop = add_operation!(ls, Symbol(:iffalse), iffalse, elementbytes, position)
 
     add_compute!(ls, LHS, :vifelse, [condop, trueop, falseop], elementbytes)
 end
 
-function add_andblock!(ls::LoopSet, condop::Operation, LHS, rhsop::Operation, elementbytes::Int)
+function add_andblock!(ls::LoopSet, condop::Operation, LHS, rhsop::Operation, elementbytes::Int, position::Int)
     if LHS isa Symbol
         altop = getop(ls, LHS)
         return add_compute!(ls, LHS, :vifelse, [condop, rhsop, altop], elementbytes)
@@ -29,27 +29,27 @@ function add_andblock!(ls::LoopSet, condop::Operation, LHS, rhsop::Operation, el
         throw("Don't know how to assign onto $LHS.")
     end        
 end
-function add_andblock!(ls::LoopSet, condop::Operation, LHS, RHS::Expr, elementbytes::Int)
-    rhsop = add_compute!(ls, gensym(:iftruerhs), RHS, elementbytes)
-    add_andblock!(ls, condop, LHS, rhsop, elementbytes)
+function add_andblock!(ls::LoopSet, condop::Operation, LHS, RHS::Expr, elementbytes::Int, position::Int)
+    rhsop = add_compute!(ls, gensym(:iftruerhs), RHS, elementbytes, position)
+    add_andblock!(ls, condop, LHS, rhsop, elementbytes, position)
 end
-function add_andblock!(ls::LoopSet, condop::Operation, LHS, RHS, elementbytes::Int)
+function add_andblock!(ls::LoopSet, condop::Operation, LHS, RHS, elementbytes::Int, position::Int)
     rhsop = getop(ls, RHS)
-    add_andblock!(ls, condop, LHS, rhsop, elementbytes)
+    add_andblock!(ls, condop, LHS, rhsop, elementbytes, position)
 end
-function add_andblock!(ls::LoopSet, condexpr::Expr, condeval::Expr, elementbytes::Int)
-    condop = add_compute!(ls, gensym(:mask), condexpr, elementbytes)
+function add_andblock!(ls::LoopSet, condexpr::Expr, condeval::Expr, elementbytes::Int, position::Int)
+    condop = add_compute!(ls, gensym(:mask), condexpr, elementbytes, position)
     @assert condeval.head === :(=)
     @assert length(condeval.args) == 2
     LHS = condeval.args[1]
     RHS = condeval.args[2]
-    add_andblock!(ls, condop, LHS, RHS, elementbytes)
+    add_andblock!(ls, condop, LHS, RHS, elementbytes, position)
 end
-function add_andblock!(ls::LoopSet, ex::Expr, elementbytes::Int)
-    add_andblock!(ls, first(ex.args)::Expr, last(ex.args)::Expr, elementbytes)
+function add_andblock!(ls::LoopSet, ex::Expr, elementbytes::Int, position::Int)
+    add_andblock!(ls, first(ex.args)::Expr, last(ex.args)::Expr, elementbytes, position)
 end
 
-function add_orblock!(ls::LoopSet, condop::Operation, LHS, rhsop::Operation, elementbytes::Int)
+function add_orblock!(ls::LoopSet, condop::Operation, LHS, rhsop::Operation, elementbytes::Int, position::Int)
     if LHS isa Symbol
         altop = getop(ls, LHS)
         return add_compute!(ls, LHS, :vifelse, [condop, altop, rhsop], elementbytes)
@@ -60,23 +60,23 @@ function add_orblock!(ls::LoopSet, condop::Operation, LHS, rhsop::Operation, ele
         throw("Don't know how to assign onto $LHS.")
     end
 end
-function add_orblock!(ls::LoopSet, condop::Operation, LHS, RHS::Expr, elementbytes::Int)
-    rhsop = add_compute!(ls, gensym(:iffalserhs), RHS, elementbytes)
-    add_orblock!(ls, condop, LHS, rhsop, elementbytes)
+function add_orblock!(ls::LoopSet, condop::Operation, LHS, RHS::Expr, elementbytes::Int, position::Int)
+    rhsop = add_compute!(ls, gensym(:iffalserhs), RHS, elementbytes, position)
+    add_orblock!(ls, condop, LHS, rhsop, elementbytes, position)
 end
-function add_orblock!(ls::LoopSet, condop::Operation, LHS, RHS, elementbytes::Int)
+function add_orblock!(ls::LoopSet, condop::Operation, LHS, RHS, elementbytes::Int, position::Int)
     rhsop = getop(ls, RHS)
-    add_orblock!(ls, condop, LHS, rhsop, elementbytes)
+    add_orblock!(ls, condop, LHS, rhsop, elementbytes, position)
 end
-function add_orblock!(ls::LoopSet, condexpr::Expr, condeval::Expr, elementbytes::Int)
-    condop = add_compute!(ls, gensym(:mask), condexpr, elementbytes)
+function add_orblock!(ls::LoopSet, condexpr::Expr, condeval::Expr, elementbytes::Int, position::Int)
+    condop = add_compute!(ls, gensym(:mask), condexpr, elementbytes, position)
     @assert condeval.head === :(=)
     @assert length(condeval.args) == 2
     LHS = condeval.args[1]
     RHS = condeval.args[2]
-    add_orblock!(ls, condop, LHS, RHS, elementbytes)
+    add_orblock!(ls, condop, LHS, RHS, elementbytes, position)
 end
-function add_orblock!(ls::LoopSet, ex::Expr, elementbytes::Int)
-    add_orblock!(ls, first(ex.args)::Expr, last(ex.args)::Expr, elementbytes)
+function add_orblock!(ls::LoopSet, ex::Expr, elementbytes::Int, position::Int)
+    add_orblock!(ls, first(ex.args)::Expr, last(ex.args)::Expr, elementbytes, position)
 end
 

src/add_stores.jl

@@ -40,8 +40,8 @@ function add_store!(
             if ref == opp.ref.ref
                 id = opp.identifier
                 break
-            else
-                @show ref opp.ref.ref
+            # else
+                # @show ref opp.ref.ref
             end
         end
     end
@@ -69,6 +69,14 @@ function add_store_ref!(ls::LoopSet, var::Symbol, ex::Expr, elementbytes::Int =
     array, raw_indices = ref_from_ref(ex)
     add_store!(ls, var, array, raw_indices, elementbytes)
 end
+function add_store_ref!(ls::LoopSet, var, ex::Expr, elementbytes::Int = 8)
+    # array, raw_indices = ref_from_ref(ex)
+    # mpref = array_reference_meta!(ls, array, raw_indices, elementbytes)
+    # c = add_constant!(ls, var, loopdependencies(mpref), gensym(:storeconst), elementbytes)
+    # add_store!(ls, name(c), mpref, elementbytes)
+    c = add_constant!(ls, var, elementbytes)
+    add_store_ref!(ls, name(c), ex, elementbytes)
+end
 function add_store_setindex!(ls::LoopSet, ex::Expr, elementbytes::Int = 8)
     array, raw_indices = ref_from_setindex(ex)
     add_store!(ls, (ex.args[2])::Symbol, array, rawindices, elementbytes)

src/broadcast.jl

@@ -163,16 +163,21 @@ function add_broadcast!(
     # this is the var name in the loop
     parents = Operation[]
     deps = Symbol[]
-    reduceddeps = Symbol[]
+    # reduceddeps = Symbol[]
     for (i,arg) ∈ enumerate(args)
         argname = gensym(:arg)
         pushpreamble!(ls, Expr(:(=), argname, Expr(:macrocall, Symbol("@inbounds"), LineNumberNode(@__LINE__,@__FILE__), Expr(:ref, bcargs, i))))
         # dynamic dispatch
         parent = add_broadcast!(ls, gensym(:temp), argname, loopsyms, arg, elementbytes)::Operation
-        pushparent!(parents, deps, reduceddeps, parent)
+        push!(parents, parent)
+        mergesetdiffv!(deps, loopdependencies(parent), reduceddependencies(parent))
+        # if !(isload(parent) || isconstant(parent))# && parent.instruction.instr ∉ (:reduced_add, :reduced_prod, :reduce_to_add, :reduce_to_prod)
+            # mergesetv!(reduceddeps, reduceddependencies(parent))
+        # end
+        # pushparent!(parents, deps, reduceddeps, parent)
     end
     op = Operation(
-        length(operations(ls)), destname, elementbytes, instr, compute, deps, reduceddeps, parents
+        length(operations(ls)), destname, elementbytes, instr, compute, deps, NOPARENTS, parents
     )
     pushop!(ls, op, destname)
 end

src/constructors.jl

@@ -3,7 +3,7 @@
 
 function Base.copyto!(ls::LoopSet, q::Expr)
     q.head === :for || throw("Expression must be a for loop.")
-    add_loop!(ls, q)
+    add_loop!(ls, q, 8)
 end
 
 function add_ci_call!(q::Expr, f, args, syms, i, mod = nothing)

src/costs.jl

@@ -146,11 +146,17 @@ const COST = Dict{Instruction,InstructionCost}(
     Instruction(:sin) => InstructionCost(18,15.0,68.0,23),
     Instruction(:cos) => InstructionCost(18,15.0,68.0,26),
     Instruction(:sincos) => InstructionCost(25,22.0,70.0,26),
+    Instruction(:sinpi) => InstructionCost(18,15.0,68.0,23),
+    Instruction(:cospi) => InstructionCost(18,15.0,68.0,26),
+    Instruction(:sincospi) => InstructionCost(25,22.0,70.0,26),
     Instruction(:log_fast) => InstructionCost(20,20.0,40.0,20),
     Instruction(:exp_fast) => InstructionCost(20,20.0,20.0,18),
     Instruction(:sin_fast) => InstructionCost(18,15.0,68.0,23),
     Instruction(:cos_fast) => InstructionCost(18,15.0,68.0,26),
     Instruction(:sincos_fast) => InstructionCost(25,22.0,70.0,26),
+    Instruction(:sinpi_fast) => InstructionCost(18,15.0,68.0,23),
+    Instruction(:cospi_fast) => InstructionCost(18,15.0,68.0,26),
+    Instruction(:sincospi_fast) => InstructionCost(25,22.0,70.0,26),
     Instruction(:identity) => InstructionCost(0,0.0,0.0,0),
     Instruction(:adjoint) => InstructionCost(0,0.0,0.0,0),
     Instruction(:transpose) => InstructionCost(0,0.0,0.0,0),

src/determinestrategy.jl

@@ -148,6 +148,7 @@ function unroll_no_reductions(ls, order, vectorized, Wshift, size_T)
     # @show compute_rt, load_rt
     # roundpow2(min(4, round(Int, (compute_rt + load_rt + 1) / compute_rt)))
     rt = max(compute_rt, load_rt)
+    rt == 0.0 && return 4
     max(1, roundpow2( min( 4, round(Int, 16 / rt) ) ))
 end
 function determine_unroll_factor(