diff --git a/hphp/runtime/vm/jit/vasm-arm.cpp b/hphp/runtime/vm/jit/vasm-arm.cpp index e972222e9abfc..8ec1e1483a90a 100644 --- a/hphp/runtime/vm/jit/vasm-arm.cpp +++ b/hphp/runtime/vm/jit/vasm-arm.cpp @@ -163,6 +163,47 @@ vixl::MemOperand M(Vptr p) { return MemOperand(X(p.base), p.disp); } +vixl::Operand toOperand(const VregShiftExtend& se, Width w) { + always_assert_flog(w == Width::Long || w == Width::Quad, + "unsupported width {} for shift/extend operand {}", + show(w), show(se)); + + auto const baseReg = (w == Width::Long + ? vixl::Register{W(Vreg32{se.reg})} + : vixl::Register{X(Vreg64{se.reg})}); + + if (se.isShift()) { + auto const amount = se.amount; + always_assert_flog(amount <= (w == Width::Long ? 31 : 63), + "shift amount {} out of range for {} with width {}", + amount, show(se), show(w)); + auto kind = se.shiftKind(); + if ((kind == vixl::LSL || kind == vixl::NO_SHIFT) && amount == 0) { + return vixl::Operand(baseReg); + } + if (kind == vixl::NO_SHIFT) kind = vixl::LSL; + return vixl::Operand(baseReg, kind, amount); + } + + auto const extend = se.extendKind(); + auto const amount = se.amount; + switch (extend) { + case vixl::UXTB: + case vixl::SXTB: + case vixl::UXTH: + case vixl::SXTH: + case vixl::UXTW: + case vixl::SXTW: + return vixl::Operand(W(Vreg32{se.reg}), extend, amount); + case vixl::UXTX: + case vixl::SXTX: + return vixl::Operand(X(Vreg64{se.reg}), extend, amount); + case vixl::NO_EXTEND: + return vixl::Operand(baseReg, extend, amount); + } + not_reached(); +} + vixl::Condition C(ConditionCode cc) { return arm::convertCC(cc); } @@ -302,6 +343,12 @@ struct Vgen { void emit(const addli& i) { a->Add(W(i.d), W(i.s1), i.s0.l(), UF(i.fl)); } void emit(const addq& i) { a->Add(X(i.d), X(i.s1), X(i.s0), UF(i.fl));} void emit(const addqi& i) { a->Add(X(i.d), X(i.s1), i.s0.q(), UF(i.fl)); } + void emit(const addshiftl& i) { + a->Add(W(i.d), W(i.s1), toOperand(i.s0, Width::Long), UF(i.fl)); + } + void emit(const addshiftq& i) { + a->Add(X(i.d), X(i.s1), toOperand(i.s0, Width::Quad), UF(i.fl)); + } void emit(const addsd& i) { a->Fadd(D(i.d), D(i.s1), D(i.s0)); } void emit(const andb& i) { a->And(W(i.d), W(i.s1), W(i.s0), UF(i.fl)); } void emit(const andbi& i) { a->And(W(i.d), W(i.s1), i.s0.ub(), UF(i.fl)); } @@ -312,6 +359,12 @@ struct Vgen { void emit(const andq& i) { a->And(X(i.d), X(i.s1), X(i.s0), UF(i.fl)); } void emit(const andqi& i) { a->And(X(i.d), X(i.s1), i.s0.q(), UF(i.fl)); } void emit(const andqi64& i) { a->And(X(i.d), X(i.s1), i.s0.q(), UF(i.fl)); } + void emit(const andshiftl& i) { + a->And(W(i.d), W(i.s1), toOperand(i.s0, Width::Long), UF(i.fl)); + } + void emit(const andshiftq& i) { + a->And(X(i.d), X(i.s1), toOperand(i.s0, Width::Quad), UF(i.fl)); + } void emit(const btrq& i) { // NB: We can't directly store the result to i.d because, in case i.s1 is // dead after the btrq, the register allocator may allocated both i.d and @@ -339,6 +392,12 @@ struct Vgen { void emit(const cmpli& i) { a->Cmp(W(i.s1), i.s0.l()); } void emit(const cmpq& i) { a->Cmp(X(i.s1), X(i.s0)); } void emit(const cmpqi& i) { a->Cmp(X(i.s1), i.s0.q()); } + void emit(const cmpshiftl& i) { + a->Cmp(W(i.s1), toOperand(i.s0, Width::Long)); + } + void emit(const cmpshiftq& i) { + a->Cmp(X(i.s1), toOperand(i.s0, Width::Quad)); + } void emit(const cmpsd& i); // TODO(CDE): csinc[bw]{} Should a) sign extend and b) set SF for overflow void emit(const csincb& i) { a->Csinc(W(i.d), W(i.t), W(i.f), C(i.cc)); } @@ -399,6 +458,18 @@ struct Vgen { void emit(const orwi& i); void emit(const orli& i); void emit(const orqi& i); + void emit(const orshiftl& i) { + a->Orr(W(i.d), W(i.s1), toOperand(i.s0, Width::Long)); + if (i.fl) { + a->Bic(vixl::wzr, W(i.d), vixl::wzr, SetFlags); + } + } + void emit(const orshiftq& i) { + a->Orr(X(i.d), X(i.s1), toOperand(i.s0, Width::Quad)); + if (i.fl) { + a->Bic(vixl::xzr, X(i.d), vixl::xzr, SetFlags); + } + } void emit(const pop& i); void emit(const popp& i); void emit(const push& i); @@ -423,6 +494,12 @@ struct Vgen { void emit(const subli& i) { a->Sub(W(i.d), W(i.s1), i.s0.l(), UF(i.fl)); } void emit(const subq& i) { a->Sub(X(i.d), X(i.s1), X(i.s0), UF(i.fl)); } void emit(const subqi& i) { a->Sub(X(i.d), X(i.s1), i.s0.q(), UF(i.fl)); } + void emit(const subshiftl& i) { + a->Sub(W(i.d), W(i.s1), toOperand(i.s0, Width::Long), UF(i.fl)); + } + void emit(const subshiftq& i) { + a->Sub(X(i.d), X(i.s1), toOperand(i.s0, Width::Quad), UF(i.fl)); + } void emit(const subsd& i) { a->Fsub(D(i.d), D(i.s1), D(i.s0)); } void emit(const testb& i){ a->Tst(W(i.s1), W(i.s0)); } void emit(const testbi& i){ a->Tst(W(i.s1), i.s0.ub()); } @@ -432,6 +509,24 @@ struct Vgen { void emit(const testli& i) { a->Tst(W(i.s1), i.s0.l()); } void emit(const testq& i) { a->Tst(X(i.s1), X(i.s0)); } void emit(const testqi& i) { a->Tst(X(i.s1), i.s0.q()); } + void emit(const testshiftl& i) { + a->Tst(W(i.s1), toOperand(i.s0, Width::Long)); + } + void emit(const testshiftq& i) { + a->Tst(X(i.s1), toOperand(i.s0, Width::Quad)); + } + void emit(const xorshiftl& i) { + a->Eor(W(i.d), W(i.s1), toOperand(i.s0, Width::Long)); + if (i.fl) { + a->Bic(vixl::wzr, W(i.d), vixl::wzr, SetFlags); + } + } + void emit(const xorshiftq& i) { + a->Eor(X(i.d), X(i.s1), toOperand(i.s0, Width::Quad)); + if (i.fl) { + a->Bic(vixl::xzr, X(i.d), vixl::xzr, SetFlags); + } + } void emit(const trap& /*i*/); void emit(const ucomisd& i) { a->Fcmp(D(i.s0), D(i.s1)); } void emit(const unpcklpd&); @@ -1547,35 +1642,99 @@ void lowerVptr(Vptr& p, Vout& v, ImmediateStyle is = kLegacyStyle, ImmediateStyl DISP = 4 }; + auto const legalScale = [] (uint8_t scale) { + return scale == 1 || scale == 2 || scale == 4 || scale == 8; + }; + + auto const scaleIndexValue = [&] (uint8_t scale, Vreg64 index) -> Vreg64 { + always_assert(scale != 0); + assertx(index.isValid()); + + switch (scale) { + case 1: + return index; + case 2: + case 4: + case 8: { + auto const amount = Log2(scale); + auto tmp = Vreg64{v.makeReg()}; + v << shlqi{amount, index, tmp, v.makeReg()}; + return tmp; + } + default: { + Vreg64 result = index; + bool haveResult = false; + auto bits = scale; + uint8_t shift = 0; + + while (bits != 0) { + if (bits & 1) { + auto term = [&] () -> Vreg64 { + if (shift == 0) return index; + auto tmp = Vreg64{v.makeReg()}; + v << shlqi{shift, index, tmp, v.makeReg()}; + return tmp; + }(); + + if (!haveResult) { + result = term; + haveResult = true; + } else { + auto tmp = Vreg64{v.makeReg()}; + v << addq{term, result, tmp, v.makeReg()}; + result = tmp; + } + } + bits >>= 1; + ++shift; + } + + assertx(haveResult); + return result; + } + } + }; + + auto const noteIllegalScale = [&] (uint8_t scale) { + if (legalScale(scale)) return; + if (Trace::moduleEnabled(Trace::vasm, 2)) { + FTRACE(2, "lowerVptr: expand non-power-of-two scale {} in {}\n", + static_cast(scale), show(p)); + } + }; + uint8_t mode = (((p.base.isValid() & 0x1) << 0) | ((p.index.isValid() & 0x1) << 1) | (((p.disp != 0) & 0x1) << 2)); + switch (mode) { case BASE: // ldr/str allow [base], nothing to lower. break; - case BASE | INDEX: - if (p.scale != 1 && p.scale != uint8_t(p.width)) { - auto t = v.makeReg(); - v << shlqi{Log2(p.scale), p.index, t, v.makeReg()}; - p.index = t; - p.scale = 1; + case BASE | INDEX: { + if (p.index.isValid() && p.scale != 1) { + auto const widthScale = static_cast(p.width); + auto const keepEncoded = + legalScale(widthScale) && p.scale == widthScale; + if (!keepEncoded) { + noteIllegalScale(p.scale); + p.index = scaleIndexValue(p.scale, p.index); + p.scale = 1; + } } break; + } - case INDEX: - // Not supported, convert to [base]. - if (p.scale > 1) { - auto t = v.makeReg(); - v << shlqi{Log2(p.scale), p.index, t, v.makeReg()}; - p.base = t; - } else { - p.base = p.index; + case INDEX: { + if (p.index.isValid()) { + noteIllegalScale(p.scale); + p.base = scaleIndexValue(p.scale, p.index); } p.index = Vreg{}; p.scale = 1; break; + } case BASE | DISP: { // if the immediate value can be directly encoded we have nothing to do @@ -1612,14 +1771,10 @@ void lowerVptr(Vptr& p, Vout& v, ImmediateStyle is = kLegacyStyle, ImmediateStyl break; } - case INDEX | DISP: - // Not supported, convert to [base, #imm] or [base, index]. - if (p.scale > 1) { - auto t = v.makeReg(); - v << shlqi{Log2(p.scale), p.index, t, v.makeReg()}; - p.base = t; - } else { - p.base = p.index; + case INDEX | DISP: { + if (p.index.isValid()) { + noteIllegalScale(p.scale); + p.base = scaleIndexValue(p.scale, p.index); } if (p.disp >= -256 && p.disp <= 255) { p.index = Vreg{}; @@ -1632,17 +1787,14 @@ void lowerVptr(Vptr& p, Vout& v, ImmediateStyle is = kLegacyStyle, ImmediateStyl p.disp = 0; } break; + } case BASE | INDEX | DISP: { - // Not supported, convert to [base, index]. - auto index = v.makeReg(); - if (p.scale > 1) { - auto t = v.makeReg(); - v << shlqi{Log2(p.scale), p.index, t, v.makeReg()}; - v << addqi{p.disp, t, index, v.makeReg()}; - } else { - v << addqi{p.disp, p.index, index, v.makeReg()}; - } + assertx(p.index.isValid()); + noteIllegalScale(p.scale); + auto const scaled = scaleIndexValue(p.scale, p.index); + auto index = Vreg64{v.makeReg()}; + v << addqi{p.disp, scaled, index, v.makeReg()}; p.index = index; p.scale = 1; p.disp = 0; diff --git a/hphp/runtime/vm/jit/vasm-copy.cpp b/hphp/runtime/vm/jit/vasm-copy.cpp index f95e232b8379a..d6e88f5960662 100644 --- a/hphp/runtime/vm/jit/vasm-copy.cpp +++ b/hphp/runtime/vm/jit/vasm-copy.cpp @@ -729,6 +729,11 @@ struct OptVisit { }); } + void use(VregShiftExtend& se) { + if (!se.reg.isValid()) return; + use(se.reg); + } + template typename std::enable_if< std::is_same::value || diff --git a/hphp/runtime/vm/jit/vasm-graph-color.cpp b/hphp/runtime/vm/jit/vasm-graph-color.cpp index d362ed56601ac..67f921915cebf 100644 --- a/hphp/runtime/vm/jit/vasm-graph-color.cpp +++ b/hphp/runtime/vm/jit/vasm-graph-color.cpp @@ -2769,6 +2769,9 @@ bool src_cmp(const Vunit& unit, Vtuple t1, Vtuple t2) { bool src_cmp(const Vunit& unit, VcallArgsId a1, VcallArgsId a2) { return unit.vcallArgs[a1] == unit.vcallArgs[a2]; } +bool src_cmp(const Vunit& unit, VregShiftExtend s1, VregShiftExtend s2) { + return s1 == s2; +} } @@ -3703,7 +3706,8 @@ struct VptrVisitor { std::is_same::value || std::is_same::value || std::is_same::value || - std::is_same::value + std::is_same::value || + std::is_same::value >::type use(T) {} void use(RegSet) {} void use(VcallArgsId) {} @@ -4380,6 +4384,7 @@ struct FoldRematWithUseVisit { void use(Vtuple t) { for (auto const r : unit.tuples[t]) onUse(r); } void use(Vptr& ptr) { onVptr(ptr); } void use(Vreg r) { onUse(r); } + void use(VregShiftExtend& se) { onUse(se.reg); } OnUse onUse; OnVptr onVptr; diff --git a/hphp/runtime/vm/jit/vasm-info.cpp b/hphp/runtime/vm/jit/vasm-info.cpp index 33d2dc0b85644..381e77a26ad66 100644 --- a/hphp/runtime/vm/jit/vasm-info.cpp +++ b/hphp/runtime/vm/jit/vasm-info.cpp @@ -163,6 +163,8 @@ bool effectsImpl(const Vinstr& inst, bool pure) { case Vinstr::addli: case Vinstr::addq: case Vinstr::addqi: + case Vinstr::addshiftl: + case Vinstr::addshiftq: case Vinstr::addsd: case Vinstr::andb: case Vinstr::andbi: @@ -173,6 +175,8 @@ bool effectsImpl(const Vinstr& inst, bool pure) { case Vinstr::andq: case Vinstr::andqi64: case Vinstr::andqi: + case Vinstr::andshiftl: + case Vinstr::andshiftq: case Vinstr::btrq: case Vinstr::cmovb: case Vinstr::cmovl: @@ -184,6 +188,8 @@ bool effectsImpl(const Vinstr& inst, bool pure) { case Vinstr::cmpli: case Vinstr::cmpq: case Vinstr::cmpqi: + case Vinstr::cmpshiftl: + case Vinstr::cmpshiftq: case Vinstr::cmpsd: case Vinstr::cmpw: case Vinstr::cmpwi: @@ -242,6 +248,8 @@ bool effectsImpl(const Vinstr& inst, bool pure) { case Vinstr::orli: case Vinstr::orq: case Vinstr::orqi: + case Vinstr::orshiftl: + case Vinstr::orshiftq: case Vinstr::reload: case Vinstr::roundsd: case Vinstr::sar: @@ -268,6 +276,8 @@ bool effectsImpl(const Vinstr& inst, bool pure) { case Vinstr::subli: case Vinstr::subq: case Vinstr::subqi: + case Vinstr::subshiftl: + case Vinstr::subshiftq: case Vinstr::subsd: case Vinstr::testb: case Vinstr::testbi: @@ -277,6 +287,8 @@ bool effectsImpl(const Vinstr& inst, bool pure) { case Vinstr::testqi: case Vinstr::testw: case Vinstr::testwi: + case Vinstr::testshiftl: + case Vinstr::testshiftq: case Vinstr::ubfmli: case Vinstr::ucomisd: case Vinstr::unpcklpd: @@ -287,6 +299,8 @@ bool effectsImpl(const Vinstr& inst, bool pure) { case Vinstr::xorl: case Vinstr::xorq: case Vinstr::xorqi: + case Vinstr::xorshiftl: + case Vinstr::xorshiftq: assertx(!touchesMemory(inst)); return pure; diff --git a/hphp/runtime/vm/jit/vasm-instr.cpp b/hphp/runtime/vm/jit/vasm-instr.cpp index 4d0acd4fff29b..c53a9e967951b 100644 --- a/hphp/runtime/vm/jit/vasm-instr.cpp +++ b/hphp/runtime/vm/jit/vasm-instr.cpp @@ -291,8 +291,10 @@ Width width(Vinstr::Opcode op) { case Vinstr::addli: case Vinstr::addlm: case Vinstr::addlim: + case Vinstr::addshiftl: case Vinstr::andl: case Vinstr::andli: + case Vinstr::andshiftl: case Vinstr::decl: case Vinstr::declm: case Vinstr::incl: @@ -301,18 +303,23 @@ Width width(Vinstr::Opcode op) { case Vinstr::shrli: case Vinstr::subl: case Vinstr::subli: + case Vinstr::subshiftl: case Vinstr::xorl: + case Vinstr::xorshiftl: case Vinstr::orlim: + case Vinstr::orshiftl: case Vinstr::cmovl: case Vinstr::csincl: case Vinstr::cmpl: case Vinstr::cmpli: case Vinstr::cmplm: case Vinstr::cmplim: + case Vinstr::cmpshiftl: case Vinstr::testl: case Vinstr::testli: case Vinstr::testlim: case Vinstr::testlm: + case Vinstr::testshiftl: case Vinstr::movl: case Vinstr::loadl: case Vinstr::loadpairl: @@ -330,9 +337,11 @@ Width width(Vinstr::Opcode op) { case Vinstr::addqmr: case Vinstr::addqrm: case Vinstr::addqim: + case Vinstr::addshiftq: case Vinstr::andq: case Vinstr::andqi: case Vinstr::andqi64: + case Vinstr::andshiftq: case Vinstr::btrq: case Vinstr::decq: case Vinstr::decqm: @@ -350,6 +359,7 @@ Width width(Vinstr::Opcode op) { case Vinstr::orq: case Vinstr::orqi: case Vinstr::orqim: + case Vinstr::orshiftq: case Vinstr::sar: case Vinstr::shl: case Vinstr::shr: @@ -358,17 +368,21 @@ Width width(Vinstr::Opcode op) { case Vinstr::shrqi: case Vinstr::subq: case Vinstr::subqi: + case Vinstr::subshiftq: case Vinstr::subqim: case Vinstr::xorq: case Vinstr::xorqi: + case Vinstr::xorshiftq: case Vinstr::cmpq: case Vinstr::cmpqi: case Vinstr::cmpqm: case Vinstr::cmpqim: + case Vinstr::cmpshiftq: case Vinstr::testq: case Vinstr::testqi: case Vinstr::testqm: case Vinstr::testqim: + case Vinstr::testshiftq: case Vinstr::cloadq: case Vinstr::cmovq: case Vinstr::csincq: diff --git a/hphp/runtime/vm/jit/vasm-instr.h b/hphp/runtime/vm/jit/vasm-instr.h index 2fa51dc98fa96..d47d6299cc845 100644 --- a/hphp/runtime/vm/jit/vasm-instr.h +++ b/hphp/runtime/vm/jit/vasm-instr.h @@ -162,6 +162,8 @@ struct Vunit; O(addqrm, I(fl), U(s1) UM(m), D(sf)) \ O(addqi, I(s0) I(fl), UH(s1,d), DH(d,s1) D(sf)) \ O(addqim, I(s0) I(fl), UM(m), D(sf)) \ + O(addshiftl, I(fl), U(s0) UH(s1,d), DH(d,s1) D(sf)) \ + O(addshiftq, I(fl), U(s0) UH(s1,d), DH(d,s1) D(sf)) \ O(addsd, Inone, U(s0) U(s1), D(d))\ O(andb, I(fl), U(s0) UH(s1,d), DH(d,s1) D(sf)) \ O(andbi, I(s0) I(fl), UH(s1,d), DH(d,s1) D(sf)) \ @@ -173,6 +175,8 @@ struct Vunit; O(andq, I(fl), U(s0) UH(s1,d), DH(d,s1) D(sf)) \ O(andqi, I(s0) I(fl), UH(s1,d), DH(d,s1) D(sf)) \ O(andqi64, I(s0) I(fl), UH(s1,d), DH(d,s1) D(sf)) \ + O(andshiftl, I(fl), U(s0) UH(s1,d), DH(d,s1) D(sf)) \ + O(andshiftq, I(fl), U(s0) UH(s1,d), DH(d,s1) D(sf)) \ O(btrq, I(s0) I(fl), UH(s1,d), DH(d,s1) D(sf)) \ O(decl, I(fl), UH(s,d), DH(d,s) D(sf))\ O(declm, I(fl), UM(m), D(sf))\ @@ -201,6 +205,8 @@ struct Vunit; O(orq, I(fl), U(s0) UH(s1,d), DH(d,s1) D(sf)) \ O(orqi, I(s0) I(fl), UH(s1,d), DH(d,s1) D(sf)) \ O(orqim, I(s0) I(fl), UM(m), D(sf))\ + O(orshiftl, I(fl), U(s0) UH(s1,d), DH(d,s1) D(sf)) \ + O(orshiftq, I(fl), U(s0) UH(s1,d), DH(d,s1) D(sf)) \ O(sar, I(fl), U(s0) U(s1), D(d) D(sf))\ O(shl, I(fl), U(s0) U(s1), D(d) D(sf))\ O(shr, I(fl), U(s0) U(s1), D(d) D(sf))\ @@ -214,6 +220,8 @@ struct Vunit; O(subq, I(fl), UA(s0) UH(s1,d), DH(d,s1) D(sf)) \ O(subqi, I(s0) I(fl), UH(s1,d), DH(d,s1) D(sf))\ O(subqim, I(s0) I(fl), UM(m), D(sf)) \ + O(subshiftl, I(fl), UA(s0) UH(s1,d), DH(d,s1) D(sf)) \ + O(subshiftq, I(fl), UA(s0) UH(s1,d), DH(d,s1) D(sf)) \ O(subsd, Inone, UA(s0) U(s1), D(d))\ O(xorb, I(fl), U(s0) UH(s1,d), DH(d,s1) D(sf)) \ O(xorbi, I(s0) I(fl), UH(s1,d), DH(d,s1) D(sf))\ @@ -222,6 +230,8 @@ struct Vunit; O(xorl, I(fl), U(s0) UH(s1,d), DH(d,s1) D(sf)) \ O(xorq, I(fl), U(s0) UH(s1,d), DH(d,s1) D(sf)) \ O(xorqi, I(s0) I(fl), UH(s1,d), DH(d,s1) D(sf))\ + O(xorshiftl, I(fl), U(s0) UH(s1,d), DH(d,s1) D(sf)) \ + O(xorshiftq, I(fl), U(s0) UH(s1,d), DH(d,s1) D(sf)) \ /* compares and tests */\ O(cmpb, I(fl), U(s0) U(s1), D(sf))\ O(cmpbi, I(s0) I(fl), U(s1), D(sf))\ @@ -239,6 +249,8 @@ struct Vunit; O(cmpqi, I(s0) I(fl), U(s1), D(sf))\ O(cmpqm, I(fl), U(s0) U(s1), D(sf))\ O(cmpqim, I(s0) I(fl), U(s1), D(sf))\ + O(cmpshiftl, I(fl), U(s0) U(s1), D(sf))\ + O(cmpshiftq, I(fl), U(s0) U(s1), D(sf))\ O(cmpsd, I(pred), UA(s0) U(s1), D(d))\ O(ucomisd, I(fl), U(s0) U(s1), D(sf))\ O(testb, I(fl), U(s0) U(s1), D(sf))\ @@ -255,6 +267,8 @@ struct Vunit; O(testlm, I(fl), U(s0) U(s1), D(sf)) \ O(testq, I(fl), U(s0) U(s1), D(sf))\ O(testqi, I(s0) I(fl), U(s1), D(sf))\ + O(testshiftl, I(fl), U(s0) U(s1), D(sf))\ + O(testshiftq, I(fl), U(s0) U(s1), D(sf))\ O(testqm, I(fl), U(s0) U(s1), D(sf))\ O(testqim, I(s0) I(fl), U(s1), D(sf))\ /* conditional operations */\ @@ -1008,6 +1022,8 @@ struct addqi { Immed s0; Vreg64 s1, d; VregSF sf; Vflags fl; }; struct addqmr { Vptr64 m; Vreg64 s1; Vreg64 d; VregSF sf; Vflags fl; }; struct addqrm { Vreg64 s1; Vptr64 m; VregSF sf; Vflags fl; }; struct addqim { Immed s0; Vptr64 m; VregSF sf; Vflags fl; }; +struct addshiftl { VregShiftExtend s0; Vreg32 s1, d; VregSF sf; Vflags fl; }; +struct addshiftq { VregShiftExtend s0; Vreg64 s1, d; VregSF sf; Vflags fl; }; struct addsd { VregDbl s0, s1, d; }; // and: s0 & {s1|m} => {d|m}, sf struct andb { Vreg8 s0, s1, d; VregSF sf; Vflags fl; }; @@ -1019,6 +1035,8 @@ struct andl { Vreg32 s0, s1, d; VregSF sf; Vflags fl; }; struct andli { Immed s0; Vreg32 s1, d; VregSF sf; Vflags fl; }; struct andq { Vreg64 s0, s1, d; VregSF sf; Vflags fl; }; struct andqi { Immed s0; Vreg64 s1, d; VregSF sf; Vflags fl; }; +struct andshiftl { VregShiftExtend s0; Vreg32 s1, d; VregSF sf; Vflags fl; }; +struct andshiftq { VregShiftExtend s0; Vreg64 s1, d; VregSF sf; Vflags fl; }; // Set bit s0 in s1 to 0, sets carry if set, sets d to s1 struct btrq { Immed s0; Vreg64 s1, d; VregSF sf; Vflags fl; }; // dec: {s|m} - 1 => {d|m}, sf @@ -1056,6 +1074,8 @@ struct orlim { Immed s0; Vptr32 m; VregSF sf; Vflags fl; }; struct orq { Vreg64 s0, s1, d; VregSF sf; Vflags fl; }; struct orqi { Immed s0; Vreg64 s1, d; VregSF sf; Vflags fl; }; struct orqim { Immed s0; Vptr64 m; VregSF sf; Vflags fl; }; +struct orshiftl { VregShiftExtend s0; Vreg32 s1, d; VregSF sf; Vflags fl; }; +struct orshiftq { VregShiftExtend s0; Vreg64 s1, d; VregSF sf; Vflags fl; }; // shift: s1 << s0 => d, sf struct sar { Vreg64 s0, s1, d; VregSF sf; Vflags fl; }; struct shl { Vreg64 s0, s1, d; VregSF sf; Vflags fl; }; @@ -1071,6 +1091,8 @@ struct subli { Immed s0; Vreg32 s1, d; VregSF sf; Vflags fl; }; struct subq { Vreg64 s0, s1, d; VregSF sf; Vflags fl; }; struct subqi { Immed s0; Vreg64 s1, d; VregSF sf; Vflags fl; }; struct subqim { Immed s0; Vptr64 m; VregSF sf; Vflags fl; }; +struct subshiftl { VregShiftExtend s0; Vreg32 s1, d; VregSF sf; Vflags fl; }; +struct subshiftq { VregShiftExtend s0; Vreg64 s1, d; VregSF sf; Vflags fl; }; struct subsd { VregDbl s0, s1, d; }; // xor: s0 ^ s1 => d, sf struct xorb { Vreg8 s0, s1, d; VregSF sf; Vflags fl; }; @@ -1080,6 +1102,8 @@ struct xorwi { Immed s0; Vreg16 s1, d; VregSF sf; Vflags fl; }; struct xorl { Vreg32 s0, s1, d; VregSF sf; Vflags fl; }; struct xorq { Vreg64 s0, s1, d; VregSF sf; Vflags fl; }; struct xorqi { Immed s0; Vreg64 s1, d; VregSF sf; Vflags fl; }; +struct xorshiftl { VregShiftExtend s0; Vreg32 s1, d; VregSF sf; Vflags fl; }; +struct xorshiftq { VregShiftExtend s0; Vreg64 s1, d; VregSF sf; Vflags fl; }; /* * Compares and tests. @@ -1101,6 +1125,8 @@ struct cmpq { Vreg64 s0; Vreg64 s1; VregSF sf; Vflags fl; }; struct cmpqi { Immed s0; Vreg64 s1; VregSF sf; Vflags fl; }; struct cmpqm { Vreg64 s0; Vptr64 s1; VregSF sf; Vflags fl; }; struct cmpqim { Immed s0; Vptr64 s1; VregSF sf; Vflags fl; }; +struct cmpshiftl { VregShiftExtend s0; Vreg32 s1; VregSF sf; Vflags fl; }; +struct cmpshiftq { VregShiftExtend s0; Vreg64 s1; VregSF sf; Vflags fl; }; struct cmpsd { ComparisonPred pred; VregDbl s0, s1, d; }; struct ucomisd { VregDbl s0, s1; VregSF sf; Vflags fl; }; // s1 & s0 => sf @@ -1118,6 +1144,8 @@ struct testlim { Immed s0; Vptr32 s1; VregSF sf; Vflags fl; }; struct testlm { Vreg32 s0; Vptr32 s1; VregSF sf; Vflags fl; }; struct testq { Vreg64 s0, s1; VregSF sf; Vflags fl; }; struct testqi { Immed s0; Vreg64 s1; VregSF sf; Vflags fl; }; +struct testshiftl { VregShiftExtend s0; Vreg32 s1; VregSF sf; Vflags fl; }; +struct testshiftq { VregShiftExtend s0; Vreg64 s1; VregSF sf; Vflags fl; }; struct testqm { Vreg64 s0; Vptr64 s1; VregSF sf; Vflags fl; }; struct testqim { Immed s0; Vptr64 s1; VregSF sf; Vflags fl; }; diff --git a/hphp/runtime/vm/jit/vasm-print.cpp b/hphp/runtime/vm/jit/vasm-print.cpp index c058674c6d7ac..8bd39b85c6e4e 100644 --- a/hphp/runtime/vm/jit/vasm-print.cpp +++ b/hphp/runtime/vm/jit/vasm-print.cpp @@ -219,6 +219,10 @@ struct FormatVisitor { } } + void print(VregShiftExtend se) { + str << sep() << show(se); + } + const char* sep() { return comma ? ", " : (comma = true, ""); } const Vunit& unit; @@ -262,6 +266,58 @@ std::string show(const VregList& l) { ); } +std::string show(VregShiftExtend se) { + if (!se.isValid()) return "%?"; + + auto const regStr = show(se.reg); + auto renderShift = [&]() -> std::string { + auto const kind = se.shiftKind(); + auto const amount = static_cast(se.amount); + switch (kind) { + case vixl::NO_SHIFT: + return amount == 0 ? std::string{} : + folly::sformat(" no_shift #{}", amount); + case vixl::LSL: + if (amount == 0) return std::string{}; + return folly::sformat(" lsl #{}", amount); + case vixl::LSR: + return folly::sformat(" lsr #{}", amount); + case vixl::ASR: + return folly::sformat(" asr #{}", amount); + case vixl::ROR: + return folly::sformat(" ror #{}", amount); + } + not_reached(); + }; + auto renderExtend = [&]() -> std::string { + auto const kind = se.extendKind(); + auto const amount = static_cast(se.amount); + auto const name = [&]() -> const char* { + switch (kind) { + case vixl::NO_EXTEND: return "no_extend"; + case vixl::UXTB: return "uxtb"; + case vixl::UXTH: return "uxth"; + case vixl::UXTW: return "uxtw"; + case vixl::UXTX: return "uxtx"; + case vixl::SXTB: return "sxtb"; + case vixl::SXTH: return "sxth"; + case vixl::SXTW: return "sxtw"; + case vixl::SXTX: return "sxtx"; + } + not_reached(); + }(); + if (amount == 0) return folly::sformat(" {}", name); + return folly::sformat(" {} #{}", name, amount); + }; + + if (se.isShift()) { + auto const suffix = renderShift(); + if (suffix.empty()) return regStr; + return folly::sformat("{}{}", regStr, suffix); + } + return folly::sformat("{}{}", regStr, renderExtend()); +} + std::string show(Vptr p) { std::string str; switch(arch()) { diff --git a/hphp/runtime/vm/jit/vasm-print.h b/hphp/runtime/vm/jit/vasm-print.h index aa52a1b050de5..3e5278909b893 100644 --- a/hphp/runtime/vm/jit/vasm-print.h +++ b/hphp/runtime/vm/jit/vasm-print.h @@ -34,6 +34,7 @@ struct VregSet; /////////////////////////////////////////////////////////////////////////////// std::string show(Vreg r); +std::string show(VregShiftExtend se); std::string show(Vptr p); std::string show(Vconst c); std::string show(const VregSet&); diff --git a/hphp/runtime/vm/jit/vasm-reg.h b/hphp/runtime/vm/jit/vasm-reg.h index 14af93eae1ed0..4040db266d0f3 100644 --- a/hphp/runtime/vm/jit/vasm-reg.h +++ b/hphp/runtime/vm/jit/vasm-reg.h @@ -20,6 +20,7 @@ #include "hphp/runtime/vm/jit/vasm.h" #include "hphp/util/asm-x64.h" +#include "hphp/vixl/a64/constants-a64.h" #include @@ -1198,6 +1199,84 @@ std::string show(Width w); /////////////////////////////////////////////////////////////////////////////// +struct VregShiftExtend { + enum class Variant : uint8_t { + Shift, + Extend, + }; + + VregShiftExtend() + : reg{} + , variant(Variant::Shift) + , shift(vixl::LSL) + , amount(0) + {} + + explicit VregShiftExtend(Vreg r) + : reg(r) + , variant(Variant::Shift) + , shift(vixl::LSL) + , amount(0) + {} + + static VregShiftExtend makeShift(Vreg reg, vixl::Shift kind, uint8_t amount) { + assertx(amount <= 63); + VregShiftExtend result{reg}; + result.variant = Variant::Shift; + result.shift = kind; + result.amount = amount; + return result; + } + + static VregShiftExtend makeExtend(Vreg reg, vixl::Extend kind, uint8_t amount) { + assertx(amount <= 4); + VregShiftExtend result{reg}; + result.variant = Variant::Extend; + result.extend = kind; + result.amount = amount; + return result; + } + + bool isValid() const { return reg.isValid(); } + bool isShift() const { return variant == Variant::Shift; } + bool isExtend() const { return variant == Variant::Extend; } + bool operator==(const VregShiftExtend& other) const { + if (reg != other.reg || variant != other.variant || amount != other.amount) { + return false; + } + if (isShift()) return shift == other.shift; + return extend == other.extend; + } + + vixl::Shift shiftKind() const { + assertx(isShift()); + return shift; + } + + vixl::Extend extendKind() const { + assertx(isExtend()); + return extend; + } + + Vreg reg; + Variant variant; + union { + vixl::Shift shift; + vixl::Extend extend; + }; + uint8_t amount; +}; + +inline VregShiftExtend VregShift(Vreg reg, vixl::Shift kind, uint8_t amount) { + return VregShiftExtend::makeShift(reg, kind, amount); +} + +inline VregShiftExtend VregExtend(Vreg reg, vixl::Extend kind, uint8_t amount) { + return VregShiftExtend::makeExtend(reg, kind, amount); +} + +/////////////////////////////////////////////////////////////////////////////// + struct Vscaled { Vreg64 index; int scale; diff --git a/hphp/runtime/vm/jit/vasm-simplify-arm.cpp b/hphp/runtime/vm/jit/vasm-simplify-arm.cpp index cc7c363dba01e..13b269a49efd9 100644 --- a/hphp/runtime/vm/jit/vasm-simplify-arm.cpp +++ b/hphp/runtime/vm/jit/vasm-simplify-arm.cpp @@ -91,6 +91,186 @@ bool simplify(Env& env, const cmovq& inst, Vlabel b, size_t i) { /////////////////////////////////////////////////////////////////////////////// +struct OperandMatch { + VregShiftExtend operand; + size_t startIndex; + size_t len; +}; + +struct ExtendInfo { + Vreg reg; + vixl::Extend kind; +}; + +Optional match_extend_inst(const Vinstr& inst, + Width width, + Vreg dst) { + switch (inst.op) { + case Vinstr::movzbl: { + if (width != Width::Long) return {}; + auto const& mov = inst.movzbl_; + if (mov.d != dst) return {}; + return ExtendInfo{mov.s, vixl::UXTB}; + } + case Vinstr::movzwl: { + if (width != Width::Long) return {}; + auto const& mov = inst.movzwl_; + if (mov.d != dst) return {}; + return ExtendInfo{mov.s, vixl::UXTH}; + } + case Vinstr::movsbl: { + if (width != Width::Long) return {}; + auto const& mov = inst.movsbl_; + if (mov.d != dst) return {}; + return ExtendInfo{mov.s, vixl::SXTB}; + } + case Vinstr::movswl: { + if (width != Width::Long) return {}; + auto const& mov = inst.movswl_; + if (mov.d != dst) return {}; + return ExtendInfo{mov.s, vixl::SXTH}; + } + case Vinstr::movzbq: { + if (width != Width::Quad) return {}; + auto const& mov = inst.movzbq_; + if (mov.d != dst) return {}; + return ExtendInfo{mov.s, vixl::UXTB}; + } + case Vinstr::movzwq: { + if (width != Width::Quad) return {}; + auto const& mov = inst.movzwq_; + if (mov.d != dst) return {}; + return ExtendInfo{mov.s, vixl::UXTH}; + } + case Vinstr::movzlq: { + if (width != Width::Quad) return {}; + auto const& mov = inst.movzlq_; + if (mov.d != dst) return {}; + return ExtendInfo{mov.s, vixl::UXTW}; + } + case Vinstr::movsbq: { + if (width != Width::Quad) return {}; + auto const& mov = inst.movsbq_; + if (mov.d != dst) return {}; + return ExtendInfo{mov.s, vixl::SXTB}; + } + case Vinstr::movswq: { + if (width != Width::Quad) return {}; + auto const& mov = inst.movswq_; + if (mov.d != dst) return {}; + return ExtendInfo{mov.s, vixl::SXTH}; + } + case Vinstr::movslq: { + if (width != Width::Quad) return {}; + auto const& mov = inst.movslq_; + if (mov.d != dst) return {}; + return ExtendInfo{mov.s, vixl::SXTW}; + } + default: + break; + } + return {}; +} + +template +Optional match_operand_fold(Env& env, + Vlabel b, + size_t i, + Reg operand, + Width width, + bool allowExtend) { + if (i == 0) return {}; + + auto& code = env.unit.blocks[b].code; + auto const idx1 = i - 1; + auto const& inst1 = code[idx1]; + + auto const single_use = [&] (Vreg r) { + return env.use_counts[r] == 1; + }; + + auto const maybe_extend_before_shift = + [&] (auto const& shl, uint8_t amount, Width w) -> Optional { + if (!allowExtend) return {}; + if (idx1 == 0) return {}; + if (shl.d != operand) return {}; + auto const idx0 = idx1 - 1; + auto const& inst0 = code[idx0]; + auto const extend = match_extend_inst(inst0, w, shl.s1); + if (!extend) return {}; + if (!single_use(shl.s1)) return {}; + if (!single_use(shl.d) || env.use_counts[shl.sf] != 0) return {}; + if (amount > 4) return {}; + return OperandMatch{VregExtend(extend->reg, extend->kind, amount), idx0, 3}; + }; + + if (width == Width::Long && inst1.op == Vinstr::shlli) { + auto const& shl = inst1.shlli_; + auto const amount64 = shl.s0.l(); + always_assert_flog(amount64 >= 0 && amount64 <= 31, + "bad shift amount {} for shlli", amount64); + auto const amount = static_cast(amount64); + if (auto match = maybe_extend_before_shift(shl, amount, width)) { + return match; + } + if (shl.d == operand && single_use(shl.d) && env.use_counts[shl.sf] == 0) { + return OperandMatch{VregShift(shl.s1, vixl::LSL, amount), idx1, 2}; + } + } else if (width == Width::Quad && inst1.op == Vinstr::shlqi) { + auto const& shl = inst1.shlqi_; + auto const amount64 = shl.s0.q(); + always_assert_flog(amount64 >= 0 && amount64 <= 63, + "bad shift amount {} for shlqi", amount64); + auto const amount = static_cast(amount64); + if (auto match = maybe_extend_before_shift(shl, amount, width)) { + return match; + } + if (shl.d == operand && single_use(shl.d) && env.use_counts[shl.sf] == 0) { + return OperandMatch{VregShift(shl.s1, vixl::LSL, amount), idx1, 2}; + } + } + + if (allowExtend) { + if (auto extend = match_extend_inst(inst1, width, operand)) { + if (!single_use(operand)) return {}; + return OperandMatch{VregExtend(extend->reg, extend->kind, 0), idx1, 2}; + } + } + + return {}; +} + +template +bool fold_shift_operand(Env& env, + Vlabel b, + size_t i, + Width width, + Reg primary, + Reg secondary, + bool commutative, + bool allowExtend, + Emit emit) { + if (auto match = match_operand_fold(env, b, i, primary, width, allowExtend)) { + return simplify_impl(env, b, match->startIndex, [&] (Vout& v) { + emit(v, match->operand, secondary); + return match->len; + }); + } + + if (commutative) { + if (auto match = match_operand_fold(env, b, i, secondary, width, allowExtend)) { + return simplify_impl(env, b, match->startIndex, [&] (Vout& v) { + emit(v, match->operand, primary); + return match->len; + }); + } + } + + return false; +} + +/////////////////////////////////////////////////////////////////////////////// + bool simplify(Env& env, const loadb& inst, Vlabel b, size_t i) { if (if_inst(env, b, i + 1, [&] (const movzbl& mov) { // loadb{s, tmp}; movzbl{tmp, d}; -> loadzbl{s, d}; @@ -115,6 +295,128 @@ bool simplify(Env& env, const loadb& inst, Vlabel b, size_t i) { /////////////////////////////////////////////////////////////////////////////// +bool simplify(Env& env, const addl& inst, Vlabel b, size_t i) { + return fold_shift_operand(env, b, i, Width::Long, inst.s0, inst.s1, true, true, + [&](Vout& v, VregShiftExtend operand, Vreg32 other) { + v << addshiftl{operand, other, inst.d, inst.sf, inst.fl}; + }); +} + +bool simplify(Env& env, const addq& inst, Vlabel b, size_t i) { + return fold_shift_operand(env, b, i, Width::Quad, inst.s0, inst.s1, true, true, + [&](Vout& v, VregShiftExtend operand, Vreg64 other) { + v << addshiftq{operand, other, inst.d, inst.sf, inst.fl}; + }); +} + +bool simplify(Env& env, const andl& inst, Vlabel b, size_t i) { + return fold_shift_operand(env, b, i, Width::Long, inst.s0, inst.s1, true, false, + [&](Vout& v, VregShiftExtend operand, Vreg32 other) { + v << andshiftl{operand, other, inst.d, inst.sf, inst.fl}; + }); +} + +bool simplify(Env& env, const andq& inst, Vlabel b, size_t i) { + return fold_shift_operand(env, b, i, Width::Quad, inst.s0, inst.s1, true, false, + [&](Vout& v, VregShiftExtend operand, Vreg64 other) { + v << andshiftq{operand, other, inst.d, inst.sf, inst.fl}; + }); +} + +bool simplify(Env& env, const orq& inst, Vlabel b, size_t i) { + return fold_shift_operand(env, b, i, Width::Quad, inst.s0, inst.s1, true, false, + [&](Vout& v, VregShiftExtend operand, Vreg64 other) { + v << orshiftq{operand, other, inst.d, inst.sf, inst.fl}; + }); +} + +bool simplify(Env& env, const subl& inst, Vlabel b, size_t i) { + return fold_shift_operand(env, b, i, Width::Long, inst.s0, inst.s1, false, true, + [&](Vout& v, VregShiftExtend operand, Vreg32 other) { + v << subshiftl{operand, other, inst.d, inst.sf, inst.fl}; + }); +} + +bool simplify(Env& env, const subq& inst, Vlabel b, size_t i) { + return fold_shift_operand(env, b, i, Width::Quad, inst.s0, inst.s1, false, true, + [&](Vout& v, VregShiftExtend operand, Vreg64 other) { + v << subshiftq{operand, other, inst.d, inst.sf, inst.fl}; + }); +} + +bool simplify(Env& env, const cmpl& inst, Vlabel b, size_t i) { + return fold_shift_operand(env, b, i, Width::Long, inst.s0, inst.s1, false, true, + [&](Vout& v, VregShiftExtend operand, Vreg32 other) { + v << cmpshiftl{operand, other, inst.sf, inst.fl}; + }); +} + +bool simplify(Env& env, const cmpq& inst, Vlabel b, size_t i) { + return fold_shift_operand(env, b, i, Width::Quad, inst.s0, inst.s1, false, true, + [&](Vout& v, VregShiftExtend operand, Vreg64 other) { + v << cmpshiftq{operand, other, inst.sf, inst.fl}; + }); +} + +bool simplify(Env& env, const testl& inst, Vlabel b, size_t i) { + return fold_shift_operand(env, b, i, Width::Long, inst.s0, inst.s1, true, false, + [&](Vout& v, VregShiftExtend operand, Vreg32 other) { + v << testshiftl{operand, other, inst.sf, inst.fl}; + }); +} + +bool simplify(Env& env, const testq& inst, Vlabel b, size_t i) { + return fold_shift_operand(env, b, i, Width::Quad, inst.s0, inst.s1, true, false, + [&](Vout& v, VregShiftExtend operand, Vreg64 other) { + v << testshiftq{operand, other, inst.sf, inst.fl}; + }); +} + +bool simplify(Env& env, const xorl& inst, Vlabel b, size_t i) { + return fold_shift_operand(env, b, i, Width::Long, inst.s0, inst.s1, true, false, + [&](Vout& v, VregShiftExtend operand, Vreg32 other) { + v << xorshiftl{operand, other, inst.d, inst.sf, inst.fl}; + }); +} + +bool simplify(Env& env, const xorq& inst, Vlabel b, size_t i) { + return fold_shift_operand(env, b, i, Width::Quad, inst.s0, inst.s1, true, false, + [&](Vout& v, VregShiftExtend operand, Vreg64 other) { + v << xorshiftq{operand, other, inst.d, inst.sf, inst.fl}; + }); +} + +bool simplify(Env& env, const lea& inst, Vlabel b, size_t i) { + auto const ptr = inst.s; + if (!ptr.index.isValid()) return false; + + auto const tryMatch = match_operand_fold(env, b, i, + Vreg64{ptr.index}, Width::Quad, false); + if (!tryMatch) return false; + + auto const operand = tryMatch->operand; + if (!operand.isShift() || operand.shiftKind() != vixl::LSL) return false; + + auto const amount = operand.amount; + auto const newScale = static_cast(ptr.scale) << amount; + if (newScale == 0 || newScale > 8 || (newScale & (newScale - 1))) return false; + + auto const emission = simplify_impl(env, b, tryMatch->startIndex, + [&] (Vout& v) { + auto newPtr = ptr; + newPtr.index = Vreg64{operand.reg}; + newPtr.scale = static_cast(newScale); + auto newLea = inst; + newLea.s = newPtr; + v << newLea; + return tryMatch->len; + }); + + return emission; +} + +/////////////////////////////////////////////////////////////////////////////// + bool simplify(Env& env, const ldimmq& inst, Vlabel b, size_t i) { return if_inst(env, b, i + 1, [&] (const lea& ea) { // ldimmq{s, index}; lea{base[index], d} -> lea{base[s],d} diff --git a/hphp/runtime/vm/jit/vasm-visit.h b/hphp/runtime/vm/jit/vasm-visit.h index c1c0d69dba319..4a3fe2b28c15d 100644 --- a/hphp/runtime/vm/jit/vasm-visit.h +++ b/hphp/runtime/vm/jit/vasm-visit.h @@ -68,6 +68,14 @@ void visit(const Vunit&, Vptr p, F f) { if (p.index.isValid()) detail::invoke(f, p.index, width(p.index)); } +template +void visit(const Vunit&, VregShiftExtend se, F f) { + if (!se.reg.isValid()) return; + auto w = width(se.reg); + if (w == Width::AnyNF) w = Width::Quad; + detail::invoke(f, se.reg, w); +} + template void visit(const Vunit& unit, Vtuple t, F f) { for (auto r : unit.tuples[t]) detail::invoke(f, r, width(r)); @@ -276,6 +284,9 @@ struct MutableRegVisitor { } template void use(Vr& m) { Vreg r = m; use(r); m = r; } void use(Vreg& r) { r = u((Vreg)r); } + void use(VregShiftExtend& se) { + if (se.reg.isValid()) use(se.reg); + } void def(RegSet r) const { r = rd(r); } void def(Vtuple defs) { for (auto& r : unit.tuples[defs]) def(r); } diff --git a/hphp/runtime/vm/jit/vasm-xls.cpp b/hphp/runtime/vm/jit/vasm-xls.cpp index 25fcd551f2514..b15a565e52938 100644 --- a/hphp/runtime/vm/jit/vasm-xls.cpp +++ b/hphp/runtime/vm/jit/vasm-xls.cpp @@ -69,6 +69,7 @@ Constraint constraint(const Vreg8&) { return Constraint::Gpr; } Constraint constraint(const VregDbl&) { return Constraint::Simd; } Constraint constraint(const Vreg128&) { return Constraint::Simd; } Constraint constraint(const VregSF&) { return Constraint::Sf; } +Constraint constraint(const VregShiftExtend&) { return Constraint::Gpr; } bool is_wide(const Vreg128&) { return true; } template bool is_wide(const T&) { return false; } @@ -952,6 +953,13 @@ struct UseVisitor { var->ivl()->uses.push_back({kind, m_range.end, hint}); } } + void use(VregShiftExtend se, + Constraint kind, + unsigned end, + Vreg hint = Vreg{}) { + if (!se.reg.isValid()) return; + use(Vreg{se.reg}, kind, end, hint); + } private: const Vunit& m_unit; @@ -2293,6 +2301,9 @@ struct Renamer { if (m.base.isValid()) rename(m.base); if (m.index.isValid()) rename(m.index); } + void across(VregShiftExtend& se) { + if (se.reg.isValid()) rename(se.reg); + } template void across(Vp& m) { across(static_cast(m)); } void def(RegSet) {} @@ -2301,6 +2312,9 @@ struct Renamer { if (m.base.isValid()) rename(m.base); if (m.index.isValid()) rename(m.index); } + void use(VregShiftExtend& se) { + if (se.reg.isValid()) rename(se.reg); + } template void use(Vp& m) { use(static_cast(m)); } void use(VcallArgsId) { always_assert(false && "vcall unsupported in vxls"); } diff --git a/hphp/runtime/vm/jit/vasm.cpp b/hphp/runtime/vm/jit/vasm.cpp index 83f7ce974c8e2..6a8b20e566109 100644 --- a/hphp/runtime/vm/jit/vasm.cpp +++ b/hphp/runtime/vm/jit/vasm.cpp @@ -124,6 +124,11 @@ struct FpVisit { >::type use(T& reg) { if (reg == livefp) reg = rvmfp(); } + + void use(VregShiftExtend& se) { + if (!se.reg.isValid()) return; + use(se.reg); + } }; void fixupVmfpUses(Vunit& unit) {