|
18 | 18 | namespace mlir::heir::rotom { |
19 | 19 | namespace { |
20 | 20 |
|
21 | | -/// Maps a `#rotom.dim` from the layout's `dims` list to its iterator index `i*` |
22 | | -/// after preprocessing (match logical axis, size, and stride). |
23 | | -static FailureOr<int64_t> traversalIndexForRotomDim( |
24 | | - const SmallVector<DimAttr>& traversalDims, DimAttr want) { |
| 21 | +/// Maps a tensor axis id to its iterator index `i*` after preprocessing. |
| 22 | +/// Preprocessing dedupes traversal dims per logical axis (a mixed-radix split |
| 23 | +/// contributes one variable), so the axis id identifies the variable. |
| 24 | +static FailureOr<int64_t> traversalIndexForAxis( |
| 25 | + const SmallVector<DimAttr>& traversalDims, int64_t axis) { |
25 | 26 | for (int64_t i = 0; i < static_cast<int64_t>(traversalDims.size()); ++i) { |
26 | | - if (traversalDims[i].getDim() == want.getDim() && |
27 | | - traversalDims[i].getSize() == want.getSize() && |
28 | | - traversalDims[i].getStride() == want.getStride()) { |
29 | | - return i; |
30 | | - } |
| 27 | + if (traversalDims[i].getDim() == axis) return i; |
31 | 28 | } |
32 | 29 | return failure(); |
33 | 30 | } |
34 | 31 |
|
| 32 | +/// Product of the extents of `dim`'s pieces in the layout's dims list (the |
| 33 | +/// logical axis extent; a single whole-dim piece gives its own size). |
| 34 | +static int64_t fullExtentOfDim(ArrayAttr rotomDims, int64_t dim) { |
| 35 | + int64_t extent = 1; |
| 36 | + for (Attribute a : rotomDims) { |
| 37 | + auto d = cast<DimAttr>(a); |
| 38 | + if (!d.isGap() && !d.isReplicate() && d.getDim() == dim) { |
| 39 | + extent *= d.getSize(); |
| 40 | + } |
| 41 | + } |
| 42 | + return extent; |
| 43 | +} |
| 44 | + |
| 45 | +/// Whether `dim` is packed as more than one mixed-radix piece. |
| 46 | +static bool isSplitDim(ArrayAttr rotomDims, int64_t dim) { |
| 47 | + int64_t count = 0; |
| 48 | + for (Attribute a : rotomDims) { |
| 49 | + auto d = cast<DimAttr>(a); |
| 50 | + if (!d.isGap() && !d.isReplicate() && d.getDim() == dim) ++count; |
| 51 | + } |
| 52 | + return count > 1; |
| 53 | +} |
| 54 | + |
35 | 55 | static std::string modExpr(llvm::StringRef expr, int64_t mod) { |
36 | 56 | std::string out; |
37 | 57 | llvm::raw_string_ostream os(out); |
@@ -84,7 +104,7 @@ static LogicalResult emitSegmentAddress( |
84 | 104 | const SmallVector<DimAttr>& replicationDims, |
85 | 105 | int64_t numActiveTraversalComponents, size_t segStart, size_t segEnd, |
86 | 106 | const llvm::DenseSet<int64_t>& rolledGapIndices, ArrayRef<int64_t> rolls, |
87 | | - ArrayAttr rotomDims) { |
| 107 | + ArrayRef<int64_t> rollSteps, ArrayAttr rotomDims) { |
88 | 108 | // Effective extent of a piece = the range of its mixed-radix digit: modBy |
89 | 109 | // when the piece reads i mod L (the slot/low part), else sz/divBy (the whole |
90 | 110 | // dim when divBy 1, or the ct/high part sz/L of a straddling dim). |
@@ -166,56 +186,117 @@ static LogicalResult emitSegmentAddress( |
166 | 186 | traversalExprs.push_back("i" + std::to_string(i)); |
167 | 187 | } |
168 | 188 |
|
169 | | - // Apply roll(a,b) transforms left-to-right: |
170 | | - // t_a <- (t_a - t_b) mod extent(a). |
| 189 | + // Apply rolls left-to-right. A piece FROM rewrites that piece's own |
| 190 | + // mixed-radix digit in place, (digit - step * shift) mod extent(piece), |
| 191 | + // leaving its axis's other digits untouched; an `axis` FROM rewrites the |
| 192 | + // whole axis index mod its full extent, so the shift borrows across digits |
| 193 | + // (each piece then takes its digit of the rolled index). The shift is the |
| 194 | + // by endpoint's index: a traversal piece's digit (its whole index when the |
| 195 | + // axis is unsplit), a whole axis via `axis`, a slot replication's replica |
| 196 | + // index, or a gap's block index. A step multiplies the shift -- the |
| 197 | + // baby-step/giant-step pre-rotation runs against the roll's subtraction |
| 198 | + // direction, so its sign cannot be folded into the endpoint structure. |
171 | 199 | // |
172 | | - // The rewrite lands wherever the FROM dimension sits -- the ciphertext |
173 | | - // address, the slot address, or both when it straddles the boundary. BY is |
174 | | - // another traversal dimension on either axis, or a slot replication/gap |
175 | | - // block index, so a roll diagonalizes a ciphertext dimension against a slot |
176 | | - // one (one ciphertext per diagonal) or two slot dimensions (a Halevi-Shoup |
177 | | - // slot diagonal); a roll within the ciphertext axis is a free ciphertext |
| 200 | + // The rewrite lands wherever the FROM pieces sit -- the ciphertext |
| 201 | + // address, the slot address, or both when the axis straddles the boundary. |
| 202 | + // A roll diagonalizes a ciphertext dimension against a slot one (one |
| 203 | + // ciphertext per diagonal) or two slot dimensions (a Halevi-Shoup slot |
| 204 | + // diagonal); a roll within the ciphertext axis is a free ciphertext |
178 | 205 | // relabeling that enumeration never generates. |
179 | 206 | if (!rolls.empty()) { |
180 | 207 | if (!rotomDims || rolls.size() % 2 != 0) return failure(); |
181 | 208 | for (size_t i = 0; i < rolls.size(); i += 2) { |
182 | | - const int64_t fromIdx = rolls[i]; |
183 | | - const int64_t toIdx = rolls[i + 1]; |
184 | | - if (fromIdx < 0 || toIdx < 0 || |
185 | | - fromIdx >= static_cast<int64_t>(rotomDims.size()) || |
186 | | - toIdx >= static_cast<int64_t>(rotomDims.size())) { |
| 209 | + const RollEndpoint from = decodeRollEndpoint(rolls[i]); |
| 210 | + const RollEndpoint by = decodeRollEndpoint(rolls[i + 1]); |
| 211 | + const int64_t step = i / 2 < rollSteps.size() ? rollSteps[i / 2] : 1; |
| 212 | + if (!from.isAxis && from.index >= static_cast<int64_t>(rotomDims.size())) |
187 | 213 | return failure(); |
188 | | - } |
189 | | - auto fromDim = dyn_cast<DimAttr>(rotomDims[fromIdx]); |
190 | | - auto toDim = dyn_cast<DimAttr>(rotomDims[toIdx]); |
191 | | - if (!fromDim || !toDim) return failure(); |
| 214 | + if (!by.isAxis && by.index >= static_cast<int64_t>(rotomDims.size())) |
| 215 | + return failure(); |
| 216 | + DimAttr fromPiece = |
| 217 | + from.isAxis ? DimAttr() : dyn_cast<DimAttr>(rotomDims[from.index]); |
| 218 | + if (!from.isAxis && !fromPiece) return failure(); |
| 219 | + const int64_t fromAxis = from.isAxis ? from.index : fromPiece.getDim(); |
192 | 220 | FailureOr<int64_t> maybeFromTrav = |
193 | | - traversalIndexForRotomDim(traversalDims, fromDim); |
| 221 | + traversalIndexForAxis(traversalDims, fromAxis); |
194 | 222 | if (failed(maybeFromTrav)) return failure(); |
195 | 223 | const int64_t fromTrav = *maybeFromTrav; |
| 224 | + |
196 | 225 | std::string toExpr; |
197 | | - if (toDim.isGap()) { |
198 | | - // Rolling by a gap dim: the shift is the gap's existential block |
199 | | - // index, so block g holds the rolled dim cyclically shifted by g. |
200 | | - toExpr = "g" + std::to_string(gapIndexForPosition(rotomDims, toIdx)); |
201 | | - } else if (toDim.isReplicate()) { |
202 | | - // Rolling by a replication dim: the shift is the replica index, whose |
203 | | - // existential variable is named after the piece's replication slot. |
204 | | - int64_t replicationIndex = 0; |
205 | | - for (int64_t k = 0; k < toIdx; ++k) { |
206 | | - if (cast<DimAttr>(rotomDims[k]).isReplicate()) ++replicationIndex; |
207 | | - } |
208 | | - toExpr = "d" + std::to_string(numActiveTraversalComponents + |
209 | | - replicationIndex); |
210 | | - } else { |
| 226 | + if (by.isAxis) { |
| 227 | + // Rolling by a whole axis: the shift is the axis's (possibly |
| 228 | + // already-rolled) index expression. |
211 | 229 | FailureOr<int64_t> maybeToTrav = |
212 | | - traversalIndexForRotomDim(traversalDims, toDim); |
| 230 | + traversalIndexForAxis(traversalDims, by.index); |
213 | 231 | if (failed(maybeToTrav)) return failure(); |
214 | 232 | toExpr = traversalExprs[*maybeToTrav]; |
| 233 | + } else { |
| 234 | + auto toDim = dyn_cast<DimAttr>(rotomDims[by.index]); |
| 235 | + if (!toDim) return failure(); |
| 236 | + if (toDim.isGap()) { |
| 237 | + // Rolling by a gap dim: the shift is the gap's existential block |
| 238 | + // index, so block g holds the rolled index cyclically shifted by g. |
| 239 | + toExpr = |
| 240 | + "g" + std::to_string(gapIndexForPosition(rotomDims, by.index)); |
| 241 | + } else if (toDim.isReplicate()) { |
| 242 | + // Rolling by a replication dim: the shift is the replica index, |
| 243 | + // whose existential variable is named after the piece's replication |
| 244 | + // slot. |
| 245 | + int64_t replicationIndex = 0; |
| 246 | + for (int64_t k = 0; k < by.index; ++k) { |
| 247 | + if (cast<DimAttr>(rotomDims[k]).isReplicate()) ++replicationIndex; |
| 248 | + } |
| 249 | + toExpr = "d" + std::to_string(numActiveTraversalComponents + |
| 250 | + replicationIndex); |
| 251 | + } else { |
| 252 | + FailureOr<int64_t> maybeToTrav = |
| 253 | + traversalIndexForAxis(traversalDims, toDim.getDim()); |
| 254 | + if (failed(maybeToTrav)) return failure(); |
| 255 | + toExpr = traversalExprs[*maybeToTrav]; |
| 256 | + if (isSplitDim(rotomDims, toDim.getDim())) { |
| 257 | + // The by piece's digit: (i / stride) mod extent, with the modulus |
| 258 | + // redundant on the most-significant digit. |
| 259 | + const int64_t fullTo = fullExtentOfDim(rotomDims, toDim.getDim()); |
| 260 | + if (toDim.getStride() > 1) { |
| 261 | + toExpr = floorDivExpr(toExpr, toDim.getStride()); |
| 262 | + } |
| 263 | + if (toDim.getStride() * toDim.getSize() < fullTo) { |
| 264 | + toExpr = modExpr(toExpr, toDim.getSize()); |
| 265 | + } |
| 266 | + } |
| 267 | + } |
| 268 | + } |
| 269 | + |
| 270 | + if (step != 1) { |
| 271 | + toExpr = std::to_string(step) + " * (" + toExpr + ")"; |
| 272 | + } |
| 273 | + |
| 274 | + if (from.isAxis || !isSplitDim(rotomDims, fromAxis)) { |
| 275 | + // Whole-axis rewrite (the piece spelling of an unsplit axis is the |
| 276 | + // same operation: its one piece's digit is the axis index). |
| 277 | + std::string diffExpr = |
| 278 | + "(" + traversalExprs[fromTrav] + " - " + toExpr + ")"; |
| 279 | + traversalExprs[fromTrav] = |
| 280 | + modExpr(diffExpr, fullExtentOfDim(rotomDims, fromAxis)); |
| 281 | + } else { |
| 282 | + // Piece rewrite on a split axis: replace this piece's digit with |
| 283 | + // (digit - step * shift) mod extent(piece), i.e. add |
| 284 | + // stride * (newDigit - digit) to the axis index. No borrow crosses |
| 285 | + // into the other digits. |
| 286 | + const int64_t stride = fromPiece.getStride(); |
| 287 | + const int64_t extent = fromPiece.getSize(); |
| 288 | + const std::string axisExpr = traversalExprs[fromTrav]; |
| 289 | + std::string digit = axisExpr; |
| 290 | + if (stride > 1) digit = floorDivExpr(digit, stride); |
| 291 | + if (stride * extent < fullExtentOfDim(rotomDims, fromAxis)) { |
| 292 | + digit = modExpr(digit, extent); |
| 293 | + } |
| 294 | + const std::string newDigit = |
| 295 | + modExpr("(" + digit + " - " + toExpr + ")", extent); |
| 296 | + traversalExprs[fromTrav] = |
| 297 | + "(" + axisExpr + " + " + std::to_string(stride) + " * (" + |
| 298 | + newDigit + ") - " + std::to_string(stride) + " * (" + digit + "))"; |
215 | 299 | } |
216 | | - std::string diffExpr = |
217 | | - "(" + traversalExprs[fromTrav] + " - " + toExpr + ")"; |
218 | | - traversalExprs[fromTrav] = modExpr(diffExpr, fromDim.getSize()); |
219 | 300 | } |
220 | 301 | } |
221 | 302 |
|
@@ -257,7 +338,7 @@ static FailureOr<std::string> emitSplitCtSlotIsl( |
257 | 338 | const SmallVector<DimAttr>& replicationDims, |
258 | 339 | const SmallVector<DimAttr>& gapDims, int64_t numTraversalComponents, |
259 | 340 | int64_t numReplication, int64_t numGap, ArrayRef<int64_t> rolls, |
260 | | - ArrayAttr rotomDims) { |
| 341 | + ArrayRef<int64_t> rollSteps, ArrayAttr rotomDims) { |
261 | 342 | if (prefix > pieces.size()) return failure(); |
262 | 343 |
|
263 | 344 | const llvm::DenseSet<int64_t> rolledGapIndices = |
@@ -332,20 +413,21 @@ static FailureOr<std::string> emitSplitCtSlotIsl( |
332 | 413 | emitAnd(); |
333 | 414 | bool firstTerm = true; |
334 | 415 | os << "ct = "; |
335 | | - if (failed(emitSegmentAddress(os, firstTerm, pieces, pieceIndex, pieceDivBy, |
336 | | - pieceModBy, traversalDims, gapDims, |
337 | | - replicationDims, numTraversalComponents, 0, |
338 | | - prefix, rolledGapIndices, rolls, rotomDims))) |
| 416 | + if (failed(emitSegmentAddress( |
| 417 | + os, firstTerm, pieces, pieceIndex, pieceDivBy, pieceModBy, |
| 418 | + traversalDims, gapDims, replicationDims, numTraversalComponents, 0, |
| 419 | + prefix, rolledGapIndices, rolls, rollSteps, rotomDims))) |
339 | 420 | return failure(); |
340 | 421 | if (firstTerm) os << "0"; |
341 | 422 |
|
342 | 423 | emitAnd(); |
343 | 424 | firstTerm = true; |
344 | 425 | os << "slot = "; |
345 | | - if (failed(emitSegmentAddress( |
346 | | - os, firstTerm, pieces, pieceIndex, pieceDivBy, pieceModBy, |
347 | | - traversalDims, gapDims, replicationDims, numTraversalComponents, |
348 | | - prefix, pieces.size(), rolledGapIndices, rolls, rotomDims))) |
| 426 | + if (failed(emitSegmentAddress(os, firstTerm, pieces, pieceIndex, pieceDivBy, |
| 427 | + pieceModBy, traversalDims, gapDims, |
| 428 | + replicationDims, numTraversalComponents, prefix, |
| 429 | + pieces.size(), rolledGapIndices, rolls, |
| 430 | + rollSteps, rotomDims))) |
349 | 431 | return failure(); |
350 | 432 | if (firstTerm) os << "0"; |
351 | 433 |
|
@@ -388,10 +470,14 @@ static FailureOr<std::string> lowerToIslImpl(LayoutAttr layout) { |
388 | 470 | DenseI64ArrayAttr rollsAttr = layout.getRolls(); |
389 | 471 | ArrayRef<int64_t> rolls = |
390 | 472 | rollsAttr ? rollsAttr.asArrayRef() : ArrayRef<int64_t>{}; |
| 473 | + DenseI64ArrayAttr stepsAttr = layout.getRollSteps(); |
| 474 | + ArrayRef<int64_t> rollSteps = |
| 475 | + stepsAttr ? stepsAttr.asArrayRef() : ArrayRef<int64_t>{}; |
391 | 476 | return emitSplitCtSlotIsl( |
392 | 477 | data.n, data.ctPrefixLen, data.pieces, data.pieceIndex, data.pieceDivBy, |
393 | 478 | data.pieceModBy, data.traversalDims, data.replicationDims, data.gapDims, |
394 | | - numTraversalComponents, numReplication, numGap, rolls, layout.getDims()); |
| 479 | + numTraversalComponents, numReplication, numGap, rolls, rollSteps, |
| 480 | + layout.getDims()); |
395 | 481 | } |
396 | 482 |
|
397 | 483 | } // namespace |
|
0 commit comments