From 69c6c65131cf8304ccea760b17f68ae99888ec2b Mon Sep 17 00:00:00 2001 From: Alan Rojas Date: Mon, 20 Apr 2026 14:09:36 -0400 Subject: [PATCH 1/2] fix: mark stake transactions correctly when they land more than one block after broadcast - Poll verifyTransaction across the full TX_EXPIRATION_BLOCKS window via Temporal's activity retry policy (one attempt per ~block) instead of a single post-waitForNextBlock check, so txs that land two or more blocks after broadcast are no longer mis-marked as failure - Make verifyTransaction throw retriable TX_NOT_FOUND when the hash is not on-chain, and split the supplier-state fallback into a dedicated checkSupplierOnChain activity that only runs after retries are exhausted (positive-only: missing supplier stays Failure) - Discriminate the verify catch with isTxNotFoundFailure so RPC outages and other unexpected errors fail the workflow loudly (tx stays Pending) rather than being silently converted to Failure on insufficient evidence - Cap the Tier 3 block scan at the current chain head so future heights stop consuming the scan budget; the workflow-level retry drives forward progress instead - Persist code on every verification update and write a log that distinguishes chain rejection ("verification failed with code N"), supplier fallback success, and true TX_NOT_FOUND after retries --- .../src/activities/index.ts | 37 ++++---- .../src/lib/blockchain/index.ts | 9 +- .../src/workflows/ExecuteTransaction.ts | 87 +++++++++++++++++-- 3 files changed, 109 insertions(+), 24 deletions(-) diff --git a/apps/middleman-workflows/src/activities/index.ts b/apps/middleman-workflows/src/activities/index.ts index dc750f7b..36881647 100644 --- a/apps/middleman-workflows/src/activities/index.ts +++ b/apps/middleman-workflows/src/activities/index.ts @@ -455,28 +455,33 @@ export const delegatorActivities = (dal: DAL, pocketRpcClient: PocketBlockchain, /** * Verifies the transaction status by the given transaction hash. * - * @param {string} hash - The hash of the transaction to be verified. - * @return {Promise} A promise that resolves to a tuple containing the success status (boolean), the transaction code (number), and the gas used (string). Throws an error if the transaction data is incomplete or not found. + * Returns `[success, code, gasUsed]` when the tx is found on-chain. Throws a retriable + * `ApplicationFailure` when the tx is not (yet) found — the workflow's activity retry + * policy handles re-checking across blocks until the expiration window closes. */ - async verifyTransaction(hash: string, height?: number, operatorAddress?: string) { + async verifyTransaction( + hash: string, + height?: number, + ): Promise { const tx = await pocketRpcClient.getTransaction(hash, height) if (tx) { return [tx.success, tx.code, tx.gasUsed?.toString() || '0'] as const } - // Tier 4: all lookup methods failed — check supplier state directly - if (operatorAddress) { - log.warn('TX not found via any method, checking supplier state', { hash, operatorAddress }) - const supplier = await pocketRpcClient.getSupplier(operatorAddress) - if (supplier) { - log.info('Supplier exists on-chain, marking TX as success', { hash, operatorAddress }) - return [true, 0, '0'] as const - } - log.warn('Supplier not found on-chain, marking TX as failure', { hash, operatorAddress }) - return [false, -1, '0'] as const - } - - throw new Error('Transaction data is incomplete or not found') + throw ApplicationFailure.retryable( + 'Transaction not found on-chain', + 'TX_NOT_FOUND', + { hash, height }, + ) + }, + /** + * Checks whether a supplier exists on-chain with the given operator address. + * Used as a Tier 4 positive-only fallback for Stake transactions when `verifyTransaction` + * exhausts its retries without finding the tx hash. + */ + async checkSupplierOnChain(operatorAddress: string): Promise { + const supplier = await pocketRpcClient.getSupplier(operatorAddress) + return !!supplier }, /** * Creates new nodes based on the data extracted from a provided transaction. diff --git a/apps/middleman-workflows/src/lib/blockchain/index.ts b/apps/middleman-workflows/src/lib/blockchain/index.ts index 8ff5b8be..1f533a8a 100644 --- a/apps/middleman-workflows/src/lib/blockchain/index.ts +++ b/apps/middleman-workflows/src/lib/blockchain/index.ts @@ -148,7 +148,14 @@ export class Blockchain implements IBlockchain { const maxBlocks = 30; try { const comet = await connectComet(this.rpcUrl); - for (let h = height; h < height + maxBlocks; h++) { + const status = await comet.status(); + const latestHeight = status.syncInfo.latestBlockHeight; + // Only scan blocks that already exist on-chain. Future heights throw and would + // waste the scan budget; the workflow-level poll loop handles waiting for new + // blocks. Tier 3's role is to recover from a lagging/broken tx_index within + // already-produced blocks, not to wait for the chain to advance. + const endHeight = Math.min(height + maxBlocks, latestHeight); + for (let h = height; h <= endHeight; h++) { try { const block = await comet.block(h); const txs = block.block.txs; diff --git a/apps/middleman-workflows/src/workflows/ExecuteTransaction.ts b/apps/middleman-workflows/src/workflows/ExecuteTransaction.ts index dc956304..a4567869 100644 --- a/apps/middleman-workflows/src/workflows/ExecuteTransaction.ts +++ b/apps/middleman-workflows/src/workflows/ExecuteTransaction.ts @@ -1,4 +1,6 @@ import { + ActivityFailure, + ApplicationFailure, proxyActivities, WorkflowError, } from '@temporalio/workflow' @@ -7,11 +9,26 @@ import { TransactionStatus, TransactionType } from '@igniter/db/middleman/enums' import {SendTransactionResult} from "@/lib/blockchain"; const TX_EXPIRATION_BLOCKS = 30 +const TX_NOT_FOUND_ERROR_TYPE = 'TX_NOT_FOUND' interface TransactionArgs { transactionId: number; } +/** + * Returns true when `err` is the retries-exhausted wrapper around the retriable + * `TX_NOT_FOUND` ApplicationFailure thrown by the `verifyTransaction` activity. + * Any other failure (RPC unreachable, deserialization, bugs) returns false and + * should be rethrown so the workflow fails loudly rather than silently marking + * the tx as failure on insufficient evidence. + */ +function isTxNotFoundFailure(err: unknown): boolean { + if (err instanceof ActivityFailure && err.cause instanceof ApplicationFailure) { + return err.cause.type === TX_NOT_FOUND_ERROR_TYPE + } + return false +} + /** * Extracts the operator address from a transaction's unsigned payload. * Used for Tier 4 fallback (supplier state check) when TX lookup fails. @@ -44,7 +61,7 @@ export async function ExecuteTransaction(args: TransactionArgs) { updateTransaction, executeTransaction, getBlockHeight, - verifyTransaction, + checkSupplierOnChain, createNewNodesFromTransaction, notifyProviderOfStakedAddresses, notifyProviderOfFailedStakes, @@ -57,6 +74,18 @@ export async function ExecuteTransaction(args: TransactionArgs) { }, }); + // verifyTransaction polls the chain on its own retry schedule: one attempt per block + // (pocket block time ≈ 1 min) up to TX_EXPIRATION_BLOCKS, matching the on-chain + // mempool expiration window. Throws retriable `TX_NOT_FOUND` until the tx lands. + const { verifyTransaction } = proxyActivities>({ + startToCloseTimeout: "30s", + retry: { + initialInterval: "60s", + backoffCoefficient: 1, + maximumAttempts: TX_EXPIRATION_BLOCKS, + }, + }); + const transaction = await getTransaction(transactionId); if (transaction.status !== TransactionStatus.Pending) { @@ -127,20 +156,64 @@ export async function ExecuteTransaction(args: TransactionArgs) { await waitForNextBlock(txHeight); } - const [success, code, gasUsed] = await verifyTransaction( - result?.transactionHash || transaction.hash!, - transaction.executionHeight || txHeight, - extractOperatorAddress(transaction), - ); + const txHash = result?.transactionHash || transaction.hash!; + const baseHeight = transaction.executionHeight || txHeight; + const operatorAddress = extractOperatorAddress(transaction); - const txStatus = success ? TransactionStatus.Success : TransactionStatus.Failure; + let success = false; + let code = -1; + let gasUsed = '0'; + let txFoundOnChain = false; + let supplierFallbackHit = false; + try { + // Retries are driven by Temporal's activity retry policy — one attempt per block + // up to TX_EXPIRATION_BLOCKS. A found tx returns the tuple; a missing tx throws + // retriable TX_NOT_FOUND until the policy is exhausted. + [success, code, gasUsed] = await verifyTransaction(txHash, baseHeight); + txFoundOnChain = true; + } catch (err) { + // Only fall through to Tier 4 when retries were exhausted specifically with + // TX_NOT_FOUND. Any other failure (RPC unreachable, bug, unexpected shape) is + // rethrown so the workflow fails and the tx stays Pending for human triage — + // avoids false "failure" marks when we lack evidence either way. + if (!isTxNotFoundFailure(err)) { + throw err; + } + + // Retries exhausted. Tier 4: check supplier state directly. Only a positive result + // (supplier on-chain) is conclusive; a missing supplier keeps the tx marked failed. + if (operatorAddress) { + const supplierExists = await checkSupplierOnChain(operatorAddress); + if (supplierExists) { + success = true; + code = 0; + gasUsed = '0'; + supplierFallbackHit = true; + } + } + } + + const txStatus = success ? TransactionStatus.Success : TransactionStatus.Failure; const verificationHeight = await getBlockHeight(); + let verificationLog: string | undefined; + if (txFoundOnChain) { + if (code !== 0) { + verificationLog = `verification failed with code ${code}`; + } + } else if (supplierFallbackHit) { + verificationLog = 'verified via supplier state fallback (tx hash not found)'; + } else { + verificationLog = `tx not found on-chain after ${TX_EXPIRATION_BLOCKS} retries (baseHeight=${baseHeight})`; + } + await updateTransaction(transactionId, { status: txStatus, verificationHeight, consumedFee: Number(gasUsed || 0), + code, + log: verificationLog, }); if (transaction.type === TransactionType.Stake) { From e7ae6521a94456a2ce2344df9259cd29005cd325 Mon Sep 17 00:00:00 2001 From: Alan Rojas Date: Tue, 21 Apr 2026 11:35:38 -0400 Subject: [PATCH 2/2] marking transaction as failure if we couldn't verify it after 30 blocks --- .../src/workflows/ExecuteTransaction.ts | 45 +++++-------------- 1 file changed, 11 insertions(+), 34 deletions(-) diff --git a/apps/middleman-workflows/src/workflows/ExecuteTransaction.ts b/apps/middleman-workflows/src/workflows/ExecuteTransaction.ts index a4567869..65ef60a1 100644 --- a/apps/middleman-workflows/src/workflows/ExecuteTransaction.ts +++ b/apps/middleman-workflows/src/workflows/ExecuteTransaction.ts @@ -1,6 +1,4 @@ import { - ActivityFailure, - ApplicationFailure, proxyActivities, WorkflowError, } from '@temporalio/workflow' @@ -9,26 +7,11 @@ import { TransactionStatus, TransactionType } from '@igniter/db/middleman/enums' import {SendTransactionResult} from "@/lib/blockchain"; const TX_EXPIRATION_BLOCKS = 30 -const TX_NOT_FOUND_ERROR_TYPE = 'TX_NOT_FOUND' interface TransactionArgs { transactionId: number; } -/** - * Returns true when `err` is the retries-exhausted wrapper around the retriable - * `TX_NOT_FOUND` ApplicationFailure thrown by the `verifyTransaction` activity. - * Any other failure (RPC unreachable, deserialization, bugs) returns false and - * should be rethrown so the workflow fails loudly rather than silently marking - * the tx as failure on insufficient evidence. - */ -function isTxNotFoundFailure(err: unknown): boolean { - if (err instanceof ActivityFailure && err.cause instanceof ApplicationFailure) { - return err.cause.type === TX_NOT_FOUND_ERROR_TYPE - } - return false -} - /** * Extracts the operator address from a transaction's unsigned payload. * Used for Tier 4 fallback (supplier state check) when TX lookup fails. @@ -172,24 +155,18 @@ export async function ExecuteTransaction(args: TransactionArgs) { // retriable TX_NOT_FOUND until the policy is exhausted. [success, code, gasUsed] = await verifyTransaction(txHash, baseHeight); txFoundOnChain = true; - } catch (err) { - // Only fall through to Tier 4 when retries were exhausted specifically with - // TX_NOT_FOUND. Any other failure (RPC unreachable, bug, unexpected shape) is - // rethrown so the workflow fails and the tx stays Pending for human triage — - // avoids false "failure" marks when we lack evidence either way. - if (!isTxNotFoundFailure(err)) { - throw err; - } - - // Retries exhausted. Tier 4: check supplier state directly. Only a positive result - // (supplier on-chain) is conclusive; a missing supplier keeps the tx marked failed. + } catch { if (operatorAddress) { - const supplierExists = await checkSupplierOnChain(operatorAddress); - if (supplierExists) { - success = true; - code = 0; - gasUsed = '0'; - supplierFallbackHit = true; + try { + const supplierExists = await checkSupplierOnChain(operatorAddress); + if (supplierExists) { + success = true; + code = 0; + gasUsed = '0'; + supplierFallbackHit = true; + } + } catch { + // supplier check also failed — tx stays marked as Failure (success stays false) } } }