Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
ff086b9
get a failing test
OriolMunoz-da Aug 20, 2025
ea2f40a
Break when the results would be bad [ci]
OriolMunoz-da Aug 20, 2025
ee72588
[ci] fix things
OriolMunoz-da Aug 20, 2025
cae3fad
[ci] more fixes
OriolMunoz-da Aug 20, 2025
4199050
[ci] was that a gha/k8s failure or did i cause that? let's find out
OriolMunoz-da Aug 20, 2025
0c01122
[ci] refactor so many queries something is bound to break
OriolMunoz-da Aug 21, 2025
096ea57
[ci] remove many instances of getFromResources
OriolMunoz-da Aug 21, 2025
24b712a
[ci] fix splitwell
OriolMunoz-da Aug 21, 2025
3b845ce
[ci] fix bad query
OriolMunoz-da Aug 21, 2025
50077da
unused variable
OriolMunoz-da Aug 21, 2025
15d666a
[ci] bump descriptors
OriolMunoz-da Aug 21, 2025
a76111d
Merge branch 'main' into oriol/packagename-stores-interface-fallback
OriolMunoz-da Aug 25, 2025
59c5890
fix many compile errors
OriolMunoz-da Aug 25, 2025
3184d5a
add indexes with todo
OriolMunoz-da Aug 25, 2025
5910a9f
[ci] change definition of all indexes involving tid
OriolMunoz-da Aug 25, 2025
1680a64
[ci] also truncate store_last_ingested_offsets
OriolMunoz-da Aug 25, 2025
dc7e00d
Merge branch 'main' into oriol/packagename-stores-interface-fallback
OriolMunoz-da Aug 26, 2025
5bfdbca
[ci] revert bump of txlog descriptor
OriolMunoz-da Aug 26, 2025
cb9983f
wip
OriolMunoz-da Aug 26, 2025
16df3a3
fix comment
OriolMunoz-da Sep 5, 2025
5ca46b9
Merge branch 'main' into oriol/packagename-stores-interface-fallback
OriolMunoz-da Sep 5, 2025
553b6c9
[ci] update migration
OriolMunoz-da Sep 5, 2025
583b0cb
Merge branch 'main' into oriol/packagename-stores-interface-fallback
OriolMunoz-da Oct 1, 2025
df27b53
[ci] run
OriolMunoz-da Oct 1, 2025
684be68
[ci] fix migrations
OriolMunoz-da Oct 1, 2025
498948c
[ci] Ignore ACS_COMMITMENT_DEGRADATION
OriolMunoz-da Oct 1, 2025
b82028d
Merge branch 'main' into oriol/packagename-stores-interface-fallback
OriolMunoz-da Oct 1, 2025
d201c0f
[ci] run
OriolMunoz-da Oct 1, 2025
6f8f951
[ci] fix migrations again
OriolMunoz-da Oct 1, 2025
74fea0e
Merge branch 'main' into oriol/packagename-stores-interface-fallback
OriolMunoz-da Oct 1, 2025
2a04dac
Merge branch 'canton-3.4' into oriol/packagename-stores-interface-fal…
OriolMunoz-da Oct 9, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -404,7 +404,7 @@ trait LedgerApiExtensions extends AppendedClues with Matchers {
predicate: TC => Boolean = (_: TC) => true,
): Seq[TC] = {
val filterIdentifier =
PackageQualifiedName.getFromResources(templateCompanion.getTemplateIdWithPackageId)
PackageQualifiedName.fromJavaCodegenCompanion(templateCompanion)
val templateId = TemplateId(
s"#${filterIdentifier.packageName}",
filterIdentifier.qualifiedName.moduleName,
Expand Down Expand Up @@ -443,8 +443,7 @@ trait LedgerApiExtensions extends AppendedClues with Matchers {
](templateCompanion: javaapi.data.codegen.ContractCompanion[TC, TCid, T])(
partyId: PartyId
): Seq[CreatedEvent] = {
val filterIdentifier =
PackageQualifiedName.getFromResources(templateCompanion.getTemplateIdWithPackageId)
val filterIdentifier = PackageQualifiedName.fromJavaCodegenCompanion(templateCompanion)
val templateId = TemplateId(
s"#${filterIdentifier.packageName}",
filterIdentifier.qualifiedName.moduleName,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1025,11 +1025,11 @@ class DecentralizedSynchronizerMigrationIntegrationTest
pageSize = 1000,
templates = Some(
Vector(
DsoRules.TEMPLATE_ID_WITH_PACKAGE_ID,
AmuletRules.TEMPLATE_ID_WITH_PACKAGE_ID,
AnsRules.TEMPLATE_ID_WITH_PACKAGE_ID,
DsoRules.COMPANION,
AmuletRules.COMPANION,
AnsRules.COMPANION,
).map(
PackageQualifiedName.getFromResources(_)
PackageQualifiedName.fromJavaCodegenCompanion
)
),
)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -510,8 +510,8 @@ class ScanTimeBasedIntegrationTest
migrationId,
templates = Some(
Vector(
PackageQualifiedName.getFromResources(Amulet.TEMPLATE_ID_WITH_PACKAGE_ID),
PackageQualifiedName.getFromResources(AnsEntry.TEMPLATE_ID_WITH_PACKAGE_ID),
PackageQualifiedName.fromJavaCodegenCompanion(Amulet.COMPANION),
PackageQualifiedName.fromJavaCodegenCompanion(AnsEntry.COMPANION),
)
),
partyIds = Some(Vector(aliceUserParty)),
Expand Down

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import org.lfdecentralizedtrust.splice.codegen.java.splice.dsorules.VoteRequest
import org.lfdecentralizedtrust.splice.store.db.AcsQueries.AcsStoreId
import org.lfdecentralizedtrust.splice.store.db.TxLogQueries.TxLogStoreId
import org.lfdecentralizedtrust.splice.store.db.{AcsQueries, TxLogQueries}
import org.lfdecentralizedtrust.splice.util.QualifiedName
import slick.dbio.{Effect, NoStream}
import slick.jdbc.canton.ActionBasedSQLInterpolation.Implicits.actionBasedSQLInterpolationCanton
import slick.sql.SqlStreamingAction
Expand Down Expand Up @@ -110,10 +109,8 @@ trait DbVotesAcsStoreQueryBuilder extends AcsQueries with LimitHelpers with Name
acsTableName,
acsStoreId,
domainMigrationId,
where = (sql""" template_id_qualified_name = ${QualifiedName(
VoteRequest.TEMPLATE_ID_WITH_PACKAGE_ID
)}
and #$trackingCidColumnName in """ ++ voteRequestTrackingCidsSql).toActionBuilder,
VoteRequest.COMPANION,
where = (sql""" #$trackingCidColumnName in """ ++ voteRequestTrackingCidsSql).toActionBuilder,
orderLimit = sql"""limit ${sqlLimit(limit)}""",
)
}
Expand All @@ -133,10 +130,8 @@ trait DbVotesAcsStoreQueryBuilder extends AcsQueries with LimitHelpers with Name
acsTableName,
acsStoreId,
domainMigrationId,
where = (sql""" template_id_qualified_name = ${QualifiedName(
VoteRequest.TEMPLATE_ID_WITH_PACKAGE_ID
)}
and #$trackingCidColumnName = $voteRequestCid """).toActionBuilder,
VoteRequest.COMPANION,
where = (sql""" #$trackingCidColumnName = $voteRequestCid """).toActionBuilder,
).headOption
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,10 @@ package org.lfdecentralizedtrust.splice.store
import org.apache.pekko.NotUsed
import org.apache.pekko.stream.scaladsl.Source
import com.daml.ledger.api.v2.transaction_filter.{EventFormat, CumulativeFilter}
import org.lfdecentralizedtrust.splice.util.Contract.Companion.Template as TemplateCompanion
import org.lfdecentralizedtrust.splice.util.Contract.Companion.{
Interface,
Template as TemplateCompanion,
}
import com.daml.ledger.javaapi.data.{CreatedEvent, ExercisedEvent, Identifier, Template}
import com.daml.ledger.javaapi.data.codegen.{ContractId, DamlRecord}
import com.daml.metrics.api.MetricsContext
Expand All @@ -26,6 +29,7 @@ import org.lfdecentralizedtrust.splice.util.{
Contract,
ContractWithState,
PackageQualifiedName,
QualifiedName,
TemplateJsonDecoder,
}
import org.lfdecentralizedtrust.splice.util.PrettyInstances.*
Expand Down Expand Up @@ -171,22 +175,6 @@ trait MultiDomainAcsStore extends HasIngestionSink with AutoCloseable with Named
traceContext: TraceContext,
): Future[Seq[Contract[TCid, T]]]

/** At most 1000 (`notOnDomainsTotalLimit`) contracts sorted by a hash of
* contract ID and participant ID.
*
* The idea is that different apps making the same migration on different
* participants will split the work better, while preserving determinism of a
* specific running app for fault-tolerance. For the former to happen, the
* position of a contract on one list must have no correlation with that on
* another list; that is why the contract ID by itself cannot be used by
* itself as the source of the sort key.
*/
def listAssignedContractsNotOnDomainN(
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

unused except in a test (also deleted)

excludedDomain: SynchronizerId,
companions: Seq[ConstrainedTemplate],
limit: notOnDomainsTotalLimit.type = notOnDomainsTotalLimit,
)(implicit tc: TraceContext): Future[Seq[AssignedContract[?, ?]]]

private[splice] def listExpiredFromPayloadExpiry[C, TCid <: ContractId[T], T <: Template](
companion: C
)(implicit
Expand Down Expand Up @@ -470,7 +458,7 @@ object MultiDomainAcsStore extends StoreErrors {
TemplateFilter[TCid, T, R],
) =
(
PackageQualifiedName.getFromResources(templateCompanion.getTemplateIdWithPackageId),
PackageQualifiedName.fromJavaCodegenCompanion(templateCompanion),
TemplateFilter(
ev => {
val c = Contract.fromCreatedEvent(templateCompanion)(ev)
Expand Down Expand Up @@ -597,6 +585,8 @@ object MultiDomainAcsStore extends StoreErrors {

def typeId(companion: C): Identifier

def packageQualifiedName(companion: C): PackageQualifiedName

def toContractId(companion: C, contractId: String): TCid

protected def fromJson(
Expand All @@ -619,6 +609,13 @@ object MultiDomainAcsStore extends StoreErrors {
override def typeId(companion: Contract.Companion.Template[TCid, T]): Identifier =
companion.getTemplateIdWithPackageId

override def packageQualifiedName(
companion: TemplateCompanion[TCid, T]
): PackageQualifiedName = PackageQualifiedName(
companion.PACKAGE_NAME,
QualifiedName(companion.getTemplateIdWithPackageId),
)

override def toContractId(companion: Companion.Template[TCid, T], contractId: String): TCid =
companion.toContractId(new ContractId[T](contractId))

Expand Down Expand Up @@ -652,6 +649,13 @@ object MultiDomainAcsStore extends StoreErrors {
override def typeId(companion: Contract.Companion.Interface[ICid, Marker, View]): Identifier =
companion.getTemplateIdWithPackageId

override def packageQualifiedName(
companion: Interface[ICid, Marker, View]
): PackageQualifiedName = PackageQualifiedName(
companion.PACKAGE_NAME,
QualifiedName(companion.getTemplateIdWithPackageId),
)

override def toContractId(
companion: Companion.Interface[ICid, Marker, View],
contractId: String,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -218,6 +218,11 @@ trait AcsJdbcTypes {
protected implicit val qualifiedNameGetResult: GetResult[QualifiedName] =
GetResult.GetString.andThen { s => QualifiedName.assertFromString(s) }

protected implicit val packageQualifiedNameGetResult: GetResult[PackageQualifiedName] =
implicitly[GetResult[(QualifiedName, String)]].andThen { case (qualifiedName, packageName) =>
PackageQualifiedName(packageName, qualifiedName)
}

protected implicit lazy val qualifiedNameJdbcType: JdbcType[QualifiedName] =
MappedColumnType.base[QualifiedName, String](
{ _.toString }, {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,24 +26,33 @@ import scalaz.{@@, Tag}
import slick.jdbc.canton.ActionBasedSQLInterpolation.Implicits.actionBasedSQLInterpolationCanton
import slick.jdbc.canton.SQLActionBuilder
import slick.jdbc.{GetResult, PositionedResult, SetParameter}
import slick.dbio.Effect
import slick.sql.SqlStreamingAction

trait AcsQueries extends AcsJdbcTypes {

/** @param tableName Must be SQL-safe, as it needs to be interpolated unsafely.
* This is fine, as all calls to this method should use static string constants.
*/
protected def selectFromAcsTable(
protected def selectFromAcsTable[C, TCid <: ContractId[_], T](
tableName: String,
storeId: AcsStoreId,
migrationId: Long,
where: SQLActionBuilder,
companion: C,
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this should've been the case all along, instead of having where ti_q_n = ? on every query

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

agreed, thanks for pulling through that simplification!

where: SQLActionBuilder = sql"true",
orderLimit: SQLActionBuilder = sql"",
) =
)(implicit companionClass: ContractCompanion[C, TCid, T]) = {
val packageQualifiedName = companionClass.packageQualifiedName(companion)
(sql"""
select #${SelectFromAcsTableResult.sqlColumnsCommaSeparated()}
from #$tableName acs
where acs.store_id = $storeId and acs.migration_id = $migrationId and """ ++ where ++ sql"""
where acs.store_id = $storeId
and acs.migration_id = $migrationId
and acs.package_name = ${packageQualifiedName.packageName}
and acs.template_id_qualified_name = ${packageQualifiedName.qualifiedName}
and """ ++ where ++ sql"""
""" ++ orderLimit).toActionBuilder.as[AcsQueries.SelectFromAcsTableResult]
}

implicit val GetResultSelectFromAcsTable: GetResult[AcsQueries.SelectFromAcsTableResult] =
GetResult { prs =>
Expand All @@ -55,7 +64,7 @@ trait AcsQueries extends AcsJdbcTypes {
<<[Long],
<<[ContractId[Any]],
<<[String],
<<[QualifiedName],
<<[PackageQualifiedName],
<<[Json],
<<[Array[Byte]],
<<[Timestamp],
Expand All @@ -65,18 +74,27 @@ trait AcsQueries extends AcsJdbcTypes {
}

/** Similar to [[selectFromAcsTable]], but also returns the contract state (i.e., the domain to which a contract is currently assigned) */
protected def selectFromAcsTableWithState(
protected def selectFromAcsTableWithState[C, TCid <: ContractId[_], T](
tableName: String,
storeId: AcsStoreId,
migrationId: Long,
where: SQLActionBuilder,
companion: C,
additionalWhere: SQLActionBuilder = sql"",
orderLimit: SQLActionBuilder = sql"",
) =
)(implicit companionClass: ContractCompanion[C, TCid, T]): SqlStreamingAction[Vector[
SelectFromAcsTableWithStateResult
], SelectFromAcsTableWithStateResult, Effect.Read] = {
val packageQualifiedName = companionClass.packageQualifiedName(companion)
(sql"""
select #${SelectFromAcsTableWithStateResult.sqlColumnsCommaSeparated()}
from #$tableName acs
where acs.store_id = $storeId and acs.migration_id = $migrationId and """ ++ where ++ sql"""
where acs.store_id = $storeId
and acs.migration_id = $migrationId
and acs.package_name = ${packageQualifiedName.packageName}
and acs.template_id_qualified_name = ${packageQualifiedName.qualifiedName}
""" ++ additionalWhere ++ sql"""
""" ++ orderLimit).toActionBuilder.as[AcsQueries.SelectFromAcsTableWithStateResult]
}

implicit val GetResultSelectFromContractStateResult
: GetResult[AcsQueries.SelectFromContractStateResult] =
Expand Down Expand Up @@ -104,13 +122,15 @@ trait AcsQueries extends AcsJdbcTypes {
* This guarantees that the fetched contracts exist in the given offset,
* whereas two separate queries (one to fetch the contract and one to fetch the offset) don't guarantee that.
*/
protected def selectFromAcsTableWithOffset(
protected def selectFromAcsTableWithOffset[C, TCid <: ContractId[_], T](
tableName: String,
storeId: AcsStoreId,
migrationId: Long,
companion: C,
where: SQLActionBuilder,
orderLimit: SQLActionBuilder = sql"",
) =
)(implicit companionClass: ContractCompanion[C, TCid, T]) = {
val packageQualifiedName = companionClass.packageQualifiedName(companion)
(sql"""
select
acs.store_id,
Expand All @@ -120,6 +140,7 @@ trait AcsQueries extends AcsJdbcTypes {
contract_id,
template_id_package_id,
template_id_qualified_name,
package_name,
create_arguments,
created_event_blob,
created_at,
Expand All @@ -130,10 +151,13 @@ trait AcsQueries extends AcsJdbcTypes {
left join #$tableName acs
on o.store_id = acs.store_id
and o.migration_id = acs.migration_id
and acs.package_name = ${packageQualifiedName.packageName}
and acs.template_id_qualified_name = ${packageQualifiedName.qualifiedName}
and """ ++ where ++ sql"""
where sd.id = $storeId and o.migration_id = $migrationId
""" ++ orderLimit).toActionBuilder
.as[AcsQueries.SelectFromAcsTableResultWithOffset]
}

implicit val GetResultSelectFromAcsTableResultWithOffset
: GetResult[AcsQueries.SelectFromAcsTableResultWithOffset] = { (pp: PositionedResult) =>
Expand Down Expand Up @@ -161,13 +185,15 @@ trait AcsQueries extends AcsJdbcTypes {

/** Same as [[selectFromAcsTableWithOffset]], but also includes the contract state.
*/
protected def selectFromAcsTableWithStateAndOffset(
protected def selectFromAcsTableWithStateAndOffset[C, TCid <: ContractId[_], T](
tableName: String,
storeId: AcsStoreId,
migrationId: Long,
companion: C,
where: SQLActionBuilder = sql"true",
orderLimit: SQLActionBuilder = sql"",
) =
)(implicit companionClass: ContractCompanion[C, TCid, T]) = {
val packageQualifiedName = companionClass.packageQualifiedName(companion)
(sql"""
select
acs.store_id,
Expand All @@ -177,6 +203,7 @@ trait AcsQueries extends AcsJdbcTypes {
acs.contract_id,
acs.template_id_package_id,
acs.template_id_qualified_name,
acs.package_name,
acs.create_arguments,
acs.created_event_blob,
acs.created_at,
Expand All @@ -194,10 +221,13 @@ trait AcsQueries extends AcsJdbcTypes {
left join #$tableName acs
on o.store_id = acs.store_id
and o.migration_id = acs.migration_id
and acs.package_name = ${packageQualifiedName.packageName}
and acs.template_id_qualified_name = ${packageQualifiedName.qualifiedName}
and """ ++ where ++ sql"""
where sd.id = $storeId and o.migration_id = $migrationId
""" ++ orderLimit).toActionBuilder
.as[AcsQueries.SelectFromAcsTableResultWithStateAndOffset]
}

implicit val GetResultSelectFromAcsTableResultWithStateOffset
: GetResult[AcsQueries.SelectFromAcsTableResultWithStateAndOffset] = {
Expand Down Expand Up @@ -311,7 +341,7 @@ object AcsQueries {
eventNumber: Long,
contractId: ContractId[Any],
templateIdPackageId: String,
templateIdQualifiedName: QualifiedName,
packageQualifiedName: PackageQualifiedName,
createArguments: Json,
createdEventBlob: Array[Byte],
createdAt: Timestamp,
Expand All @@ -321,12 +351,22 @@ object AcsQueries {
companionClass: ContractCompanion[C, TCId, T],
decoder: TemplateJsonDecoder,
): Contract[TCId, T] = {
// safety check: if the PackageQualifiedNames don't match,
// it means that we would be returning a contract of a different template
// note that the packageId not matching is expected due to upgrades, but the name will be stable
val expectedPackageQualifiedName = companionClass.packageQualifiedName(companion)
if (expectedPackageQualifiedName != packageQualifiedName) {
throw new IllegalStateException(
s"Contract $contractId has a different package qualified name than expected. Expected: $expectedPackageQualifiedName - Got: $packageQualifiedName"
)
}

companionClass
.fromJson(companion)(
new Identifier(
templateIdPackageId,
templateIdQualifiedName.moduleName,
templateIdQualifiedName.entityName,
packageQualifiedName.qualifiedName.moduleName,
packageQualifiedName.qualifiedName.entityName,
),
contractId.contractId,
createArguments,
Expand All @@ -351,6 +391,7 @@ object AcsQueries {
${qualifier}contract_id,
${qualifier}template_id_package_id,
${qualifier}template_id_qualified_name,
${qualifier}package_name,
${qualifier}create_arguments,
${qualifier}created_event_blob,
${qualifier}created_at,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,6 @@ abstract class DbTxLogAppStore[TXE](
acsContractFilter,
txLogConfig,
domainMigrationInfo,
participantId,
retryProvider,
handleIngestionSummary,
)
Expand Down Expand Up @@ -98,7 +97,6 @@ abstract class DbAppStore(
acsContractFilter,
TxLogStore.Config.empty,
domainMigrationInfo,
participantId,
retryProvider,
handleIngestionSummary,
)
Expand Down
Loading
Loading