From 78e667253a902fd16a00656715405549498102d0 Mon Sep 17 00:00:00 2001 From: Artemiy Vereshchinskiy Date: Sun, 23 Nov 2025 17:10:59 +0700 Subject: [PATCH 1/3] Fix deduplication issue for nested upsert --- .changeset/red-boxes-float.md | 10 +++ ...portJson.upsert-nested-linking.e2e.test.ts | 86 +++++++++++++++++++ .../src/core/entity/entity-query.service.ts | 8 +- .../entity/import-export/import.service.ts | 28 +++++- 4 files changed, 127 insertions(+), 5 deletions(-) create mode 100644 .changeset/red-boxes-float.md create mode 100644 packages/javascript-sdk/tests/records.importJson.upsert-nested-linking.e2e.test.ts diff --git a/.changeset/red-boxes-float.md b/.changeset/red-boxes-float.md new file mode 100644 index 00000000..f1c7b492 --- /dev/null +++ b/.changeset/red-boxes-float.md @@ -0,0 +1,10 @@ +--- +'@rushdb/javascript-sdk': patch +'rushdb-core': patch +'rushdb-docs': patch +'@rushdb/mcp-server': patch +'rushdb-dashboard': patch +'rushdb-website': patch +--- + +Fix deduplication issue for nested upsert diff --git a/packages/javascript-sdk/tests/records.importJson.upsert-nested-linking.e2e.test.ts b/packages/javascript-sdk/tests/records.importJson.upsert-nested-linking.e2e.test.ts new file mode 100644 index 00000000..9291df12 --- /dev/null +++ b/packages/javascript-sdk/tests/records.importJson.upsert-nested-linking.e2e.test.ts @@ -0,0 +1,86 @@ +import path from 'path' +import dotenv from 'dotenv' + +dotenv.config({ path: path.resolve(__dirname, '../.env') }) + +import RushDB from '../src/index.node' + +jest.setTimeout(60_000) + +describe('records.importJson upsert nested linking (e2e)', () => { + const apiKey = process.env.RUSHDB_API_KEY + const apiUrl = process.env.RUSHDB_API_URL || 'http://localhost:3000' + + if (!apiKey) { + it('skips because RUSHDB_API_KEY is not set', () => { + expect(true).toBe(true) + }) + return + } + + const db = new RushDB(apiKey, { url: apiUrl }) + + it('reuses child record and links it to new parent when parent upsert creates a new record', async () => { + const tenantId = `nested-upsert-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 8)}` + + const payloadA = { + name: 'Abshire - Farrell', + address: '928 Conroy Village Suite 785', + foundedAt: '1949-10-06T22:07:28.709Z', + rating: 1.9, + tenantId, + department: [ + { + name: 'Sports', + description: 'The sleek and filthy Gloves comes with sky blue LED lighting for smart functionality', + tenantId + } + ] + } + + const payloadB = { + ...payloadA, + rating: 2 // slight change should force new top-level record when mergeBy is all keys + } + + // First import creates parent and child + await db.records.importJson({ label: 'Company', data: payloadA, options: { suggestTypes: true } }) + + // Second import should create new parent record but reuse existing child and link it + await db.records.importJson({ + label: 'Company', + data: payloadB, + options: { suggestTypes: true, mergeBy: [] } + }) + + // Verify there are two Company records and one Department record linked to both via default relation + const companies = await db.records.find({ labels: ['Company'], where: { tenantId } }) + expect(companies.total).toBe(2) + + const departments = await db.records.find({ labels: ['department'], where: { tenantId } }) + // label normalization in service uses original key; depending on capitalization option it might be 'department' + // Allow 1 department entry + expect(departments.total).toBe(1) + + // Fetch relations and ensure both companies are connected to the same department + const relationsResp = await db.relationships.find({}) + const rels = relationsResp.data.filter( + (r) => + r.type && + (r.type.includes('RUSHDB_DEFAULT_RELATION') || r.type.includes('__RUSHDB__RELATION__DEFAULT__')) + ) + + const departmentId = departments.data[0].id() + const companyIds = companies.data.map((c) => c.id()) + + // For each company, there must be at least one relation to the department (either direction) + const relatedPairs = new Set(rels.map((r) => `${r.sourceId}->${r.targetId}`)) + for (const cid of companyIds) { + const has = relatedPairs.has(`${cid}->${departmentId}`) || relatedPairs.has(`${departmentId}->${cid}`) + expect(has).toBe(true) + } + + // Cleanup + await db.records.delete({ where: { tenantId } }) + }) +}) diff --git a/platform/core/src/core/entity/entity-query.service.ts b/platform/core/src/core/entity/entity-query.service.ts index 3f6eb811..27f6d9ac 100755 --- a/platform/core/src/core/entity/entity-query.service.ts +++ b/platform/core/src/core/entity/entity-query.service.ts @@ -159,8 +159,10 @@ export class EntityQueryService { if (withResults) { queryBuilder.append( - `RETURN collect(DISTINCT record {${PROPERTY_WILDCARD_PROJECTION}, ${label()}}) as data` + `RETURN collect({draftId: r.id, persistedId: record.${RUSHDB_KEY_ID}}) as idmap, collect(DISTINCT record {${PROPERTY_WILDCARD_PROJECTION}, ${label()}}) as data` ) + } else { + queryBuilder.append(`RETURN collect({draftId: r.id, persistedId: record.${RUSHDB_KEY_ID}}) as idmap`) } return queryBuilder.getQuery() @@ -220,8 +222,10 @@ export class EntityQueryService { if (withResults) { queryBuilder.append( - `RETURN collect(DISTINCT record {${PROPERTY_WILDCARD_PROJECTION}, ${label()}}) as data` + `RETURN collect({draftId: r.id, persistedId: record.${RUSHDB_KEY_ID}}) as idmap, collect(DISTINCT record {${PROPERTY_WILDCARD_PROJECTION}, ${label()}}) as data` ) + } else { + queryBuilder.append(`RETURN collect({draftId: r.id, persistedId: record.${RUSHDB_KEY_ID}}) as idmap`) } return queryBuilder.getQuery() diff --git a/platform/core/src/core/entity/import-export/import.service.ts b/platform/core/src/core/entity/import-export/import.service.ts index 72770b4d..8d5e7f49 100644 --- a/platform/core/src/core/entity/import-export/import.service.ts +++ b/platform/core/src/core/entity/import-export/import.service.ts @@ -307,6 +307,8 @@ export class ImportService { // @TODO: Accumulate result only if records <= 1000. Otherwise - ignore options.returnResult let result = [] + // Map draft record ids (generated during serialization) to actual persisted record ids after upsert/create + const draftToPersistedId = new Map() for (let i = 0; i < records.length; i += CHUNK_SIZE) { const recordsChunk = records.slice(i, i + CHUNK_SIZE) @@ -317,13 +319,33 @@ export class ImportService { projectId }) + // Extract id map and results (if requested) + const idmap = data.records?.[0]?.get('idmap') ?? [] + if (Array.isArray(idmap)) { + for (const item of idmap) { + if (item && item.draftId && item.persistedId) { + draftToPersistedId.set(item.draftId, item.persistedId) + } + } + } + if (options.returnResult) { - result = result.concat(data.records?.[0]?.get('data')) + const chunkData = data.records?.[0]?.get('data') + if (Array.isArray(chunkData)) { + result = result.concat(chunkData) + } } } - for (let i = 0; i < relations.length; i += CHUNK_SIZE) { - const relationsChunk = relations.slice(i, i + CHUNK_SIZE) + // Remap relations to persisted IDs in case upsert matched existing records + const remappedRelations = relations.map((rel) => ({ + source: draftToPersistedId.get(rel.source) ?? rel.source, + target: draftToPersistedId.get(rel.target) ?? rel.target, + type: rel.type + })) + + for (let i = 0; i < remappedRelations.length; i += CHUNK_SIZE) { + const relationsChunk = remappedRelations.slice(i, i + CHUNK_SIZE) await this.processRelationshipsChunk({ relationsChunk, projectId, From c4e3ed4f9e2abb5021c79de3b74d028a95ff1952 Mon Sep 17 00:00:00 2001 From: Artemy Vereshinsky Date: Sun, 23 Nov 2025 23:55:54 +0700 Subject: [PATCH 2/3] Update packages/javascript-sdk/tests/records.importJson.upsert-nested-linking.e2e.test.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../tests/records.importJson.upsert-nested-linking.e2e.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/javascript-sdk/tests/records.importJson.upsert-nested-linking.e2e.test.ts b/packages/javascript-sdk/tests/records.importJson.upsert-nested-linking.e2e.test.ts index 9291df12..f1d3ac92 100644 --- a/packages/javascript-sdk/tests/records.importJson.upsert-nested-linking.e2e.test.ts +++ b/packages/javascript-sdk/tests/records.importJson.upsert-nested-linking.e2e.test.ts @@ -63,7 +63,7 @@ describe('records.importJson upsert nested linking (e2e)', () => { expect(departments.total).toBe(1) // Fetch relations and ensure both companies are connected to the same department - const relationsResp = await db.relationships.find({}) + const relationsResp = await db.relationships.find({ where: { tenantId } }) const rels = relationsResp.data.filter( (r) => r.type && From d28281b82017fae0282e91ca17eaee18bee4c520 Mon Sep 17 00:00:00 2001 From: Artemy Vereshinsky Date: Sun, 23 Nov 2025 23:56:00 +0700 Subject: [PATCH 3/3] Update packages/javascript-sdk/tests/records.importJson.upsert-nested-linking.e2e.test.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../tests/records.importJson.upsert-nested-linking.e2e.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/javascript-sdk/tests/records.importJson.upsert-nested-linking.e2e.test.ts b/packages/javascript-sdk/tests/records.importJson.upsert-nested-linking.e2e.test.ts index f1d3ac92..0b0b1863 100644 --- a/packages/javascript-sdk/tests/records.importJson.upsert-nested-linking.e2e.test.ts +++ b/packages/javascript-sdk/tests/records.importJson.upsert-nested-linking.e2e.test.ts @@ -59,7 +59,7 @@ describe('records.importJson upsert nested linking (e2e)', () => { const departments = await db.records.find({ labels: ['department'], where: { tenantId } }) // label normalization in service uses original key; depending on capitalization option it might be 'department' - // Allow 1 department entry + // The label comes from the original key 'department' in the payload expect(departments.total).toBe(1) // Fetch relations and ensure both companies are connected to the same department