Skip to content

Commit 5c8da47

Browse files
RohitR311RohitR311
authored
Merge branch 'develop' into recorder-ui-shift
2 parents cdee69a + 2b85a04 commit 5c8da47

File tree

12 files changed

+326
-255
lines changed

12 files changed

+326
-255
lines changed

‎maxun-core/src/interpret.ts‎

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1246,9 +1246,9 @@ export default class Interpreter extends EventEmitter {
12461246
if (checkLimit()) return allResults;
12471247

12481248
let loadMoreCounter = 0;
1249-
// let previousResultCount = allResults.length;
1250-
// let noNewItemsCounter = 0;
1251-
// const MAX_NO_NEW_ITEMS = 2;
1249+
let previousResultCount = allResults.length;
1250+
let noNewItemsCounter = 0;
1251+
const MAX_NO_NEW_ITEMS = 5;
12521252

12531253
while (true) {
12541254
if (this.isAborted) {
@@ -1332,21 +1332,21 @@ export default class Interpreter extends EventEmitter {
13321332

13331333
await scrapeCurrentPage();
13341334

1335-
// const currentResultCount = allResults.length;
1336-
// const newItemsAdded = currentResultCount > previousResultCount;
1335+
const currentResultCount = allResults.length;
1336+
const newItemsAdded = currentResultCount > previousResultCount;
13371337

1338-
// if (!newItemsAdded) {
1339-
// noNewItemsCounter++;
1340-
// debugLog(`No new items added after click (${noNewItemsCounter}/${MAX_NO_NEW_ITEMS})`);
1338+
if (!newItemsAdded) {
1339+
noNewItemsCounter++;
1340+
debugLog(`No new items added after click (${noNewItemsCounter}/${MAX_NO_NEW_ITEMS})`);
13411341

1342-
// if (noNewItemsCounter >= MAX_NO_NEW_ITEMS) {
1343-
// debugLog(`Stopping after ${MAX_NO_NEW_ITEMS} clicks with no new items`);
1344-
// return allResults;
1345-
// }
1346-
// } else {
1347-
// noNewItemsCounter = 0;
1348-
// previousResultCount = currentResultCount;
1349-
// }
1342+
if (noNewItemsCounter >= MAX_NO_NEW_ITEMS) {
1343+
debugLog(`Stopping after ${MAX_NO_NEW_ITEMS} clicks with no new items`);
1344+
return allResults;
1345+
}
1346+
} else {
1347+
noNewItemsCounter = 0;
1348+
previousResultCount = currentResultCount;
1349+
}
13501350

13511351
if (checkLimit()) return allResults;
13521352

‎maxun-core/src/preprocessor.ts‎

Lines changed: 18 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ export default class Preprocessor {
5555
*/
5656
static getParams(workflow: WorkflowFile): string[] {
5757
const getParamsRecurse = (object: any): string[] => {
58-
if (typeof object === 'object') {
58+
if (typeof object === 'object' && object !== null) {
5959
// Recursion base case
6060
if (object.$param) {
6161
return [object.$param];
@@ -141,14 +141,24 @@ export default class Preprocessor {
141141
}
142142

143143
const out = object;
144-
// for every key (child) of the object
144+
145145
Object.keys(object!).forEach((key) => {
146-
// if the field has only one key, which is `k`
147-
if (Object.keys((<any>object)[key]).length === 1 && (<any>object)[key][k]) {
148-
// process the current special tag (init param, hydrate regex...)
149-
(<any>out)[key] = f((<any>object)[key][k]);
150-
} else {
151-
initSpecialRecurse((<any>object)[key], k, f);
146+
const childValue = (<any>object)[key];
147+
148+
if (!childValue || typeof childValue !== 'object') {
149+
return;
150+
}
151+
152+
try {
153+
const childKeys = Object.keys(childValue);
154+
155+
if (childKeys.length === 1 && childValue[k]) {
156+
(<any>out)[key] = f(childValue[k]);
157+
} else {
158+
initSpecialRecurse(childValue, k, f);
159+
}
160+
} catch (error) {
161+
console.warn(`Error processing key "${key}" in initSpecialRecurse:`, error);
152162
}
153163
});
154164
return out;

‎server/src/browser-management/classes/RemoteBrowser.ts‎

Lines changed: 62 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,11 @@ export class RemoteBrowser {
201201
private networkRequestTimeout: NodeJS.Timeout | null = null;
202202
private pendingNetworkRequests: string[] = [];
203203
private readonly NETWORK_QUIET_PERIOD = 8000;
204+
private readonly INITIAL_LOAD_QUIET_PERIOD = 3000;
205+
private networkWaitStartTime: number = 0;
206+
private progressInterval: NodeJS.Timeout | null = null;
207+
private hasShownInitialLoader: boolean = false;
208+
private isInitialLoadInProgress: boolean = false;
204209

205210
/**
206211
* Initializes a new instances of the {@link Generator} and {@link WorkflowInterpreter} classes and
@@ -432,42 +437,74 @@ export class RemoteBrowser {
432437
if (!this.currentPage) return;
433438

434439
this.currentPage.on("domcontentloaded", async () => {
435-
logger.info("DOM content loaded - triggering snapshot");
436-
await this.makeAndEmitDOMSnapshot();
440+
if (!this.isInitialLoadInProgress) {
441+
logger.info("DOM content loaded - triggering snapshot");
442+
await this.makeAndEmitDOMSnapshot();
443+
}
437444
});
438445

439446
this.currentPage.on("response", async (response) => {
440447
const url = response.url();
441-
if (
442-
response.request().resourceType() === "document" ||
443-
url.includes("api/") ||
444-
url.includes("ajax")
445-
) {
448+
const isDocumentRequest = response.request().resourceType() === "document";
449+
450+
if (!this.hasShownInitialLoader && isDocumentRequest && !url.includes("about:blank")) {
451+
this.hasShownInitialLoader = true;
452+
this.isInitialLoadInProgress = true;
446453
this.pendingNetworkRequests.push(url);
447454

448455
if (this.networkRequestTimeout) {
449456
clearTimeout(this.networkRequestTimeout);
450457
this.networkRequestTimeout = null;
451458
}
452459

460+
if (this.progressInterval) {
461+
clearInterval(this.progressInterval);
462+
this.progressInterval = null;
463+
}
464+
465+
this.networkWaitStartTime = Date.now();
466+
this.progressInterval = setInterval(() => {
467+
const elapsed = Date.now() - this.networkWaitStartTime;
468+
const navigationProgress = Math.min((elapsed / this.INITIAL_LOAD_QUIET_PERIOD) * 40, 35);
469+
const totalProgress = 60 + navigationProgress;
470+
this.emitLoadingProgress(totalProgress, this.pendingNetworkRequests.length);
471+
}, 500);
472+
453473
logger.debug(
454-
`Network request received: ${url}. Total pending: ${this.pendingNetworkRequests.length}`
474+
`Initial load network request received: ${url}. Using ${this.INITIAL_LOAD_QUIET_PERIOD}ms quiet period`
455475
);
456476

457477
this.networkRequestTimeout = setTimeout(async () => {
458478
logger.info(
459-
`Network quiet period reached. Processing ${this.pendingNetworkRequests.length} requests`
479+
`Initial load network quiet period reached (${this.INITIAL_LOAD_QUIET_PERIOD}ms)`
460480
);
461481

482+
if (this.progressInterval) {
483+
clearInterval(this.progressInterval);
484+
this.progressInterval = null;
485+
}
486+
487+
this.emitLoadingProgress(100, this.pendingNetworkRequests.length);
488+
462489
this.pendingNetworkRequests = [];
463490
this.networkRequestTimeout = null;
491+
this.isInitialLoadInProgress = false;
464492

465493
await this.makeAndEmitDOMSnapshot();
466-
}, this.NETWORK_QUIET_PERIOD);
494+
}, this.INITIAL_LOAD_QUIET_PERIOD);
467495
}
468496
});
469497
}
470498

499+
private emitLoadingProgress(progress: number, pendingRequests: number): void {
500+
this.socket.emit("domLoadingProgress", {
501+
progress: Math.round(progress),
502+
pendingRequests,
503+
userId: this.userId,
504+
timestamp: Date.now(),
505+
});
506+
}
507+
471508
private async setupPageEventListeners(page: Page) {
472509
page.on('framenavigated', async (frame) => {
473510
if (frame === page.mainFrame()) {
@@ -521,7 +558,13 @@ export class RemoteBrowser {
521558
const MAX_RETRIES = 3;
522559
let retryCount = 0;
523560
let success = false;
524-
561+
562+
this.socket.emit("dom-snapshot-loading", {
563+
userId: this.userId,
564+
timestamp: Date.now(),
565+
});
566+
this.emitLoadingProgress(0, 0);
567+
525568
while (!success && retryCount < MAX_RETRIES) {
526569
try {
527570
this.browser = <Browser>(await chromium.launch({
@@ -545,7 +588,9 @@ export class RemoteBrowser {
545588
if (!this.browser || this.browser.isConnected() === false) {
546589
throw new Error('Browser failed to launch or is not connected');
547590
}
548-
591+
592+
this.emitLoadingProgress(20, 0);
593+
549594
const proxyConfig = await getDecryptedProxyConfig(userId);
550595
let proxyOptions: { server: string, username?: string, password?: string } = { server: '' };
551596

@@ -623,6 +668,8 @@ export class RemoteBrowser {
623668

624669
this.currentPage = await this.context.newPage();
625670

671+
this.emitLoadingProgress(40, 0);
672+
626673
await this.setupPageEventListeners(this.currentPage);
627674

628675
const viewportSize = await this.currentPage.viewportSize();
@@ -645,7 +692,9 @@ export class RemoteBrowser {
645692
// Still need to set up the CDP session even if blocker fails
646693
this.client = await this.currentPage.context().newCDPSession(this.currentPage);
647694
}
648-
695+
696+
this.emitLoadingProgress(60, 0);
697+
649698
success = true;
650699
logger.log('debug', `Browser initialized successfully for user ${userId}`);
651700
} catch (error: any) {
@@ -1521,9 +1570,6 @@ export class RemoteBrowser {
15211570
this.isDOMStreamingActive = true;
15221571
logger.info("DOM streaming started successfully");
15231572

1524-
// Initial DOM snapshot
1525-
await this.makeAndEmitDOMSnapshot();
1526-
15271573
this.setupScrollEventListener();
15281574
this.setupPageChangeListeners();
15291575
} catch (error) {

‎server/src/workflow-management/classes/Interpreter.ts‎

Lines changed: 54 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,16 @@ export class WorkflowInterpreter {
116116
*/
117117
private currentScrapeListIndex: number = 0;
118118

119+
/**
120+
* Track action counts to generate unique names
121+
*/
122+
private actionCounts: Record<string, number> = {};
123+
124+
/**
125+
* Track used action names to prevent duplicates
126+
*/
127+
private usedActionNames: Set<string> = new Set();
128+
119129
/**
120130
* Current run ID for real-time persistence
121131
*/
@@ -379,6 +389,8 @@ export class WorkflowInterpreter {
379389
};
380390
this.binaryData = [];
381391
this.currentScrapeListIndex = 0;
392+
this.actionCounts = {};
393+
this.usedActionNames = new Set();
382394
this.currentRunId = null;
383395
this.persistenceBuffer = [];
384396
this.persistenceInProgress = false;
@@ -394,6 +406,43 @@ export class WorkflowInterpreter {
394406
logger.log('debug', `Set run ID for real-time persistence: ${runId}`);
395407
};
396408

409+
/**
410+
* Generates a unique action name for data storage
411+
* @param actionType The type of action (scrapeList, scrapeSchema, etc.)
412+
* @param providedName Optional name provided by the action
413+
* @returns A unique action name
414+
*/
415+
private getUniqueActionName = (actionType: string, providedName?: string | null): string => {
416+
if (providedName && providedName.trim() !== '' && !this.usedActionNames.has(providedName)) {
417+
this.usedActionNames.add(providedName);
418+
return providedName;
419+
}
420+
421+
if (!this.actionCounts[actionType]) {
422+
this.actionCounts[actionType] = 0;
423+
}
424+
425+
let uniqueName: string;
426+
let counter = this.actionCounts[actionType];
427+
428+
do {
429+
counter++;
430+
if (actionType === 'scrapeList') {
431+
uniqueName = `List ${counter}`;
432+
} else if (actionType === 'scrapeSchema') {
433+
uniqueName = `Text ${counter}`;
434+
} else if (actionType === 'screenshot') {
435+
uniqueName = `Screenshot ${counter}`;
436+
} else {
437+
uniqueName = `${actionType} ${counter}`;
438+
}
439+
} while (this.usedActionNames.has(uniqueName));
440+
441+
this.actionCounts[actionType] = counter;
442+
this.usedActionNames.add(uniqueName);
443+
return uniqueName;
444+
};
445+
397446
/**
398447
* Persists extracted data to database with intelligent batching for performance
399448
* Falls back to immediate persistence for critical operations
@@ -525,20 +574,8 @@ export class WorkflowInterpreter {
525574
}
526575

527576
let actionName = this.currentActionName || "";
528-
529-
if (!actionName) {
530-
if (!Array.isArray(data) && Object.keys(data).length === 1) {
531-
const soleKey = Object.keys(data)[0];
532-
const soleValue = data[soleKey];
533-
if (Array.isArray(soleValue) || typeof soleValue === "object") {
534-
actionName = soleKey;
535-
data = soleValue;
536-
}
537-
}
538-
}
539-
540-
if (!actionName) {
541-
actionName = "Unnamed Action";
577+
if (typeKey === "scrapeList") {
578+
actionName = this.getUniqueActionName(typeKey, this.currentActionName);
542579
}
543580

544581
const flattened = Array.isArray(data)
@@ -570,9 +607,10 @@ export class WorkflowInterpreter {
570607
const { name, data, mimeType } = payload;
571608

572609
const base64Data = data.toString("base64");
610+
const uniqueName = this.getUniqueActionName('screenshot', name);
573611

574612
const binaryItem = {
575-
name,
613+
name: uniqueName,
576614
mimeType,
577615
data: base64Data
578616
};
@@ -582,7 +620,7 @@ export class WorkflowInterpreter {
582620
await this.persistBinaryDataToDatabase(binaryItem);
583621

584622
this.socket.emit("binaryCallback", {
585-
name,
623+
name: uniqueName,
586624
data: base64Data,
587625
mimeType
588626
});

‎src/components/browser/BrowserContent.tsx‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import {
1313
export const BrowserContent = () => {
1414
const { socket } = useSocketStore();
1515

16-
const [tabs, setTabs] = useState<string[]>(["current"]);
16+
const [tabs, setTabs] = useState<string[]>(["Loading..."]);
1717
const [tabIndex, setTabIndex] = React.useState(0);
1818
const [showOutputData, setShowOutputData] = useState(false);
1919
const { browserWidth } = useBrowserDimensionsStore();
@@ -125,7 +125,7 @@ export const BrowserContent = () => {
125125
useEffect(() => {
126126
getCurrentTabs()
127127
.then((response) => {
128-
if (response) {
128+
if (response && response.length > 0) {
129129
setTabs(response);
130130
}
131131
})

0 commit comments

Comments
 (0)