fix(playwright): track and drain in-flight rendering detections on teardown

docs/public-api/crawlee-playwright.api.md

@@ -74,6 +74,8 @@ export class AdaptivePlaywrightCrawler<ContextExtension = Dictionary<never>, Ext
     constructor(options?: AdaptivePlaywrightCrawlerOptions<ContextExtension, ExtendedContext, Routes, StatisticStateExtension>);
     // (undocumented)
     protected buildContextPipeline(): ContextPipeline_2<CrawlingContext_2, AdaptivePlaywrightCrawlerContext>;
+    drainRenderingDetections(): Promise<void>;
+    get inFlightRenderingTypeDetectionCount(): number;
     // (undocumented)
     protected init(): Promise<void>;
     // (undocumented)

packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts

@@ -322,6 +322,11 @@
      */
     readonly #attemptWritePolicy: Partial<StorageWritePolicy>;
 
+    /**
+     * Holds currently in-flight rendering detection promises.
+     */
+    readonly #activeDetections = new Set<Promise<unknown>>();
+
     #teardownHooks: (() => Promise<unknown>)[] = [];
 
     constructor(
@@ -522,7 +527,7 @@
         });
     }
 
     private async adaptCheerioContext(cheerioContext: CheerioCrawlingContext) {
         return {
             get page(): Page {
                 throw new Error('Page object was used in HTTP-only request handler');
@@ -545,7 +550,7 @@
         };
     }
 
     private async adaptPlaywrightContext(playwrightContext: PlaywrightCrawlingContext) {
         // Capture the original response to avoid infinite recursion when the getter is copied to the context
         const originalResponse = playwrightContext.response;
 
@@ -591,7 +596,7 @@
      * time, before the `try`* - the `ok: false` branch of the returned {@apilink Result} carries no
      * result, and failed attempts are routine here. The caller owns the outcome and disposal.
      */
     private async crawlOne(
         renderingType: RenderingType,
         context: CrawlingContext,
         useStateFunction: (defaultValue?: Dictionary) => Promise<Dictionary>,
@@ -753,40 +758,49 @@
             await browserRun.result.commit();
 
             if (shouldDetectRenderingType) {
-                crawlingContext.log.debug(`Detecting rendering type for ${crawlingContext.request.url}`);
-                // The detection attempt's transaction is never committed - its writes exist only for the
-                // result comparison.
-                const plainHTTPRun = await this.crawlOne(
-                    'static',
-                    crawlingContext,
-                    stateTracker.getStateCopy.bind(stateTracker),
-                    transactions,
-                );
+                const detectionPromise = (async () => {
+                    crawlingContext.log.debug(`Detecting rendering type for ${crawlingContext.request.url}`);
+                    // The detection attempt's transaction is never committed - its writes exist only for the
+                    // result comparison.
+                    const plainHTTPRun = await this.crawlOne(
+                        'static',
+                        crawlingContext,
+                        stateTracker.getStateCopy.bind(stateTracker),
+                        transactions,
+                    );
 
-                const detectionResult: RenderingType | undefined = (() => {
-                    if (!plainHTTPRun.ok) {
-                        return 'clientOnly';
-                    }
+                    const detectionResult: RenderingType | undefined = (() => {
+                        if (!plainHTTPRun.ok) {
+                            return 'clientOnly';
+                        }
 
-                    const comparisonResult = this.#resultComparator(plainHTTPRun.result, browserRun.result);
-                    if (comparisonResult === true || comparisonResult === 'equal') {
-                        return 'static';
-                    }
+                        const comparisonResult = this.#resultComparator(plainHTTPRun.result, browserRun.result);
+                        if (comparisonResult === true || comparisonResult === 'equal') {
+                            return 'static';
+                        }
 
-                    if (comparisonResult === false || comparisonResult === 'different') {
-                        return 'clientOnly';
-                    }
+                        if (comparisonResult === false || comparisonResult === 'different') {
+                            return 'clientOnly';
+                        }
+
+                        return undefined;
+                    })();
 
-                    return undefined;
+                    crawlingContext.log.debug(
+                        `Detected rendering type ${detectionResult} for ${crawlingContext.request.url}`,
+                    );
+
+                    if (detectionResult !== undefined) {
+                        this.#renderingTypePredictor.value.storeResult(crawlingContext.request, detectionResult);
+                    }
                 })();
 
-                crawlingContext.log.debug(
-                    `Detected rendering type ${detectionResult} for ${crawlingContext.request.url}`,
-                );
+                this.#activeDetections.add(detectionPromise);
+                void detectionPromise.finally(() => {
+                    this.#activeDetections.delete(detectionPromise);
+                });
 
-                if (detectionResult !== undefined) {
-                    this.#renderingTypePredictor.value.storeResult(crawlingContext.request, detectionResult);
-                }
+                await detectionPromise;
             }
         } finally {
             // A still-open transaction here belongs to a discarded attempt - roll it back, then release.
@@ -797,7 +811,7 @@
         }
     }
 
     private async enqueueLinks(
         urls: readonly string[],
         options: EnqueueLinksOptions,
         request: RestrictedCrawlingContext['request'],
@@ -820,7 +834,7 @@
         });
     }
 
     private createLogProxy(log: CrawleeLogger, logs: LogProxyCall[]) {
         return new Proxy(log, {
             get(target: CrawleeLogger, propertyName: (typeof proxyLogMethods)[number]) {
                 if (proxyLogMethods.includes(propertyName)) {
@@ -839,7 +853,24 @@
         });
     }
 
+    /**
+     * Number of rendering-type detections currently running in the background.
+     */
+    get inFlightRenderingTypeDetectionCount(): number {
+        return this.#activeDetections.size;
+    }
+
+    /**
+     * Waits for all in-flight rendering-type detections to settle.
+     */
+    async drainRenderingDetections(): Promise<void> {
+        while (this.#activeDetections.size > 0) {
+            await Promise.allSettled(Array.from(this.#activeDetections));
+        }
+    }
+
     override async teardown() {
+        await this.drainRenderingDetections();
         await super.teardown();
         // Mirrors the owned-only `initialize()` in `init()` - without this, the predictor we built keeps its
         // PERSIST_STATE listener registered after the crawl and never gets a final write.

test/core/crawlers/adaptive_playwright_crawler.test.ts

@@ -1023,4 +1023,119 @@ describe('AdaptivePlaywrightCrawler', () => {
 
         expect(lastDynamicRequestUserAgent).toBe(distinctiveUserAgent);
     });
+
+    describe('in-flight rendering type detections', () => {
+        test('are counted while running and settled once the crawl ends', async () => {
+            const renderingTypePredictor = makeRiggedRenderingTypePredictor({
+                detectionProbabilityRecommendation: 1,
+                renderingType: 'clientOnly',
+            });
+
+            // A detection re-runs the user handler over plain HTTP, so the handler can observe whether
+            // it is itself running inside a detection.
+            const observedCounts: number[] = [];
+
+            const crawler = await makeOneshotCrawler(
+                {
+                    requestHandler: async () => {
+                        observedCounts.push(crawler.inFlightRenderingTypeDetectionCount);
+                    },
+                    renderingTypePredictor,
+                },
+                [`http://${HOSTNAME}:${port}/static`],
+            );
+
+            await crawler.run();
+
+            expect(observedCounts).toEqual([0, 1]);
+            expect(crawler.inFlightRenderingTypeDetectionCount).toBe(0);
+        });
+
+        // A crawler whose plain-HTTP detection attempt parks until released, so that a detection can be
+        // observed mid-flight.
+        const makeCrawlerWithBlockedDetection = async () => {
+            const renderingTypePredictor = makeRiggedRenderingTypePredictor({
+                detectionProbabilityRecommendation: 1,
+                renderingType: 'clientOnly',
+            });
+
+            let releaseDetection!: () => void;
+            const detectionReleased = new Promise<void>((resolve) => {
+                releaseDetection = resolve;
+            });
+            let announceDetection!: () => void;
+            const detectionStarted = new Promise<void>((resolve) => {
+                announceDetection = resolve;
+            });
+
+            let handlerCalls = 0;
+            const crawler = await makeOneshotCrawler(
+                {
+                    requestHandler: async () => {
+                        handlerCalls += 1;
+                        // The second call is the plain-HTTP detection attempt.
+                        if (handlerCalls === 2) {
+                            announceDetection();
+                            await detectionReleased;
+                        }
+                    },
+                    renderingTypePredictor,
+                },
+                [`http://${HOSTNAME}:${port}/static`],
+            );
+
+            return { crawler, renderingTypePredictor, detectionStarted, releaseDetection };
+        };
+
+        test('hold up drainRenderingDetections until their result is stored', async () => {
+            const { crawler, renderingTypePredictor, detectionStarted, releaseDetection } =
+                await makeCrawlerWithBlockedDetection();
+
+            const runPromise = crawler.run();
+            await detectionStarted;
+
+            expect(crawler.inFlightRenderingTypeDetectionCount).toBe(1);
+
+            let drained = false;
+            const drainPromise = crawler.drainRenderingDetections().then(() => {
+                drained = true;
+            });
+
+            await sleep(100);
+            expect(drained).toBe(false);
+            expect(renderingTypePredictor.storeResult).not.toHaveBeenCalled();
+
+            releaseDetection();
+            await drainPromise;
+
+            expect(drained).toBe(true);
+            expect(crawler.inFlightRenderingTypeDetectionCount).toBe(0);
+            expect(renderingTypePredictor.storeResult).toHaveBeenCalledOnce();
+
+            await runPromise;
+        });
+
+        test('hold up teardown until their result is stored', async () => {
+            const { crawler, renderingTypePredictor, detectionStarted, releaseDetection } =
+                await makeCrawlerWithBlockedDetection();
+
+            const runPromise = crawler.run();
+            await detectionStarted;
+
+            let tornDown = false;
+            const teardownPromise = crawler.teardown().then(() => {
+                tornDown = true;
+            });
+
+            await sleep(100);
+            expect(tornDown).toBe(false);
+
+            releaseDetection();
+            await teardownPromise;
+
+            expect(renderingTypePredictor.storeResult).toHaveBeenCalledOnce();
+
+            await runPromise;
+        });
+    });
 });