feat(HNT-2086): add crawl domain message and event types

Author: mmiermans

packages/crawl-common/src/index.ts

@@ -12,7 +12,12 @@ export type {
 export type {
   CorpusItem,
   CrawlArticleMessage,
+  DiscoveryContext,
+  CrawlArticleDiscoveryMessage,
   ArticleAuthor,
   ArticleBreadcrumb,
   ArticleEvent,
+  ArticleDiscoveryEvent,
+  LiveArticle,
+  PublisherList,
 } from './types/index.js';

packages/crawl-common/src/types/events.ts

@@ -26,3 +26,24 @@ export interface ArticleEvent {
   breadcrumbs?: ArticleBreadcrumb[];
   language?: string;
 }
+
+/**
+ * Event published to the article-discoveries Pub/Sub topic and
+ * written to the crawl.article_discoveries BigQuery table via a
+ * BigQuery subscription. One event per discovered article and
+ * context, so topic and surface_id come from the context the
+ * page was crawled for, while source_url is that page.
+ */
+export interface ArticleDiscoveryEvent {
+  url: string;
+  source_url: string;
+  crawled_at: string;
+  published_at?: string;
+  headline?: string;
+  authors?: ArticleAuthor[];
+  summary?: string;
+  language?: string;
+  topic?: string;
+  page_position?: number;
+  surface_id: string;
+}

packages/crawl-common/src/types/index.ts

@@ -1,6 +1,14 @@
-export type { CorpusItem, CrawlArticleMessage } from './messages.js';
+export type {
+  CorpusItem,
+  CrawlArticleMessage,
+  DiscoveryContext,
+  CrawlArticleDiscoveryMessage,
+  LiveArticle,
+  PublisherList,
+} from './messages.js';
 export type {
   ArticleAuthor,
   ArticleBreadcrumb,
   ArticleEvent,
+  ArticleDiscoveryEvent,
 } from './events.js';

packages/crawl-common/src/types/messages.ts

@@ -27,5 +27,53 @@ export interface CrawlArticleMessage {
   source_url: string;
   crawl_id: string;
   enqueued_at: string;
+  // Refresh window for this article, set by the producer: discovery uses
+  // the article fetch TTL, the agent uses the live-article interval. The
+  // worker gates re-fetch and writes its fetch claim against this value.
+  // Optional so a rolling deploy does not reject messages enqueued before
+  // the field existed; the worker falls back to the configured fetch TTL.
+  refresh_interval_minutes?: number;
   corpus_item?: CorpusItem;
 }
+
+/**
+ * Surface and topic a discovered page is crawled for. A page
+ * can be crawled for several surfaces, so each discovery job
+ * carries one context per (surface, topic) pair.
+ */
+export interface DiscoveryContext {
+  surface_id: string;
+  topic: string;
+}
+
+/**
+ * Pub/Sub message consumed from the crawl-article-discovery
+ * subscription. Tells the discovery worker which page to crawl,
+ * how recently it may have been crawled, and the contexts to
+ * attribute discovered articles to.
+ */
+export interface CrawlArticleDiscoveryMessage {
+  url: string;
+  interval_minutes: number;
+  contexts: DiscoveryContext[];
+}
+
+/**
+ * A live (curated) article in the agent's publisher list. The agent
+ * enqueues a crawl-article job carrying this corpus_item so the
+ * worker can re-extract and sync editorial metadata.
+ */
+export interface LiveArticle {
+  url: string;
+  corpus_item: CorpusItem;
+}
+
+/**
+ * The agent's publisher list, loaded from JSON. pages drive
+ * discovery crawls; live_articles are re-crawled directly to keep
+ * curated metadata fresh. Phase 5 replaces this with the Corpus API.
+ */
+export interface PublisherList {
+  pages: CrawlArticleDiscoveryMessage[];
+  live_articles: LiveArticle[];
+}

packages/crawl-common/vitest.config.ts

@@ -4,5 +4,8 @@ export default defineConfig({
   test: {
     globals: true,
     include: ['src/**/*.spec.ts', 'src/**/*.integration.ts'],
+    // Emulator/container integration tests need headroom over the 5s
+    // default, especially when packages run in parallel.
+    testTimeout: 30_000,
   },
 });