feat(HNT-2086): validate inbound crawl messages and the publisher list

Author: mmiermans

packages/crawl-common/src/index.ts

@@ -21,3 +21,10 @@ export type {
   LiveArticle,
   PublisherList,
 } from './types/index.js';
+export {
+  MessageValidationError,
+  validateCrawlArticleMessage,
+  validateCrawlArticleDiscoveryMessage,
+  validateLiveArticle,
+  validatePublisherList,
+} from './validation/index.js';

packages/crawl-common/src/validation/index.ts

@@ -0,0 +1,7 @@
+export {
+  MessageValidationError,
+  validateCrawlArticleMessage,
+  validateCrawlArticleDiscoveryMessage,
+  validateLiveArticle,
+  validatePublisherList,
+} from './messages.js';

packages/crawl-common/src/validation/messages.spec.ts

@@ -0,0 +1,270 @@
+import { describe, expect, it } from 'vitest';
+import {
+  MessageValidationError,
+  validateCrawlArticleMessage,
+  validateCrawlArticleDiscoveryMessage,
+  validatePublisherList,
+} from './messages.js';
+
+/** Return a shallow copy of obj without the given key. */
+function omit<T extends object>(obj: T, key: keyof T): Partial<T> {
+  const copy = { ...obj };
+  delete copy[key];
+  return copy;
+}
+
+const VALID_ARTICLE = {
+  url: 'https://example.com/news/article-1',
+  source_url: 'https://example.com/news',
+  crawl_id: 'crawl-001',
+  enqueued_at: '2025-06-01T12:00:00Z',
+};
+
+const VALID_CORPUS_ITEM = {
+  external_id: 'ext-123',
+  title: 'Original Headline',
+  excerpt: 'Original excerpt text',
+  authors: [{ name: 'Jane Doe' }],
+  status: 'CORPUS',
+  language: 'EN',
+  publisher: 'Example News',
+  image_url: 'https://s3.amazonaws.com/image.jpg',
+  topic: 'TECHNOLOGY',
+  is_time_sensitive: false,
+};
+
+const VALID_DISCOVERY = {
+  url: 'https://example.com/news',
+  interval_minutes: 20,
+  contexts: [{ surface_id: 'NEW_TAB_EN_US', topic: 'technology' }],
+};
+
+describe('validateCrawlArticleMessage', () => {
+  it('accepts a discovered-article message without corpus_item', () => {
+    expect(validateCrawlArticleMessage(VALID_ARTICLE)).toEqual(VALID_ARTICLE);
+  });
+
+  it('accepts a live-article message with a full corpus_item', () => {
+    const message = { ...VALID_ARTICLE, corpus_item: VALID_CORPUS_ITEM };
+    expect(validateCrawlArticleMessage(message)).toEqual(message);
+  });
+
+  it('keeps a blank excerpt, which curators may leave empty', () => {
+    const message = {
+      ...VALID_ARTICLE,
+      corpus_item: { ...VALID_CORPUS_ITEM, excerpt: '' },
+    };
+    expect(validateCrawlArticleMessage(message).corpus_item?.excerpt).toBe('');
+  });
+
+  it('keeps a blank publisher, which the Corpus DB may store empty', () => {
+    const message = {
+      ...VALID_ARTICLE,
+      corpus_item: { ...VALID_CORPUS_ITEM, publisher: '' },
+    };
+    expect(validateCrawlArticleMessage(message).corpus_item?.publisher).toBe(
+      '',
+    );
+  });
+
+  it.each([null, undefined, 42, 'string', []])(
+    'rejects a non-object payload (%s)',
+    (raw) => {
+      expect(() => validateCrawlArticleMessage(raw)).toThrow(
+        MessageValidationError,
+      );
+    },
+  );
+
+  it.each(['url', 'source_url', 'crawl_id', 'enqueued_at'] as const)(
+    'rejects a missing %s',
+    (field) => {
+      expect(() =>
+        validateCrawlArticleMessage(omit(VALID_ARTICLE, field)),
+      ).toThrow(new RegExp(field));
+    },
+  );
+
+  it('rejects an empty url', () => {
+    expect(() =>
+      validateCrawlArticleMessage({ ...VALID_ARTICLE, url: '   ' }),
+    ).toThrow(/url must be a non-empty string/);
+  });
+
+  it('rejects a non-string crawl_id', () => {
+    expect(() =>
+      validateCrawlArticleMessage({ ...VALID_ARTICLE, crawl_id: 7 }),
+    ).toThrow(/crawl_id/);
+  });
+
+  it('rejects a corpus_item missing a required field', () => {
+    const corpus = omit(VALID_CORPUS_ITEM, 'topic');
+    expect(() =>
+      validateCrawlArticleMessage({ ...VALID_ARTICLE, corpus_item: corpus }),
+    ).toThrow(/corpus_item.topic/);
+  });
+
+  it('rejects a corpus_item with a non-boolean is_time_sensitive', () => {
+    expect(() =>
+      validateCrawlArticleMessage({
+        ...VALID_ARTICLE,
+        corpus_item: { ...VALID_CORPUS_ITEM, is_time_sensitive: 'no' },
+      }),
+    ).toThrow(/is_time_sensitive must be a boolean/);
+  });
+
+  it('rejects a corpus_item author without a name', () => {
+    expect(() =>
+      validateCrawlArticleMessage({
+        ...VALID_ARTICLE,
+        corpus_item: { ...VALID_CORPUS_ITEM, authors: [{}] },
+      }),
+    ).toThrow(/authors\[0\].name/);
+  });
+
+  it('accepts and preserves a positive refresh_interval_minutes', () => {
+    const message = { ...VALID_ARTICLE, refresh_interval_minutes: 20 };
+    expect(validateCrawlArticleMessage(message)).toEqual(message);
+  });
+
+  it('omits refresh_interval_minutes when absent (older messages)', () => {
+    const result = validateCrawlArticleMessage(VALID_ARTICLE);
+    expect(result.refresh_interval_minutes).toBeUndefined();
+  });
+
+  it('rejects a non-numeric refresh_interval_minutes', () => {
+    expect(() =>
+      validateCrawlArticleMessage({
+        ...VALID_ARTICLE,
+        refresh_interval_minutes: '20',
+      }),
+    ).toThrow(/refresh_interval_minutes/);
+  });
+
+  it.each([0, -5])(
+    'rejects a non-positive refresh_interval_minutes (%s)',
+    (value) => {
+      expect(() =>
+        validateCrawlArticleMessage({
+          ...VALID_ARTICLE,
+          refresh_interval_minutes: value,
+        }),
+      ).toThrow(/refresh_interval_minutes must be a positive number/);
+    },
+  );
+});
+
+describe('validateCrawlArticleDiscoveryMessage', () => {
+  it('accepts a well-formed discovery message', () => {
+    expect(validateCrawlArticleDiscoveryMessage(VALID_DISCOVERY)).toEqual(
+      VALID_DISCOVERY,
+    );
+  });
+
+  it('rejects a missing interval_minutes', () => {
+    expect(() =>
+      validateCrawlArticleDiscoveryMessage(
+        omit(VALID_DISCOVERY, 'interval_minutes'),
+      ),
+    ).toThrow(/interval_minutes must be a finite number/);
+  });
+
+  it('rejects a non-numeric interval_minutes', () => {
+    expect(() =>
+      validateCrawlArticleDiscoveryMessage({
+        ...VALID_DISCOVERY,
+        interval_minutes: '20',
+      }),
+    ).toThrow(/interval_minutes/);
+  });
+
+  it.each([0, -5])('rejects a non-positive interval_minutes (%s)', (value) => {
+    expect(() =>
+      validateCrawlArticleDiscoveryMessage({
+        ...VALID_DISCOVERY,
+        interval_minutes: value,
+      }),
+    ).toThrow(/interval_minutes must be a positive number/);
+  });
+
+  it('rejects an empty contexts array', () => {
+    expect(() =>
+      validateCrawlArticleDiscoveryMessage({
+        ...VALID_DISCOVERY,
+        contexts: [],
+      }),
+    ).toThrow(/contexts must not be empty/);
+  });
+
+  it('rejects a context missing surface_id', () => {
+    expect(() =>
+      validateCrawlArticleDiscoveryMessage({
+        ...VALID_DISCOVERY,
+        contexts: [{ topic: 'technology' }],
+      }),
+    ).toThrow(/contexts\[0\].surface_id/);
+  });
+});
+
+describe('validatePublisherList', () => {
+  const VALID_LIST = {
+    pages: [VALID_DISCOVERY],
+    live_articles: [{ url: VALID_ARTICLE.url, corpus_item: VALID_CORPUS_ITEM }],
+  };
+
+  it('accepts a well-formed publisher list', () => {
+    expect(validatePublisherList(VALID_LIST)).toEqual(VALID_LIST);
+  });
+
+  it('accepts empty page and live-article lists', () => {
+    expect(validatePublisherList({ pages: [], live_articles: [] })).toEqual({
+      pages: [],
+      live_articles: [],
+    });
+  });
+
+  it('rejects a non-array pages field', () => {
+    expect(() =>
+      validatePublisherList({ pages: {}, live_articles: [] }),
+    ).toThrow(/pages must be an array/);
+  });
+
+  it('rejects an invalid page entry', () => {
+    expect(() =>
+      validatePublisherList({
+        pages: [omit(VALID_DISCOVERY, 'url')],
+        live_articles: [],
+      }),
+    ).toThrow(/url must be a non-empty string/);
+  });
+
+  it('rejects duplicate page URLs', () => {
+    expect(() =>
+      validatePublisherList({
+        pages: [VALID_DISCOVERY, VALID_DISCOVERY],
+        live_articles: [],
+      }),
+    ).toThrow(/pages has a duplicate url/);
+  });
+
+  it('rejects duplicate live-article URLs', () => {
+    const entry = { url: VALID_ARTICLE.url, corpus_item: VALID_CORPUS_ITEM };
+    expect(() =>
+      validatePublisherList({ pages: [], live_articles: [entry, entry] }),
+    ).toThrow(/live_articles has a duplicate url/);
+  });
+
+  it('rejects a live article with a malformed corpus_item', () => {
+    expect(() =>
+      validatePublisherList({
+        pages: [],
+        live_articles: [
+          {
+            url: VALID_ARTICLE.url,
+            corpus_item: omit(VALID_CORPUS_ITEM, 'topic'),
+          },
+        ],
+      }),
+    ).toThrow(/corpus_item.topic/);
+  });
+});