Skip to content

Commit d7fdf3d

Browse files
committed
feat(route): enhance Substack feeds
Refs #22800
1 parent 0ab0ad9 commit d7fdf3d

6 files changed

Lines changed: 614 additions & 16 deletions

File tree

lib/config.test.ts

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,15 @@ describe('config', () => {
6161
delete process.env.MEDIUM_COOKIE_34;
6262
});
6363

64+
it('substack cookie', async () => {
65+
process.env.SUBSTACK_COOKIE = 'substack.sid=session';
66+
67+
const { config } = await import('./config');
68+
expect(config.substack.cookie).toBe('substack.sid=session');
69+
70+
delete process.env.SUBSTACK_COOKIE;
71+
});
72+
6473
it('discourse config', async () => {
6574
process.env.DISCOURSE_CONFIG_12 = JSON.stringify({ a: 1 });
6675
process.env.DISCOURSE_CONFIG_34 = JSON.stringify({ b: 2 });

lib/config.ts

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,7 @@ type ConfigEnvKeys =
197197
| 'SPOTIFY_CLIENT_SECRET'
198198
| 'SPOTIFY_REFRESHTOKEN'
199199
| 'SSPAI_BEARERTOKEN'
200+
| 'SUBSTACK_COOKIE'
200201
| 'TELEGRAM_TOKEN'
201202
| 'TELEGRAM_SESSION'
202203
| 'TELEGRAM_API_ID'
@@ -616,6 +617,9 @@ export type Config = {
616617
sspai: {
617618
bearertoken?: string;
618619
};
620+
substack: {
621+
cookie?: string;
622+
};
619623
telegram: {
620624
token?: string;
621625
session?: string;
@@ -1119,6 +1123,9 @@ const calculateValue = () => {
11191123
sspai: {
11201124
bearertoken: envs.SSPAI_BEARERTOKEN,
11211125
},
1126+
substack: {
1127+
cookie: envs.SUBSTACK_COOKIE,
1128+
},
11221129
telegram: {
11231130
token: envs.TELEGRAM_TOKEN,
11241131
session: envs.TELEGRAM_SESSION,

lib/routes/substack/notes.ts

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
import { config } from '@/config';
2+
import InvalidParameterError from '@/errors/types/invalid-parameter';
3+
import type { Route } from '@/types';
4+
import { ViewType } from '@/types';
5+
import cache from '@/utils/cache';
6+
import ofetch from '@/utils/ofetch';
7+
import { buildSubstackNoteItem, isSubstackNote, type SubstackActivityFeed, type SubstackPost, type SubstackPublicationResponse } from '@/utils/substack';
8+
import { isValidHost } from '@/utils/valid-host';
9+
10+
export const route: Route = {
11+
path: '/notes/:user',
12+
categories: ['blog'],
13+
view: ViewType.SocialMedia,
14+
example: '/substack/notes/norsemanmarkettiming',
15+
parameters: { user: 'Substack publication subdomain' },
16+
features: {
17+
requireConfig: [
18+
{
19+
name: 'SUBSTACK_COOKIE',
20+
optional: true,
21+
description: 'Complete Cookie header from a logged-in Substack session. Use only on a trusted self-hosted instance; protected Notes are returned only when that account already has access.',
22+
},
23+
],
24+
requirePuppeteer: false,
25+
antiCrawler: true,
26+
supportBT: false,
27+
supportPodcast: false,
28+
supportScihub: false,
29+
},
30+
name: 'Notes',
31+
maintainers: ['pseudoyu'],
32+
handler,
33+
description: `Returns the Notes activity published by a Substack publication's author. Set SUBSTACK\\_COOKIE on a trusted RSSHub instance to use an authenticated Substack session; protected activity is returned only when that account already has access.`,
34+
};
35+
36+
async function handler(ctx) {
37+
const user = ctx.req.param('user');
38+
39+
if (!isValidHost(user)) {
40+
throw new InvalidParameterError('Invalid user');
41+
}
42+
43+
const baseUrl = `https://${user}.substack.com`;
44+
const cookie = config.substack.cookie;
45+
const publicationHeaders = {
46+
Referer: baseUrl,
47+
'User-Agent': config.ua,
48+
...(cookie && { Cookie: cookie }),
49+
};
50+
const readerHeaders = {
51+
Referer: 'https://substack.com/',
52+
'User-Agent': config.ua,
53+
...(cookie && { Cookie: cookie }),
54+
};
55+
56+
const archive = await cache.tryGet(`substack:archive-profile:v2:${user}`, () =>
57+
ofetch<SubstackPost[]>(`${baseUrl}/api/v1/archive`, {
58+
headers: publicationHeaders,
59+
query: {
60+
sort: 'new',
61+
search: '',
62+
offset: 0,
63+
limit: 1,
64+
},
65+
})
66+
);
67+
const publicationId = archive[0]?.publication_id;
68+
if (!publicationId) {
69+
throw new Error(`Unable to resolve the Substack publication for ${user}`);
70+
}
71+
72+
const publicationResponse = await cache.tryGet(`substack:publication:v2:${publicationId}`, () =>
73+
ofetch<SubstackPublicationResponse>(`https://substack.com/api/v1/publication/public/${publicationId}`, {
74+
headers: readerHeaders,
75+
})
76+
);
77+
const publication = publicationResponse.pub;
78+
const profileId = publication?.author_id || publication?.primary_user_id;
79+
if (!profileId || !publication?.author_handle) {
80+
throw new Error(`Unable to resolve the primary Substack author for ${user}`);
81+
}
82+
83+
const profile = {
84+
id: profileId,
85+
name: publication.author_name,
86+
handle: publication.author_handle,
87+
photo_url: publication.author_photo_url,
88+
bio: publication.hero_text,
89+
primaryPublication: {
90+
logo_url: publication.logo_url,
91+
},
92+
};
93+
const activity = await cache.tryGet(`substack:notes:v2:${profileId}`, () =>
94+
ofetch<SubstackActivityFeed>(`https://substack.com/api/v1/reader/feed/profile/${profileId}`, {
95+
query: {
96+
types: 'note',
97+
},
98+
headers: readerHeaders,
99+
})
100+
);
101+
const item = (activity.items ?? []).filter((activityItem) => isSubstackNote(activityItem)).map((activityItem) => buildSubstackNoteItem(activityItem.comment, profile));
102+
const handle = publication.author_handle;
103+
const name = publication.author_name || user;
104+
105+
return {
106+
title: `${name} on Substack Notes`,
107+
description: publication.hero_text || `${name}'s Substack Notes`,
108+
link: `https://substack.com/@${handle}/notes`,
109+
image: publication.author_photo_url || publication.logo_url || '',
110+
item,
111+
};
112+
}

lib/routes/substack/subscribe.ts

Lines changed: 47 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,40 @@
1+
import pMap from 'p-map';
2+
3+
import { config } from '@/config';
14
import InvalidParameterError from '@/errors/types/invalid-parameter';
25
import type { Route } from '@/types';
36
import { ViewType } from '@/types';
7+
import cache from '@/utils/cache';
8+
import logger from '@/utils/logger';
49
import ofetch from '@/utils/ofetch';
5-
import { parseDate } from '@/utils/parse-date';
610
import parser from '@/utils/rss-parser';
11+
import { buildSubstackPostItem, getSubstackPostSlug, type SubstackPost } from '@/utils/substack';
712
import { isValidHost } from '@/utils/valid-host';
813

914
export const route: Route = {
1015
path: '/subscribe/:user',
1116
categories: ['blog'],
1217
view: ViewType.SocialMedia,
13-
example: '/substack/subscribe/mangoread',
14-
parameters: { user: 'Username of the Substack' },
18+
example: '/substack/subscribe/norsemanmarkettiming',
19+
parameters: { user: 'Substack publication subdomain' },
1520
features: {
16-
requireConfig: false,
21+
requireConfig: [
22+
{
23+
name: 'SUBSTACK_COOKIE',
24+
optional: true,
25+
description: 'Complete Cookie header from a logged-in Substack session. Use only on a trusted self-hosted instance; protected posts are returned only when that account already has access.',
26+
},
27+
],
1728
requirePuppeteer: false,
18-
antiCrawler: false,
29+
antiCrawler: true,
1930
supportBT: false,
2031
supportPodcast: false,
2132
supportScihub: false,
2233
},
23-
name: 'Substack Subscription',
34+
name: 'Subscription',
2435
maintainers: ['pseudoyu'],
2536
handler,
37+
description: `Fetches each post from Substack's post API so the feed contains the body available to the requester. Set SUBSTACK\\_COOKIE on a trusted RSSHub instance to use an authenticated Substack session; paid posts are returned only when that account already has access.`,
2638
};
2739

2840
async function handler(ctx) {
@@ -32,21 +44,40 @@ async function handler(ctx) {
3244
throw new InvalidParameterError('Invalid user');
3345
}
3446

35-
const response = await ofetch(`https://${user}.substack.com/feed`);
47+
const baseUrl = `https://${user}.substack.com`;
48+
const cookie = config.substack.cookie;
49+
const headers = {
50+
Referer: baseUrl,
51+
'User-Agent': config.ua,
52+
...(cookie && { Cookie: cookie }),
53+
};
54+
const response = await ofetch<string>(`${baseUrl}/feed`, { headers });
3655
const feed = await parser.parseString(response);
3756

57+
const item = await pMap(
58+
feed.items,
59+
async (item) => {
60+
const slug = getSubstackPostSlug(item.link);
61+
if (!slug) {
62+
return buildSubstackPostItem(item, undefined, user);
63+
}
64+
65+
try {
66+
const post = await cache.tryGet(`substack:post:v2:${user}:${slug}`, () => ofetch<SubstackPost>(`${baseUrl}/api/v1/posts/${encodeURIComponent(slug)}`, { headers }));
67+
return buildSubstackPostItem(item, post, user);
68+
} catch (error) {
69+
logger.warn(`[substack/subscribe] Failed to fetch post ${slug}: ${error instanceof Error ? error.message : String(error)}`);
70+
return buildSubstackPostItem(item, undefined, user);
71+
}
72+
},
73+
{ concurrency: 5 }
74+
);
75+
3876
return {
3977
title: feed.title ?? 'Substack',
4078
description: feed.description ?? `${user}'s Substack`,
41-
link: feed.link ?? `https://${user}.substack.com`,
79+
link: feed.link ?? baseUrl,
4280
image: feed.image?.url ?? '',
43-
item: feed.items.map((item) => ({
44-
title: item.title ?? 'Untitled',
45-
description: item['content:encoded'] ?? item.content ?? '',
46-
link: item.link ?? '',
47-
pubDate: item.pubDate ? parseDate(item.pubDate) : undefined,
48-
guid: item.guid ?? '',
49-
author: item.creator ?? user,
50-
})),
81+
item,
5182
};
5283
}

lib/utils/substack.test.ts

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
import { describe, expect, it } from 'vitest';
2+
3+
import { buildSubstackNoteItem, buildSubstackPostItem, getSubstackPostSlug, renderSubstackNote } from './substack';
4+
5+
describe('Substack utilities', () => {
6+
it('extracts a post slug from a canonical URL', () => {
7+
expect(getSubstackPostSlug('https://example.substack.com/p/a-paid-post')).toBe('a-paid-post');
8+
expect(getSubstackPostSlug('https://example.substack.com/archive')).toBeUndefined();
9+
});
10+
11+
it('uses the post API body and metadata instead of the feed preview', () => {
12+
const item = buildSubstackPostItem(
13+
{
14+
title: 'Feed title',
15+
link: 'https://example.substack.com/p/full-post',
16+
'content:encoded': '<p>Feed preview</p>',
17+
creator: 'Feed author',
18+
},
19+
{
20+
id: 42,
21+
title: 'Full post',
22+
slug: 'full-post',
23+
canonical_url: 'https://example.substack.com/p/full-post',
24+
body_html: '<p>Subscriber-visible body</p>',
25+
post_date: '2026-07-17T12:00:00.000Z',
26+
publishedBylines: [{ name: 'Author' }],
27+
postTags: [{ name: 'Markets' }],
28+
cover_image: 'https://example.com/cover.jpg',
29+
},
30+
'example'
31+
);
32+
33+
expect(item).toMatchObject({
34+
title: 'Full post',
35+
description: '<p>Subscriber-visible body</p>',
36+
link: 'https://example.substack.com/p/full-post',
37+
guid: 'https://example.substack.com/p/full-post',
38+
author: 'Author',
39+
category: ['Markets'],
40+
image: 'https://example.com/cover.jpg',
41+
});
42+
expect(item.pubDate).toEqual(new Date('2026-07-17T12:00:00.000Z'));
43+
});
44+
45+
it('keeps the feed body when post detail is unavailable', () => {
46+
const item = buildSubstackPostItem(
47+
{
48+
title: 'Feed title',
49+
link: 'https://example.substack.com/p/fallback',
50+
'content:encoded': '<p>Feed body</p>',
51+
guid: 'post-42',
52+
},
53+
undefined,
54+
'example'
55+
);
56+
57+
expect(item).toMatchObject({
58+
title: 'Feed title',
59+
description: '<p>Feed body</p>',
60+
link: 'https://example.substack.com/p/fallback',
61+
guid: 'https://example.substack.com/p/fallback',
62+
});
63+
});
64+
65+
it('renders rich Notes content and attachments safely', () => {
66+
const note = {
67+
id: 123,
68+
body: 'A complete note',
69+
date: '2026-07-16T19:41:28.774Z',
70+
name: 'Writer',
71+
handle: 'writer',
72+
body_json: {
73+
type: 'doc',
74+
content: [
75+
{
76+
type: 'paragraph',
77+
content: [
78+
{ type: 'text', text: 'Bold & ', marks: [{ type: 'bold' }] },
79+
{ type: 'text', text: 'linked', marks: [{ type: 'link', attrs: { href: 'https://example.com/read' } }] },
80+
],
81+
},
82+
],
83+
},
84+
attachments: [
85+
{ type: 'image' as const, imageUrl: 'https://example.com/chart.png', imageWidth: 1200, imageHeight: 800 },
86+
{
87+
type: 'link' as const,
88+
linkMetadata: {
89+
url: 'https://example.com/source',
90+
title: 'Source',
91+
description: 'Details & context',
92+
},
93+
},
94+
],
95+
};
96+
97+
const description = renderSubstackNote(note);
98+
expect(description).toContain('<strong>Bold &amp; </strong>');
99+
expect(description).toContain('<a href="https://example.com/read">linked</a>');
100+
expect(description).toContain('<img src="https://example.com/chart.png" width="1200" height="800">');
101+
expect(description).toContain('<figcaption>Details &amp; context</figcaption>');
102+
103+
const item = buildSubstackNoteItem(note, { name: 'Writer', handle: 'writer' });
104+
expect(item).toMatchObject({
105+
title: 'A complete note',
106+
link: 'https://substack.com/@writer/note/c-123',
107+
guid: 'https://substack.com/@writer/note/c-123',
108+
author: [
109+
{
110+
name: 'Writer',
111+
url: 'https://substack.com/@writer',
112+
},
113+
],
114+
});
115+
expect(item.pubDate).toEqual(new Date('2026-07-16T19:41:28.774Z'));
116+
});
117+
118+
it('escapes a plain-text Note fallback', () => {
119+
expect(renderSubstackNote({ body: '<script>alert(1)</script>\nsecond line' })).toBe('<p>&lt;script&gt;alert(1)&lt;/script&gt;<br>second line</p>');
120+
});
121+
122+
it('handles attachment-only Notes and rejects unsafe attachment URLs', () => {
123+
const note = {
124+
id: 456,
125+
attachments: [
126+
{ type: 'image' as const, imageUrl: 'https://example.com/chart.heic' },
127+
{ type: 'link' as const, linkMetadata: { url: 'javascript:alert(1)', title: 'Unsafe' } },
128+
],
129+
};
130+
131+
const item = buildSubstackNoteItem(note, { name: 'Writer', handle: 'writer' });
132+
expect(item.title).toBe('Note by Writer');
133+
expect(item.description).toBe('<p><img src="https://example.com/chart.heic"></p>');
134+
});
135+
});

0 commit comments

Comments
 (0)