feat: nitro hook robots:init and robots:robots-txt:input

docs/content/3.nitro-api/2.nitro-hooks.md

@@ -3,36 +3,82 @@ title: Nitro Hooks
 description: Learn how to use Nitro hooks to modify the robots final output.
 ---
 
-## `'robots:config'`{lang="ts"}
+## `'robots:init'`{lang="ts"}
 
 **Type:** `(ctx: HookContext) => void | Promise<void>`{lang="ts"}
 
 ```ts
 interface HookContext {
   groups: RobotsGroupResolved[]
   sitemaps: string[]
-  context: 'robots.txt' | 'init'
-  event?: H3Event // undefined on `init`
+  errors: string[]
 }
 ```
 
-Modify the robots config before it's used to generate the indexing rules.
+Modify the robots config during Nitro initialization. This is called once when Nitro starts.
 
-This is called when Nitro starts `init` as well as when generating the robots.txt `robots.txt`.
+Use this hook when you need to fetch or compute robot rules at startup and cache them for all subsequent requests.
 
-```ts [server/plugins/robots-ignore-routes.ts]
+```ts [server/plugins/robots-init.ts]
 export default defineNitroPlugin((nitroApp) => {
-  nitroApp.hooks.hook('robots:config', async (ctx) => {
-    // extend the robot.txt rules at runtime
-    if (ctx.context === 'init') {
-      // probably want to cache this
-      const ignoredRoutes = await $fetch('/api/ignored-routes')
-      ctx.groups[0].disallow.push(...ignoredRoutes)
+  nitroApp.hooks.hook('robots:init', async (ctx) => {
+    // Fetch ignored routes at startup and cache them
+    const ignoredRoutes = await $fetch('/api/ignored-routes')
+    ctx.groups[0].disallow.push(...ignoredRoutes)
+  })
+})
+```
+
+## `'robots:robots-txt:input'`{lang="ts"}
+
+**Type:** `(ctx: HookContext) => void | Promise<void>`{lang="ts"}
+
+```ts
+interface HookContext {
+  groups: RobotsGroupResolved[]
+  sitemaps: string[]
+  errors: string[]
+  event: H3Event
+}
+```
+
+Modify the robots config before generating the robots.txt file. This is called on each robots.txt request.
+
+Use this hook when you need to customize the robots.txt output based on the request context (e.g., headers).
+
+```ts [server/plugins/robots-dynamic.ts]
+export default defineNitroPlugin((nitroApp) => {
+  nitroApp.hooks.hook('robots:robots-txt:input', async (ctx) => {
+    // Dynamically adjust rules based on request
+    const isDevelopment = ctx.event.headers.get('x-development') === 'true'
+    if (isDevelopment) {
+      ctx.groups[0].disallow.push('/staging/*')
     }
   })
 })
 ```
 
+## `'robots:config'`{lang="ts"} <Badge type="warning">Deprecated</Badge>
+
+**Type:** `(ctx: HookContext) => void | Promise<void>`{lang="ts"}
+
+::callout{type="warning"}
+This hook is deprecated. Use `robots:init` for initialization or `robots:robots-txt:input` for robots.txt generation instead.
+::
+
+```ts
+interface HookContext {
+  groups: RobotsGroupResolved[]
+  sitemaps: string[]
+  context: 'robots.txt' | 'init'
+  event?: H3Event // undefined on `init`
+}
+```
+
+This hook was used for both initialization and robots.txt generation. It has been split into two separate hooks for better clarity:
+- Use `robots:init` when `context === 'init'`
+- Use `robots:robots-txt:input` when `context === 'robots.txt'`
+
 ## `'robots:robots-txt'`{lang="ts"}
 
 **Type:** `(ctx: HookContext) => void | Promise<void>`{lang="ts"}

src/module.ts

@@ -541,7 +541,9 @@ export default defineNuxtModule<ModuleOptions>({
   }
   interface NitroRuntimeHooks {
     'robots:config': (ctx: import('${typesPath}').HookRobotsConfigContext) => void | Promise<void>
+    'robots:init': (ctx: import('${typesPath}').HookRobotsInitContext) => void | Promise<void>
     'robots:robots-txt': (ctx: import('${typesPath}').HookRobotsTxtContext) => void | Promise<void>
+    'robots:robots-txt:input': (ctx: import('${typesPath}').HookRobotsTxtInputContext) => void | Promise<void>
   }`
         return `// Generated by nuxt-robots
 

src/runtime/server/plugins/initContext.ts

@@ -1,10 +1,11 @@
 import type { NitroApp } from 'nitropack/types'
+import type { HookRobotsConfigContext, HookRobotsInitContext } from '../../types'
 import { defineNitroPlugin, getRouteRules } from 'nitropack/runtime'
 import { withoutTrailingSlash } from 'ufo'
-import { createPatternMap } from '../../../util'
+import { createPatternMap, normalizeGroup } from '../../../util'
 import { useRuntimeConfigNuxtRobots } from '../composables/useRuntimeConfigNuxtRobots'
 import { logger } from '../logger'
-import { resolveRobotsTxtContext } from '../util'
+import { normalizeRobotsContext } from '../util'
 
 const PRERENDER_NO_SSR_ROUTES = new Set(['/index.html', '/200.html', '/404.html'])
 
@@ -19,7 +20,35 @@ export default defineNitroPlugin(async (nitroApp: NitroApp) => {
   }
 
   nitroApp._robots = {} as typeof nitroApp._robots
-  await resolveRobotsTxtContext(undefined, nitroApp)
+
+  // Get and normalize base context from runtime config
+  const { groups, sitemap: sitemaps } = useRuntimeConfigNuxtRobots()
+  const baseCtx = normalizeRobotsContext({
+    groups: JSON.parse(JSON.stringify(groups)),
+    sitemaps: JSON.parse(JSON.stringify(sitemaps)),
+  })
+
+  // Call robots:init hook
+  const initCtx: HookRobotsInitContext = {
+    ...baseCtx,
+  }
+  await nitroApp.hooks.callHook('robots:init', initCtx)
+
+  // Backwards compatibility: also call deprecated robots:config hook
+  const deprecatedCtx: HookRobotsConfigContext = {
+    ...initCtx,
+    event: undefined,
+    context: 'init',
+  }
+  await nitroApp.hooks.callHook('robots:config', deprecatedCtx)
+
+  // Sync changes back and re-normalize
+  initCtx.groups = deprecatedCtx.groups.map(normalizeGroup)
+  initCtx.sitemaps = deprecatedCtx.sitemaps
+  initCtx.errors = deprecatedCtx.errors
+
+  // Store in nitro app
+  nitroApp._robots.ctx = { ...initCtx, context: 'init', event: undefined }
   const nuxtContentUrls = new Set<string>()
   if (isNuxtContentV2) {
     let urls: string[] | undefined

src/runtime/server/routes/robots-txt.ts

@@ -1,12 +1,12 @@
-import type { HookRobotsConfigContext, HookRobotsTxtContext } from '../../types'
+import type { HookRobotsConfigContext, HookRobotsTxtContext, HookRobotsTxtInputContext } from '../../types'
 import { logger } from '#robots/server/logger'
 import { withSiteUrl } from '#site-config/server/composables/utils'
 import { defineEventHandler, setHeader } from 'h3'
 import { useNitroApp } from 'nitropack/runtime'
-import { asArray, generateRobotsTxt } from '../../util'
+import { generateRobotsTxt, normalizeGroup } from '../../util'
 import { getSiteRobotConfig } from '../composables/getSiteRobotConfig'
 import { useRuntimeConfigNuxtRobots } from '../composables/useRuntimeConfigNuxtRobots'
-import { resolveRobotsTxtContext } from '../util'
+import { normalizeRobotsContext } from '../util'
 
 export default defineEventHandler(async (e) => {
   const nitroApp = useNitroApp()
@@ -26,13 +26,43 @@ export default defineEventHandler(async (e) => {
     ],
   }
   if (indexable) {
-    robotsTxtCtx = await resolveRobotsTxtContext(e)
-    // normalise
-    robotsTxtCtx.sitemaps = [...new Set(
-      asArray(robotsTxtCtx.sitemaps)
-        // validate sitemaps are absolute
-        .map(s => !s.startsWith('http') ? withSiteUrl(e, s, { withBase: true, absolute: true }) : s),
-    )]
+    // Start from cached init context (includes robots:init modifications)
+    // or get fresh from runtime config and normalize
+    const { groups, sitemap: sitemaps } = useRuntimeConfigNuxtRobots(e)
+    const baseCtx = nitroApp._robots.ctx
+      ? {
+          groups: nitroApp._robots.ctx.groups,
+          sitemaps: JSON.parse(JSON.stringify(nitroApp._robots.ctx.sitemaps)),
+          errors: [],
+        }
+      : normalizeRobotsContext({ groups, sitemaps, errors: [] })
+
+    // Call robots:robots-txt:input hook
+    const inputCtx: HookRobotsTxtInputContext = {
+      ...baseCtx,
+      event: e,
+    }
+    await nitroApp.hooks.callHook('robots:robots-txt:input', inputCtx)
+
+    // Backwards compatibility: also call deprecated robots:config hook
+    const deprecatedCtx: HookRobotsConfigContext = {
+      ...inputCtx,
+      context: 'robots.txt',
+    }
+    await nitroApp.hooks.callHook('robots:config', deprecatedCtx)
+
+    // Sync changes back and re-normalize
+    inputCtx.groups = deprecatedCtx.groups.map(normalizeGroup)
+    inputCtx.sitemaps = deprecatedCtx.sitemaps
+    inputCtx.errors = deprecatedCtx.errors
+
+    // Update nitro._robots.ctx so getPathRobotConfig can access the latest groups
+    nitroApp._robots.ctx = { ...inputCtx, context: 'robots.txt', event: e }
+
+    robotsTxtCtx = inputCtx
+    // Make sitemaps absolute (already normalized and deduplicated in normalizeRobotsContext)
+    robotsTxtCtx.sitemaps = robotsTxtCtx.sitemaps
+      .map(s => !s.startsWith('http') ? withSiteUrl(e, s, { withBase: true, absolute: true }) : s)
     if (isNuxtContentV2) {
       const contentWithRobotRules = await e.$fetch<string[]>('/__robots__/nuxt-content.json', {
         headers: {

src/runtime/server/util.ts

@@ -1,20 +1,22 @@
-import type { H3Event } from 'h3'
-import type { NitroApp } from 'nitropack'
-import type { HookRobotsConfigContext } from '../types'
-import { useNitroApp } from 'nitropack/runtime'
-import { normalizeGroup } from '../../util'
-import { useRuntimeConfigNuxtRobots } from './composables/useRuntimeConfigNuxtRobots'
+import type { ParsedRobotsTxt, RobotsGroupInput } from '../types'
+import { asArray, normalizeGroup } from '../../util'
 
-export async function resolveRobotsTxtContext(e: H3Event | undefined, nitro: NitroApp = useNitroApp()) {
-  const { groups, sitemap: sitemaps } = useRuntimeConfigNuxtRobots(e)
-  // make the config writable
-  const generateRobotsTxtCtx: HookRobotsConfigContext = {
-    event: e,
-    context: e ? 'robots.txt' : 'init',
-    ...JSON.parse(JSON.stringify({ groups, sitemaps })),
+/**
+ * Pure normalization function for robots context
+ * - Groups are normalized with _indexable and _rules
+ * - Sitemaps are converted to array, deduplicated, and filtered for valid strings
+ * - Errors are converted to array and filtered for valid strings
+ *
+ * Note: URL absolutization (withSiteUrl) happens separately in robots-txt.ts since it requires H3Event
+ */
+export function normalizeRobotsContext(input: Partial<ParsedRobotsTxt>): ParsedRobotsTxt {
+  return {
+    groups: asArray(input.groups).map(g => normalizeGroup(g as RobotsGroupInput)),
+    sitemaps: [...new Set(
+      asArray(input.sitemaps)
+        .filter(s => typeof s === 'string' && s.trim().length > 0),
+    )],
+    errors: asArray(input.errors)
+      .filter(e => typeof e === 'string' && e.trim().length > 0),
   }
-  await nitro.hooks.callHook('robots:config', generateRobotsTxtCtx)
-  generateRobotsTxtCtx.groups = generateRobotsTxtCtx.groups.map(normalizeGroup)
-  nitro._robots.ctx = generateRobotsTxtCtx
-  return generateRobotsTxtCtx
 }

src/runtime/types.ts

@@ -92,6 +92,21 @@ export interface HookRobotsConfigContext extends ParsedRobotsTxt {
   context: 'robots.txt' | 'init'
 }
 
+/**
+ * Hook context for robots:init
+ * Called once during Nitro initialization
+ */
+export interface HookRobotsInitContext extends ParsedRobotsTxt {
+}
+
+/**
+ * Hook context for robots:robots-txt:input
+ * Called on each robots.txt request
+ */
+export interface HookRobotsTxtInputContext extends ParsedRobotsTxt {
+  event: H3Event
+}
+
 // Bot Detection Types
 export interface BotDetectionContext {
   isBot: boolean