Skip to content

Commit ce2df2e

Browse files
committed
docs(inference): document probe budget helpers
Fixes NVIDIA#10260 Signed-off-by: Deepak Jain <deepujain@gmail.com>
1 parent 5edea59 commit ce2df2e

3 files changed

Lines changed: 7 additions & 3 deletions

File tree

src/lib/actions/sandbox/inference-invocation-probe.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ export const REBUILD_INFERENCE_INVOCATION_TIMEOUT_MS = 100_000;
4949
export const READINESS_INFERENCE_INVOCATION_TIMEOUT_MS = 30_000;
5050
const INFERENCE_INVOCATION_MAX_RESPONSE_BYTES = 64 * 1024;
5151

52+
/** Build the protocol-specific request used to verify sandbox inference. */
5253
function buildProbeRequest(input: SandboxInferenceInvocationInput): {
5354
endpoint: string;
5455
headers: string[];

src/lib/inference/health.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@ function useStatusProbeTiming(argv: string[]): string[] {
106106
);
107107
}
108108

109+
/** Set the visible-output budget on a generated chat-completions probe. */
109110
function capStatusProbeOutput(
110111
argv: string[],
111112
maxTokens: number = HEALTH_PROBE_MAX_TOKENS,
@@ -123,6 +124,7 @@ function capStatusProbeOutput(
123124
return next;
124125
}
125126

127+
/** Build curl arguments for an authenticated chat-completions health probe. */
126128
function buildChatCompletionsStatusProbeCurlArgs(
127129
model: string,
128130
endpoint: string,

src/lib/inference/max-tokens-field.ts

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,9 +29,10 @@
2929
*/
3030
export const MIN_PROBE_REPLY_TOKENS = 16;
3131

32-
// Gemini 2.5 can spend part of the output budget on reasoning before it emits
33-
// visible content. Match the established compatible-endpoint smoke budget so
34-
// a healthy route has room to return the requested acknowledgement (#10260).
32+
/**
33+
* Gemini probe budget with room for reasoning before visible content.
34+
* Matches the established compatible-endpoint smoke budget (#10260).
35+
*/
3536
export const GEMINI_PROBE_REPLY_TOKENS = 256;
3637

3738
// Matched by prefix rather than exact id: Azure OpenAI deployments append

0 commit comments

Comments
 (0)