diff --git a/.changeset/eager-news-serve.md b/.changeset/eager-news-serve.md new file mode 100644 index 000000000000..979b52bc4db1 --- /dev/null +++ b/.changeset/eager-news-serve.md @@ -0,0 +1,5 @@ +--- +'@sveltejs/kit': minor +--- + +feat: File upload progress is available on `myForm.fields.someFile.progress()` diff --git a/.changeset/new-rivers-run.md b/.changeset/new-rivers-run.md new file mode 100644 index 000000000000..257e21a14651 --- /dev/null +++ b/.changeset/new-rivers-run.md @@ -0,0 +1,5 @@ +--- +'@sveltejs/kit': minor +--- + +feat: File uploads inside `form` remote functions are now streamed - form data can be accessed before large files finish uploading. diff --git a/packages/kit/src/exports/public.d.ts b/packages/kit/src/exports/public.d.ts index 3516689dce67..fb87dbed168a 100644 --- a/packages/kit/src/exports/public.d.ts +++ b/packages/kit/src/exports/public.d.ts @@ -1880,7 +1880,12 @@ type RemoteFormFieldMethods = { set(input: T): T; /** Validation issues, if any */ issues(): RemoteFormIssue[] | undefined; -}; +} & (T extends File + ? { + /** Current upload progress, from 0 to 1 */ + progress(): number; + } + : object); export type RemoteFormFieldValue = string | string[] | number | boolean | File | File[]; diff --git a/packages/kit/src/runtime/app/server/remote/form.js b/packages/kit/src/runtime/app/server/remote/form.js index c1fe5a1d2c10..d6b3f73342a5 100644 --- a/packages/kit/src/runtime/app/server/remote/form.js +++ b/packages/kit/src/runtime/app/server/remote/form.js @@ -4,7 +4,6 @@ import { get_request_store } from '@sveltejs/kit/internal/server'; import { DEV } from 'esm-env'; import { - convert_formdata, create_field_proxy, set_nested_value, throw_on_old_property_access, @@ -104,19 +103,7 @@ export function form(validate_or_fn, maybe_fn) { type: 'form', name: '', id: '', - /** @param {FormData} form_data */ - fn: async (form_data) => { - const validate_only = form_data.get('sveltekit:validate_only') === 'true'; - - let data = maybe_fn ? convert_formdata(form_data) : undefined; - - if (data && data.id === undefined) { - const id = form_data.get('sveltekit:id'); - if (typeof id === 'string') { - data.id = JSON.parse(id); - } - } - + fn: async (data, meta, form_data) => { // TODO 3.0 remove this warning if (DEV && !data) { const error = () => { @@ -152,12 +139,12 @@ export function form(validate_or_fn, maybe_fn) { const { event, state } = get_request_store(); const validated = await schema?.['~standard'].validate(data); - if (validate_only) { + if (meta.validate_only) { return validated?.issues?.map((issue) => normalize_issue(issue, true)) ?? []; } if (validated?.issues !== undefined) { - handle_issues(output, validated.issues, event.isRemoteRequest, form_data); + handle_issues(output, validated.issues, form_data); } else { if (validated !== undefined) { data = validated.value; @@ -178,7 +165,7 @@ export function form(validate_or_fn, maybe_fn) { ); } catch (e) { if (e instanceof ValidationError) { - handle_issues(output, e.issues, event.isRemoteRequest, form_data); + handle_issues(output, e.issues, form_data); } else { throw e; } @@ -226,7 +213,8 @@ export function form(validate_or_fn, maybe_fn) { (get_cache(__)[''] ??= {}).input = input; }, - () => issues + () => issues, + () => 0 /* upload progress is always 0 on the server */ ); } }); @@ -297,15 +285,14 @@ export function form(validate_or_fn, maybe_fn) { /** * @param {{ issues?: InternalRemoteFormIssue[], input?: Record, result: any }} output * @param {readonly StandardSchemaV1.Issue[]} issues - * @param {boolean} is_remote_request - * @param {FormData} form_data + * @param {FormData | null} form_data - null if the form is progressively enhanced */ -function handle_issues(output, issues, is_remote_request, form_data) { +function handle_issues(output, issues, form_data) { output.issues = issues.map((issue) => normalize_issue(issue, true)); // if it was a progressively-enhanced submission, we don't need // to return the input — it's already there - if (!is_remote_request) { + if (form_data) { output.input = {}; for (let key of form_data.keys()) { diff --git a/packages/kit/src/runtime/client/remote-functions/form.svelte.js b/packages/kit/src/runtime/client/remote-functions/form.svelte.js index 02bbf9239720..2a90678bec77 100644 --- a/packages/kit/src/runtime/client/remote-functions/form.svelte.js +++ b/packages/kit/src/runtime/client/remote-functions/form.svelte.js @@ -18,7 +18,11 @@ import { set_nested_value, throw_on_old_property_access, build_path_string, - normalize_issue + normalize_issue, + serialize_binary_form, + BINARY_FORM_CONTENT_TYPE, + deep_get, + get_file_paths } from '../../form-utils.js'; /** @@ -55,6 +59,7 @@ export function form(id) { /** @param {string | number | boolean} [key] */ function create_instance(key) { + const action_id_without_key = id; const action_id = id + (key != undefined ? `/${JSON.stringify(key)}` : ''); const action = '?/remote=' + encodeURIComponent(action_id); @@ -63,6 +68,11 @@ export function form(id) { */ let input = $state({}); + /** + * @type {Record} + */ + let upload_progress = $state({}); + /** @type {InternalRemoteFormIssue[]} */ let raw_issues = $state.raw([]); @@ -157,10 +167,10 @@ export function form(id) { } /** - * @param {FormData} data + * @param {FormData} form_data * @returns {Promise & { updates: (...args: any[]) => any }} */ - function submit(data) { + function submit(form_data) { // Store a reference to the current instance and increment the usage count for the duration // of the request. This ensures that the instance is not deleted in case of an optimistic update // (e.g. when deleting an item in a list) that fails and wants to surface an error to the user afterwards. @@ -182,26 +192,55 @@ export function form(id) { try { await Promise.resolve(); - if (updates.length > 0) { - data.set('sveltekit:remote_refreshes', JSON.stringify(updates.map((u) => u._key))); - } + const data = convert(form_data); - const response = await fetch(`${base}/${app_dir}/remote/${action_id}`, { - method: 'POST', - body: data, - headers: { - 'x-sveltekit-pathname': location.pathname, - 'x-sveltekit-search': location.search - } + const { blob, file_offsets } = serialize_binary_form(data, { + remote_refreshes: updates.map((u) => u._key) }); - if (!response.ok) { - // We only end up here in case of a network error or if the server has an internal error - // (which shouldn't happen because we handle errors on the server and always send a 200 response) - throw new Error('Failed to execute remote function'); - } + /** @type {string} */ + const response_text = await new Promise((resolve, reject) => { + const xhr = new XMLHttpRequest(); + xhr.addEventListener('readystatechange', () => { + switch (xhr.readyState) { + case 2 /* HEADERS_RECEIVED */: + if (xhr.status !== 200) { + // We only end up here if the server has an internal error + // (which shouldn't happen because we handle errors on the server and always send a 200 response) + reject(new Error('Failed to execute remote function')); + } + break; + case 4 /* DONE */: + if (xhr.status !== 200) { + reject(new Error('Failed to execute remote function')); + break; + } + resolve(xhr.responseText); + break; + } + }); + if (file_offsets) { + const file_paths = get_file_paths(data); + xhr.upload.addEventListener('progress', (ev) => { + for (const file of file_offsets) { + let progress = (ev.loaded - file.start) / file.file.size; + if (progress <= 0) continue; + if (progress > 1) progress = 1; + const path = file_paths.get(file.file); + if (!path) continue; + deep_set(upload_progress, path, progress); + } + }); + } + // Use `action_id_without_key` here because the id is included in the body via `convert(data)` + xhr.open('POST', `${base}/${app_dir}/remote/${action_id_without_key}`); + xhr.setRequestHeader('Content-Type', BINARY_FORM_CONTENT_TYPE); + xhr.setRequestHeader('x-sveltekit-pathname', location.pathname); + xhr.setRequestHeader('x-sveltekit-search', location.search); + xhr.send(blob); + }); - const form_result = /** @type { RemoteFunctionResponse} */ (await response.json()); + const form_result = /** @type { RemoteFunctionResponse} */ (JSON.parse(response_text)); if (form_result.type === 'result') { ({ issues: raw_issues = [], result } = devalue.parse(form_result.result, app.decoders)); @@ -370,6 +409,7 @@ export function form(id) { if (file) { set_nested_value(input, name, file); + set_nested_value(upload_progress, name, 0); } else { // Remove the property by setting to undefined and clean up const path_parts = name.split(/\.|\[|\]/).filter(Boolean); @@ -399,6 +439,7 @@ export function form(id) { await tick(); input = convert_formdata(new FormData(form)); + upload_progress = {}; }); return () => { @@ -501,7 +542,8 @@ export function form(id) { touched[key] = true; } }, - () => issues + () => issues, + (path) => deep_get(upload_progress, path) ?? 0 ) }, result: { @@ -532,7 +574,9 @@ export function form(id) { /** @type {InternalRemoteFormIssue[]} */ let array = []; - const validated = await preflight_schema?.['~standard'].validate(convert(form_data)); + const data = convert(form_data); + + const validated = await preflight_schema?.['~standard'].validate(data); if (validate_id !== id) { return; @@ -541,11 +585,16 @@ export function form(id) { if (validated?.issues) { array = validated.issues.map((issue) => normalize_issue(issue, false)); } else if (!preflightOnly) { - form_data.set('sveltekit:validate_only', 'true'); - - const response = await fetch(`${base}/${app_dir}/remote/${action_id}`, { + const response = await fetch(`${base}/${app_dir}/remote/${action_id_without_key}`, { method: 'POST', - body: form_data + headers: { + 'Content-Type': BINARY_FORM_CONTENT_TYPE, + 'x-sveltekit-pathname': location.pathname, + 'x-sveltekit-search': location.search + }, + body: serialize_binary_form(data, { + validate_only: true + }).blob }); const result = await response.json(); @@ -637,12 +686,6 @@ function clone(element) { */ function validate_form_data(form_data, enctype) { for (const key of form_data.keys()) { - if (key.startsWith('sveltekit:')) { - throw new Error( - 'FormData keys starting with `sveltekit:` are reserved for internal use and should not be set manually' - ); - } - if (/^\$[.[]?/.test(key)) { throw new Error( '`$` is used to collect all FormData validation issues and cannot be used as the `name` of a form control' diff --git a/packages/kit/src/runtime/form-utils.js b/packages/kit/src/runtime/form-utils.js index c879fa2be5e2..fd5985b88daa 100644 --- a/packages/kit/src/runtime/form-utils.js +++ b/packages/kit/src/runtime/form-utils.js @@ -1,8 +1,10 @@ /** @import { RemoteForm } from '@sveltejs/kit' */ -/** @import { InternalRemoteFormIssue } from 'types' */ +/** @import { BinaryFormMeta, InternalRemoteFormIssue } from 'types' */ /** @import { StandardSchemaV1 } from '@standard-schema/spec' */ import { DEV } from 'esm-env'; +import * as devalue from 'devalue'; +import { text_decoder } from './utils.js'; /** * Sets a value in a nested object using a path string, mutating the original object @@ -31,10 +33,6 @@ export function convert_formdata(data) { const result = {}; for (let key of data.keys()) { - if (key.startsWith('sveltekit:')) { - continue; - } - const is_array = key.endsWith('[]'); /** @type {any[]} */ let values = data.getAll(key); @@ -64,6 +62,348 @@ export function convert_formdata(data) { return result; } +export const BINARY_FORM_CONTENT_TYPE = 'application/x-sveltekit-formdata'; +const BINARY_FORM_VERSION = 0; + +/** + * The binary format is as follows: + * - 1 byte: Format version + * - 4 bytes: Length of the header (u32) + * - 2 bytes: Length of the file offset table (u16) + * - header: devalue.stringify([data, meta]) + * - file offset table: JSON.stringify([offset1, offset2, ...]) (empty if no files) (offsets start from the end of the table) + * - file1, file2, ... + * @param {Record} data + * @param {BinaryFormMeta} meta + */ +export function serialize_binary_form(data, meta) { + /** @type {Array} */ + const blob_parts = [new Uint8Array([BINARY_FORM_VERSION])]; + + /** @type {Array<[file: File, index: number]>} */ + const files = []; + + if (!meta.remote_refreshes?.length) { + delete meta.remote_refreshes; + } + + const encoded_header = devalue.stringify([data, meta], { + File: (file) => { + if (!(file instanceof File)) return; + + files.push([file, files.length]); + return [file.name, file.type, file.size, file.lastModified, files.length - 1]; + } + }); + + let encoded_file_offsets = ''; + /** @type {Array | undefined} */ + let file_offsets; + if (files.length) { + // Sort small files to the front + files.sort(([a], [b]) => a.size - b.size); + + file_offsets = new Array(files.length); + let start = 0; + for (const [file, index] of files) { + file_offsets[index] = start; + start += file.size; + } + encoded_file_offsets = JSON.stringify(file_offsets); + } + + const length_buffer = new Uint8Array(4); + const length_view = new DataView(length_buffer.buffer); + + length_view.setUint32(0, encoded_header.length, true); + blob_parts.push(length_buffer.slice()); + + length_view.setUint16(0, encoded_file_offsets.length, true); + blob_parts.push(length_buffer.slice(0, 2)); + + blob_parts.push(encoded_header); + blob_parts.push(encoded_file_offsets); + + for (const [file] of files) { + blob_parts.push(file); + } + + const file_offset_start = 1 + 4 + 2 + encoded_header.length + encoded_file_offsets.length; + return { + blob: new Blob(blob_parts), + file_offsets: file_offsets?.map((o, i) => ({ + start: o + file_offset_start, + file: files[i][0] + })) + }; +} + +/** + * @param {Request} request + * @returns {Promise<{ data: Record; meta: BinaryFormMeta; form_data: FormData | null }>} + */ +export async function deserialize_binary_form(request) { + if (request.headers.get('content-type') !== BINARY_FORM_CONTENT_TYPE) { + const form_data = await request.formData(); + return { data: convert_formdata(form_data), meta: {}, form_data }; + } + if (!request.body) { + throw new Error('Could not deserialize binary form: no body'); + } + + const reader = request.body.getReader(); + + /** @type {Array | undefined>>} */ + const chunks = []; + + /** + * @param {number} index + * @returns {Promise | undefined>} + */ + async function get_chunk(index) { + if (index in chunks) return chunks[index]; + + let i = chunks.length; + while (i <= index) { + chunks[i] = reader.read().then((chunk) => chunk.value); + i++; + } + return chunks[index]; + } + + /** + * @param {number} offset + * @param {number} length + * @returns {Promise} + */ + async function get_buffer(offset, length) { + /** @type {Uint8Array} */ + let start_chunk; + let chunk_start = 0; + /** @type {number} */ + let chunk_index; + for (chunk_index = 0; ; chunk_index++) { + const chunk = await get_chunk(chunk_index); + if (!chunk) return null; + + const chunk_end = chunk_start + chunk.byteLength; + // If this chunk contains the target offset + if (offset >= chunk_start && offset < chunk_end) { + start_chunk = chunk; + break; + } + chunk_start = chunk_end; + } + // If the buffer is completely contained in one chunk, do a subarray + if (offset + length <= chunk_start + start_chunk.byteLength) { + return start_chunk.subarray(offset - chunk_start, offset + length - chunk_start); + } + // Otherwise, copy the data into a new buffer + const buffer = new Uint8Array(length); + buffer.set(start_chunk.subarray(offset - chunk_start)); + let cursor = start_chunk.byteLength - offset + chunk_start; + while (cursor < length) { + chunk_index++; + let chunk = await get_chunk(chunk_index); + if (!chunk) return null; + if (chunk.byteLength > length - cursor) { + chunk = chunk.subarray(0, length - cursor); + } + buffer.set(chunk, cursor); + cursor += chunk.byteLength; + } + + return buffer; + } + + const header = await get_buffer(0, 1 + 4 + 2); + if (!header) throw new Error('Could not deserialize binary form: too short'); + + if (header[0] !== BINARY_FORM_VERSION) { + throw new Error( + `Could not deserialize binary form: got version ${header[0]}, expected version ${BINARY_FORM_VERSION}` + ); + } + const header_view = new DataView(header.buffer); + const data_length = header_view.getUint32(1, true); + const file_offsets_length = header_view.getUint16(5, true); + + // Read the form data + const data_buffer = await get_buffer(1 + 4 + 2, data_length); + if (!data_buffer) throw new Error('Could not deserialize binary form: data too short'); + + /** @type {Array} */ + let file_offsets; + /** @type {number} */ + let files_start_offset; + if (file_offsets_length > 0) { + // Read the file offset table + const file_offsets_buffer = await get_buffer(1 + 4 + 2 + data_length, file_offsets_length); + if (!file_offsets_buffer) + throw new Error('Could not deserialize binary form: file offset table too short'); + + file_offsets = /** @type {Array} */ ( + JSON.parse(text_decoder.decode(file_offsets_buffer)) + ); + files_start_offset = 1 + 4 + 2 + data_length + file_offsets_length; + } + + const [data, meta] = devalue.parse(text_decoder.decode(data_buffer), { + File: ([name, type, size, last_modified, index]) => { + return new Proxy( + new LazyFile( + name, + type, + size, + last_modified, + get_chunk, + files_start_offset + file_offsets[index] + ), + { + getPrototypeOf() { + // Trick validators into thinking this is a normal File + return File.prototype; + } + } + ); + } + }); + + // Read the request body asyncronously so it doesn't stall + void (async () => { + let has_more = true; + while (has_more) { + const chunk = await get_chunk(chunks.length); + has_more = !!chunk; + } + })(); + + return { data, meta, form_data: null }; +} + +/** @implements {File} */ +class LazyFile { + /** @type {(index: number) => Promise | undefined>} */ + #get_chunk; + /** @type {number} */ + #offset; + /** + * @param {string} name + * @param {string} type + * @param {number} size + * @param {number} last_modified + * @param {(index: number) => Promise | undefined>} get_chunk + * @param {number} offset + */ + constructor(name, type, size, last_modified, get_chunk, offset) { + this.name = name; + this.type = type; + this.size = size; + this.lastModified = last_modified; + this.webkitRelativePath = ''; + this.#get_chunk = get_chunk; + this.#offset = offset; + + // TODO - hacky, required for private members to be accessed on proxy + this.arrayBuffer = this.arrayBuffer.bind(this); + this.bytes = this.bytes.bind(this); + this.slice = this.slice.bind(this); + this.stream = this.stream.bind(this); + this.text = this.text.bind(this); + } + /** @type {ArrayBuffer | undefined} */ + #buffer; + async arrayBuffer() { + this.#buffer ??= await new Response(this.stream()).arrayBuffer(); + return this.#buffer; + } + async bytes() { + return new Uint8Array(await this.arrayBuffer()); + } + /** + * @param {number=} start + * @param {number=} end + * @param {string=} contentType + */ + slice(start = 0, end = this.size, contentType = this.type) { + // https://github.com/nodejs/node/blob/a5f3cd8cb5ba9e7911d93c5fd3ebc6d781220dd8/lib/internal/blob.js#L240 + if (start < 0) { + start = Math.max(this.size + start, 0); + } else { + start = Math.min(start, this.size); + } + + if (end < 0) { + end = Math.max(this.size + end, 0); + } else { + end = Math.min(end, this.size); + } + const size = Math.max(end - start, 0); + const file = new LazyFile( + this.name, + contentType, + size, + this.lastModified, + this.#get_chunk, + this.#offset + start + ); + + return file; + } + stream() { + let cursor = 0; + let chunk_index = 0; + return new ReadableStream({ + start: async (controller) => { + let chunk_start = 0; + let start_chunk = null; + for (chunk_index = 0; ; chunk_index++) { + const chunk = await this.#get_chunk(chunk_index); + if (!chunk) return null; + + const chunk_end = chunk_start + chunk.byteLength; + // If this chunk contains the target offset + if (this.#offset >= chunk_start && this.#offset < chunk_end) { + start_chunk = chunk; + break; + } + chunk_start = chunk_end; + } + // If the buffer is completely contained in one chunk, do a subarray + if (this.#offset + this.size <= chunk_start + start_chunk.byteLength) { + controller.enqueue( + start_chunk.subarray(this.#offset - chunk_start, this.#offset + this.size - chunk_start) + ); + controller.close(); + } else { + controller.enqueue(start_chunk.subarray(this.#offset - chunk_start)); + cursor = start_chunk.byteLength - this.#offset + chunk_start; + } + }, + pull: async (controller) => { + chunk_index++; + let chunk = await this.#get_chunk(chunk_index); + if (!chunk) { + controller.error('Could not deserialize binary form: incomplete file data'); + controller.close(); + return; + } + if (chunk.byteLength > this.size - cursor) { + chunk = chunk.subarray(0, this.size - cursor); + } + controller.enqueue(chunk); + cursor += chunk.byteLength; + if (cursor >= this.size) { + controller.close(); + } + } + }); + } + async text() { + return text_decoder.decode(await this.arrayBuffer()); + } +} + const path_regex = /^[a-zA-Z_$]\w*(\.[a-zA-Z_$]\w*|\[\d+\])*$/; /** @@ -90,6 +430,27 @@ function check_prototype_pollution(key) { } } +/** + * Finds the paths to every File in an object + * @param {unknown} object + * @param {Map} paths + * @param {string[]} path + */ +export function get_file_paths(object, paths = new Map(), path = []) { + if (Array.isArray(object)) { + for (let i = 0; i < object.length; i++) { + get_file_paths(object[i], paths, [...path, i.toString()]); + } + } else if (object instanceof File) { + paths.set(object, path); + } else if (typeof object === 'object' && object !== null) { + for (const [key, value] of Object.entries(object)) { + get_file_paths(value, paths, [...path, key]); + } + } + return paths; +} + /** * Sets a value in a nested object using an array of keys, mutating the original object. * @param {Record} object @@ -207,10 +568,18 @@ export function deep_get(object, path) { * @param {() => Record} get_input - Function to get current input data * @param {(path: (string | number)[], value: any) => void} set_input - Function to set input data * @param {() => Record} get_issues - Function to get current issues + * @param {(path: (string | number)[]) => number} get_progress - Function to get upload progress of a file * @param {(string | number)[]} path - Current access path * @returns {any} Proxy object with name(), value(), and issues() methods */ -export function create_field_proxy(target, get_input, set_input, get_issues, path = []) { +export function create_field_proxy( + target, + get_input, + set_input, + get_issues, + get_progress, + path = [] +) { const get_value = () => { return deep_get(get_input(), path); }; @@ -221,7 +590,7 @@ export function create_field_proxy(target, get_input, set_input, get_issues, pat // Handle array access like jobs[0] if (/^\d+$/.test(prop)) { - return create_field_proxy({}, get_input, set_input, get_issues, [ + return create_field_proxy({}, get_input, set_input, get_issues, get_progress, [ ...path, parseInt(prop, 10) ]); @@ -234,11 +603,17 @@ export function create_field_proxy(target, get_input, set_input, get_issues, pat set_input(path, newValue); return newValue; }; - return create_field_proxy(set_func, get_input, set_input, get_issues, [...path, prop]); + return create_field_proxy(set_func, get_input, set_input, get_issues, get_progress, [ + ...path, + prop + ]); } if (prop === 'value') { - return create_field_proxy(get_value, get_input, set_input, get_issues, [...path, prop]); + return create_field_proxy(get_value, get_input, set_input, get_issues, get_progress, [ + ...path, + prop + ]); } if (prop === 'issues' || prop === 'allIssues') { @@ -260,7 +635,19 @@ export function create_field_proxy(target, get_input, set_input, get_issues, pat })); }; - return create_field_proxy(issues_func, get_input, set_input, get_issues, [...path, prop]); + return create_field_proxy(issues_func, get_input, set_input, get_issues, get_progress, [ + ...path, + prop + ]); + } + + if (prop === 'progress') { + const progress_func = () => get_progress(path); + + return create_field_proxy(progress_func, get_input, set_input, get_issues, get_progress, [ + ...path, + prop + ]); } if (prop === 'as') { @@ -409,11 +796,17 @@ export function create_field_proxy(target, get_input, set_input, get_issues, pat }); }; - return create_field_proxy(as_func, get_input, set_input, get_issues, [...path, 'as']); + return create_field_proxy(as_func, get_input, set_input, get_issues, get_progress, [ + ...path, + 'as' + ]); } // Handle property access (nested fields) - return create_field_proxy({}, get_input, set_input, get_issues, [...path, prop]); + return create_field_proxy({}, get_input, set_input, get_issues, get_progress, [ + ...path, + prop + ]); } }); } diff --git a/packages/kit/src/runtime/form-utils.spec.js b/packages/kit/src/runtime/form-utils.spec.js index 77b7e355f4a8..88409435cbe0 100644 --- a/packages/kit/src/runtime/form-utils.spec.js +++ b/packages/kit/src/runtime/form-utils.spec.js @@ -1,5 +1,13 @@ -import { describe, expect, test } from 'vitest'; -import { convert_formdata, split_path } from './form-utils.js'; +import { beforeAll, describe, expect, test } from 'vitest'; +import { + BINARY_FORM_CONTENT_TYPE, + convert_formdata, + deserialize_binary_form, + serialize_binary_form, + split_path +} from './form-utils.js'; +import buffer from 'node:buffer'; +import { text_encoder } from './utils.js'; describe('split_path', () => { const good = [ @@ -90,3 +98,119 @@ describe('convert_formdata', () => { }); } }); + +describe('binary form serializer', () => { + beforeAll(() => { + // TODO: remove after dropping support for Node 18 + if (!('File' in globalThis)) { + // @ts-ignore + globalThis.File = buffer.File; + } + }); + test.each([ + { + data: {}, + meta: {} + }, + { + data: { foo: 'foo', nested: { prop: 'prop' } }, + meta: { pathname: '/foo', validate_only: true } + } + ])('simple', async (input) => { + const { blob } = serialize_binary_form(input.data, input.meta); + const res = await deserialize_binary_form( + new Request('http://test', { + method: 'POST', + body: blob, + headers: { + 'Content-Type': BINARY_FORM_CONTENT_TYPE + } + }) + ); + expect(res.form_data).toBeNull(); + expect(res.data).toEqual(input.data); + expect(res.meta).toEqual(input.meta ?? {}); + }); + test('file uploads', async () => { + const { blob } = serialize_binary_form( + { + small: new File(['a'], 'a.txt', { type: 'text/plain' }), + large: new File([new Uint8Array(1024).fill('a'.charCodeAt(0))], 'large.txt', { + type: 'text/plain', + lastModified: 100 + }) + }, + {} + ); + // Split the stream into 1 byte chunks to make sure all the chunking deserialization works + const stream = blob.stream().pipeThrough( + new TransformStream({ + transform(chunk, controller) { + for (const byte of chunk) { + controller.enqueue(new Uint8Array([byte])); + } + } + }) + ); + const res = await deserialize_binary_form( + new Request('http://test', { + method: 'POST', + body: stream, + // @ts-expect-error duplex required in node + duplex: 'half', + headers: { + 'Content-Type': BINARY_FORM_CONTENT_TYPE + } + }) + ); + const { small, large } = res.data; + expect(small.name).toBe('a.txt'); + expect(small.type).toBe('text/plain'); + expect(small.size).toBe(1); + expect(await small.text()).toBe('a'); + + expect(large.name).toBe('large.txt'); + expect(large.type).toBe('text/plain'); + expect(large.size).toBe(1024); + expect(large.lastModified).toBe(100); + const buffer = new Uint8Array(large.size); + let cursor = 0; + for await (const chunk of large.stream()) { + buffer.set(chunk, cursor); + cursor += chunk.byteLength; + } + expect(buffer).toEqual(new Uint8Array(1024).fill('a'.charCodeAt(0))); + // text should be callable after stream is consumed + expect(await large.text()).toBe('a'.repeat(1024)); + }); + test('LazyFile methods', async () => { + const { blob } = serialize_binary_form( + { + file: new File(['Hello World'], 'a.txt') + }, + {} + ); + const res = await deserialize_binary_form( + new Request('http://test', { + method: 'POST', + body: blob, + headers: { + 'Content-Type': BINARY_FORM_CONTENT_TYPE + } + }) + ); + /** @type {File} */ + const file = res.data.file; + const expected = text_encoder.encode('Hello World'); + expect(await file.text()).toBe('Hello World'); + expect(await file.arrayBuffer()).toEqual(expected.buffer); + expect(await file.bytes()).toEqual(expected); + expect(await new Response(file.stream()).arrayBuffer()).toEqual(expected.buffer); + const ello_slice = file.slice(1, 5, 'test/content-type'); + expect(ello_slice.type).toBe('test/content-type'); + expect(await ello_slice.text()).toBe('ello'); + const world_slice = file.slice(-5); + expect(await world_slice.text()).toBe('World'); + expect(world_slice.type).toBe(file.type); + }); +}); diff --git a/packages/kit/src/runtime/server/remote.js b/packages/kit/src/runtime/server/remote.js index 88ae0d41387c..d52b4be8d87b 100644 --- a/packages/kit/src/runtime/server/remote.js +++ b/packages/kit/src/runtime/server/remote.js @@ -12,6 +12,7 @@ import { normalize_error } from '../../utils/error.js'; import { check_incorrect_fail_use } from './page/actions.js'; import { DEV } from 'esm-env'; import { record_span } from '../telemetry/record_span.js'; +import { deserialize_binary_form } from '../form-utils.js'; /** @type {typeof handle_remote_call_internal} */ export async function handle_remote_call(event, state, options, manifest, id) { @@ -116,25 +117,22 @@ async function handle_remote_call_internal(event, state, options, manifest, id) ); } - const form_data = await event.request.formData(); - form_client_refreshes = /** @type {string[]} */ ( - JSON.parse(/** @type {string} */ (form_data.get('sveltekit:remote_refreshes')) ?? '[]') - ); - form_data.delete('sveltekit:remote_refreshes'); + const { data, meta, form_data } = await deserialize_binary_form(event.request); // If this is a keyed form instance (created via form.for(key)), add the key to the form data (unless already set) - if (additional_args) { - form_data.set('sveltekit:id', decodeURIComponent(additional_args)); + // Note that additional_args will only be set if the form is not enhanced, as enhanced forms transfer the key inside `data`. + if (additional_args && !('id' in data)) { + data.id = JSON.parse(decodeURIComponent(additional_args)); } const fn = info.fn; - const data = await with_request_store({ event, state }, () => fn(form_data)); + const result = await with_request_store({ event, state }, () => fn(data, meta, form_data)); return json( /** @type {RemoteFunctionResponse} */ ({ type: 'result', - result: stringify(data, transport), - refreshes: data.issues ? {} : await serialize_refreshes(form_client_refreshes) + result: stringify(result, transport), + refreshes: result.issues ? undefined : await serialize_refreshes(meta.remote_refreshes) }) ); } @@ -178,7 +176,7 @@ async function handle_remote_call_internal(event, state, options, manifest, id) /** @type {RemoteFunctionResponse} */ ({ type: 'redirect', location: error.location, - refreshes: await serialize_refreshes(form_client_refreshes ?? []) + refreshes: await serialize_refreshes(form_client_refreshes) }) ); } @@ -204,24 +202,26 @@ async function handle_remote_call_internal(event, state, options, manifest, id) } /** - * @param {string[]} client_refreshes + * @param {string[]=} client_refreshes */ async function serialize_refreshes(client_refreshes) { const refreshes = state.refreshes ?? {}; - for (const key of client_refreshes) { - if (refreshes[key] !== undefined) continue; + if (client_refreshes) { + for (const key of client_refreshes) { + if (refreshes[key] !== undefined) continue; - const [hash, name, payload] = key.split('/'); + const [hash, name, payload] = key.split('/'); - const loader = manifest._.remotes[hash]; - const fn = (await loader?.())?.default?.[name]; + const loader = manifest._.remotes[hash]; + const fn = (await loader?.())?.default?.[name]; - if (!fn) error(400, 'Bad Request'); + if (!fn) error(400, 'Bad Request'); - refreshes[key] = with_request_store({ event, state }, () => - fn(parse_remote_arg(payload, transport)) - ); + refreshes[key] = with_request_store({ event, state }, () => + fn(parse_remote_arg(payload, transport)) + ); + } } if (Object.keys(refreshes).length === 0) { @@ -291,16 +291,14 @@ async function handle_remote_form_post_internal(event, state, manifest, id) { } try { - const form_data = await event.request.formData(); const fn = /** @type {RemoteInfo & { type: 'form' }} */ (/** @type {any} */ (form).__).fn; - // If this is a keyed form instance (created via form.for(key)), add the key to the form data (unless already set) - if (action_id && !form_data.has('id')) { - // The action_id is URL-encoded JSON, decode and parse it - form_data.set('sveltekit:id', decodeURIComponent(action_id)); + const { data, meta, form_data } = await deserialize_binary_form(event.request); + if (action_id && !('id' in data)) { + data.id = JSON.parse(decodeURIComponent(action_id)); } - await with_request_store({ event, state }, () => fn(form_data)); + await with_request_store({ event, state }, () => fn(data, meta, form_data)); // We don't want the data to appear on `let { form } = $props()`, which is why we're not returning it. // It is instead available on `myForm.result`, setting of which happens within the remote `form` function. diff --git a/packages/kit/src/types/internal.d.ts b/packages/kit/src/types/internal.d.ts index 84242df90417..6384201af551 100644 --- a/packages/kit/src/types/internal.d.ts +++ b/packages/kit/src/types/internal.d.ts @@ -552,6 +552,11 @@ export type ValidatedKitConfig = Omit, 'adapter'> & adapter?: Adapter; }; +export type BinaryFormMeta = { + remote_refreshes?: string[]; + validate_only?: boolean; +}; + export type RemoteInfo = | { type: 'query' | 'command'; @@ -572,7 +577,11 @@ export type RemoteInfo = type: 'form'; id: string; name: string; - fn: (data: FormData) => Promise; + fn: ( + body: Record, + meta: BinaryFormMeta, + form_data: FormData | null + ) => Promise; } | { type: 'prerender'; diff --git a/packages/kit/src/utils/http.js b/packages/kit/src/utils/http.js index bfd948d4cbf6..f9cb33c652ed 100644 --- a/packages/kit/src/utils/http.js +++ b/packages/kit/src/utils/http.js @@ -1,3 +1,5 @@ +import { BINARY_FORM_CONTENT_TYPE } from '../runtime/form-utils.js'; + /** * Given an Accept header and a list of possible content types, pick * the most suitable one to respond with @@ -74,6 +76,7 @@ export function is_form_content_type(request) { request, 'application/x-www-form-urlencoded', 'multipart/form-data', - 'text/plain' + 'text/plain', + BINARY_FORM_CONTENT_TYPE ); } diff --git a/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/+page.svelte b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/+page.svelte new file mode 100644 index 000000000000..db601b1fc2b5 --- /dev/null +++ b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/+page.svelte @@ -0,0 +1,22 @@ + + +
+ +

File 1: (progress: {upload.fields.file1.progress()})

+ +

File 2: (progress: {upload.fields.deep.files[0].progress()})

+ +

File 3: (progress: {upload.fields.deep.files[1].progress()})

+ + +
+
+ +
+ +
{JSON.stringify(upload.result)}
diff --git a/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts new file mode 100644 index 000000000000..9c36d1a75a77 --- /dev/null +++ b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts @@ -0,0 +1,27 @@ +import { form } from '$app/server'; +import * as v from 'valibot'; + +export const upload = form( + v.object({ + text: v.string(), + file1: v.file(), + deep: v.object({ + files: v.array(v.file()) + }), + read_files: v.optional(v.boolean()) + }), + async (data) => { + if (!data.read_files) { + return { + text: data.text, + file1: data.file1.size, + files: data.deep.files.map((f) => f.size) + }; + } + return { + text: data.text, + file1: await data.file1.text(), + files: await Promise.all(data.deep.files.map((f) => f.text())) + }; + } +); diff --git a/packages/kit/test/apps/basics/test/test.js b/packages/kit/test/apps/basics/test/test.js index 2518529e27d5..4d2582255437 100644 --- a/packages/kit/test/apps/basics/test/test.js +++ b/packages/kit/test/apps/basics/test/test.js @@ -2025,6 +2025,102 @@ test.describe('remote functions', () => { await page.fill('input', 'hello'); await expect(page.locator('select')).toHaveValue('one'); }); + test('file uploads work', async ({ page }) => { + await page.goto('/remote/form/file-upload'); + + await page.locator('input[name="file1"]').setInputFiles({ + name: 'a.txt', + mimeType: 'text/plain', + buffer: Buffer.from('a') + }); + await page.locator('input[name="deep.files[0]"]').setInputFiles({ + name: 'b.txt', + mimeType: 'text/plain', + buffer: Buffer.from('b') + }); + await page.locator('input[name="deep.files[1]"]').setInputFiles({ + name: 'c.txt', + mimeType: 'text/plain', + buffer: Buffer.from('c') + }); + await page.locator('input[type="checkbox"]').check(); + await page.locator('button').click(); + + await expect(page.locator('pre')).toHaveText( + JSON.stringify({ + text: 'Hello world', + file1: 'a', + files: ['b', 'c'] + }) + ); + }); + test('large file uploads work', async ({ page }) => { + await page.goto('/remote/form/file-upload'); + + await page.locator('input[name="file1"]').setInputFiles({ + name: 'a.txt', + mimeType: 'text/plain', + buffer: Buffer.alloc(1024 * 1024 * 10) + }); + await page.locator('input[name="deep.files[0]"]').setInputFiles({ + name: 'b.txt', + mimeType: 'text/plain', + buffer: Buffer.from('b') + }); + await page.locator('input[name="deep.files[1]"]').setInputFiles({ + name: 'c.txt', + mimeType: 'text/plain', + buffer: Buffer.from('c') + }); + await page.locator('button').click(); + + await expect(page.locator('pre')).toHaveText( + JSON.stringify({ + text: 'Hello world', + file1: 1024 * 1024 * 10, + files: [1, 1] + }) + ); + }); + test('file upload progress works', async ({ page, context, javaScriptEnabled }) => { + if (!javaScriptEnabled) return; + await page.goto('/remote/form/file-upload'); + const cdp = await context.newCDPSession(page); + await cdp.send('Network.emulateNetworkConditions', { + offline: false, + latency: 0, + downloadThroughput: -1, + uploadThroughput: 1024 * 1024 * 5 // throttle so it'll take 2 seconds to upload + }); + try { + const progress = page.locator('#progress1'); + expect(progress).toHaveText('0'); + await page.locator('input[name="file1"]').setInputFiles({ + name: 'a.txt', + mimeType: 'text/plain', + buffer: Buffer.alloc(1024 * 1024 * 10) + }); + await page.locator('input[name="deep.files[0]"]').setInputFiles({ + name: 'b.txt', + mimeType: 'text/plain', + buffer: Buffer.from('b') + }); + await page.locator('input[name="deep.files[1]"]').setInputFiles({ + name: 'c.txt', + mimeType: 'text/plain', + buffer: Buffer.from('c') + }); + await page.locator('button').click(); + await expect(progress).not.toHaveText('0'); + } finally { + await cdp.send('Network.emulateNetworkConditions', { + offline: false, + latency: 0, + downloadThroughput: -1, + uploadThroughput: -1 + }); + } + }); }); test.describe('params prop', () => { diff --git a/packages/kit/types/index.d.ts b/packages/kit/types/index.d.ts index 0ddf7ca08844..39737a20ada9 100644 --- a/packages/kit/types/index.d.ts +++ b/packages/kit/types/index.d.ts @@ -1856,7 +1856,12 @@ declare module '@sveltejs/kit' { set(input: T): T; /** Validation issues, if any */ issues(): RemoteFormIssue[] | undefined; - }; + } & (T extends File + ? { + /** Current upload progress, from 0 to 1 */ + progress(): number; + } + : object); export type RemoteFormFieldValue = string | string[] | number | boolean | File | File[];