diff --git a/src/extract/parser.ts b/src/extract/parser.ts index 6eacd51..5c8c7cf 100644 --- a/src/extract/parser.ts +++ b/src/extract/parser.ts @@ -14,8 +14,10 @@ type MsgInfoWithCharIdx = BaseMsg & { idx: number }; export function parseFunctionCall(mapping: KeywordMapping, tokens: Token[]): MsgInfoWithCharIdx[] { let idx = -1; let t: Token | undefined = undefined; + let previousTokenKind: TokenKind | undefined = undefined; function advance() { + previousTokenKind = tokens[idx]?.kind || undefined; idx += 1; return tokens[idx]; } @@ -48,14 +50,19 @@ export function parseFunctionCall(mapping: KeywordMapping, tokens: Token[]): Msg break; } assertIsDefined(t.value); - stringArgs.push(t.value); + if (previousTokenKind === TokenKind.Plus) { + stringArgs[stringArgs.length - 1] += t.value; + } else { + stringArgs.push(t.value); + } t = advance(); if (!t) { break; } - if (t.kind !== TokenKind.Comma) { + if (t.kind !== TokenKind.Comma && t.kind !== TokenKind.Plus) { break; } + t = advance(); if (!t) { break; diff --git a/src/extract/tokenizer.ts b/src/extract/tokenizer.ts index 3cc0bf6..9f2c37b 100644 --- a/src/extract/tokenizer.ts +++ b/src/extract/tokenizer.ts @@ -9,6 +9,7 @@ export enum TokenKind { String = "String", Keyword = "Keyword", Unrecognized = "Unrecognized", + Plus = "Plus", } export type Token = { @@ -92,6 +93,9 @@ export function tokenize(mapping: KeywordMapping, src: string): Token[] { case ",": addToken(TokenKind.Comma, idx); break; + case "+": + addToken(TokenKind.Plus, idx); + break; case '"': case "'": case "`": @@ -101,7 +105,9 @@ export function tokenize(mapping: KeywordMapping, src: string): Token[] { const prevTokenKind = tokens[tokens.length - 1]?.kind; if ( !unrecognizedContent.trim() && - (prevTokenKind === TokenKind.ParenLeft || prevTokenKind === TokenKind.Comma) + (prevTokenKind === TokenKind.ParenLeft || + prevTokenKind === TokenKind.Comma || + prevTokenKind === TokenKind.Plus) ) { addToken(TokenKind.String, idx, readString(c)); break; diff --git a/tests/extract.test.ts b/tests/extract.test.ts index a2892aa..a8c1497 100644 --- a/tests/extract.test.ts +++ b/tests/extract.test.ts @@ -5,6 +5,19 @@ import { cwd } from "process"; import { execSync } from "child_process"; import { describe, it, expect } from "vitest"; +// Setup the project structure inside the temp directory +async function setupExtractionEnv(tmpDir: string) { + for (const d of ["src", "scripts", "node_modules"]) { + await symlink(join(cwd(), d), join(tmpDir, d)); + } + await writeFile(join(tmpDir, "package.json"), JSON.stringify({ name: "test", type: "commonjs" })); + + await writeFile( + join(tmpDir, "gettext.config.js"), + `module.exports = { input: { path: './srctest' }, output: { path: './srctest/lang' } };`, + ); +} + describe("Extractor Script Tests", () => { type WithTempDirTest = (tmpDir: string) => Promise; @@ -23,16 +36,7 @@ describe("Extractor Script Tests", () => { it("should correctly extract a message from a $gettext call with a trailing comma", async () => { await withTempDir(async (tmpDir) => { - // 1. Setup the project structure inside the temp directory - for (const d of ["src", "scripts", "node_modules"]) { - await symlink(join(cwd(), d), join(tmpDir, d)); - } - await writeFile(join(tmpDir, "package.json"), JSON.stringify({ name: "test", type: "commonjs" })); - - await writeFile( - join(tmpDir, "gettext.config.js"), - `module.exports = { input: { path: './srctest' }, output: { path: './srctest/lang' } };`, - ); + await setupExtractionEnv(tmpDir); await mkdir(join(tmpDir, "srctest", "lang"), { recursive: true }); await writeFile( @@ -58,4 +62,32 @@ describe("Extractor Script Tests", () => { expect(potContent).toContain("#: srctest/MultiLineLiteralWithComma.js:2"); }); }); + + it("should correctly extract a message from a $gettext call with multiple lines", async () => { + await withTempDir(async (tmpDir) => { + await setupExtractionEnv(tmpDir); + + await mkdir(join(tmpDir, "srctest", "lang"), { recursive: true }); + await writeFile( + join(tmpDir, "srctest", "MultiLine.js"), + ` + const myText = $gettext( + 'This is a multiline template string that is just too long ' + + 'to fit on one line in the code and previously wasn\\'t ' + + 'extracted correctly.' + ); + `, + ); + + execSync(`sh -c 'cd ${tmpDir}; tsx ./scripts/gettext_extract.ts'`); + + // Verify that the output .pot file is correct. + const potContent = (await readFile(join(tmpDir, "srctest", "lang", "messages.pot"))).toString(); + console.debug(potContent); + expect(potContent).toContain( + 'msgid "This is a multiline template string that is just too long to fit on one line in the code and previously wasn\'t extracted correctly."\n', + ); + expect(potContent).toContain("#: srctest/MultiLine.js:2"); + }); + }); }); diff --git a/tests/tokenizer.spec.ts b/tests/tokenizer.spec.ts index db198b7..b56ecbc 100644 --- a/tests/tokenizer.spec.ts +++ b/tests/tokenizer.spec.ts @@ -31,6 +31,15 @@ describe("tokenizer", () => { ]); }); + it("deals with string concatenation", () => { + expect(tokenize(keywords, `("test " + "newlines")`)).toEqual([ + { kind: TokenKind.ParenLeft, idx: 0 }, + { kind: TokenKind.String, value: "test ", idx: 1 }, + { kind: TokenKind.Plus, idx: 9 }, + { kind: TokenKind.String, value: "newlines", idx: 11 }, + ]); + }); + it("read vue file", () => { const src = `