fix: add a plus token and concatenate string arguments, add tests

src/extract/parser.ts

@@ -14,8 +14,10 @@ type MsgInfoWithCharIdx = BaseMsg & { idx: number };
 export function parseFunctionCall(mapping: KeywordMapping, tokens: Token[]): MsgInfoWithCharIdx[] {
   let idx = -1;
   let t: Token | undefined = undefined;
+  let previousTokenKind: TokenKind | undefined = undefined;
 
   function advance() {
+    previousTokenKind = tokens[idx]?.kind || undefined;
     idx += 1;
     return tokens[idx];
   }
@@ -48,14 +50,19 @@ export function parseFunctionCall(mapping: KeywordMapping, tokens: Token[]): Msg
         break;
       }
       assertIsDefined(t.value);
-      stringArgs.push(t.value);
+      if (previousTokenKind === TokenKind.Plus) {
+        stringArgs[stringArgs.length - 1] += t.value;
+      } else {
+        stringArgs.push(t.value);
+      }
       t = advance();
       if (!t) {
         break;
       }
-      if (t.kind !== TokenKind.Comma) {
+      if (t.kind !== TokenKind.Comma && t.kind !== TokenKind.Plus) {
         break;
       }
+
       t = advance();
       if (!t) {
         break;

src/extract/tokenizer.ts

@@ -9,6 +9,7 @@ export enum TokenKind {
   String = "String",
   Keyword = "Keyword",
   Unrecognized = "Unrecognized",
+  Plus = "Plus",
 }
 
 export type Token = {
@@ -92,6 +93,9 @@ export function tokenize(mapping: KeywordMapping, src: string): Token[] {
       case ",":
         addToken(TokenKind.Comma, idx);
         break;
+      case "+":
+        addToken(TokenKind.Plus, idx);
+        break;
       case '"':
       case "'":
       case "`":
@@ -101,7 +105,9 @@ export function tokenize(mapping: KeywordMapping, src: string): Token[] {
         const prevTokenKind = tokens[tokens.length - 1]?.kind;
         if (
           !unrecognizedContent.trim() &&
-          (prevTokenKind === TokenKind.ParenLeft || prevTokenKind === TokenKind.Comma)
+          (prevTokenKind === TokenKind.ParenLeft ||
+            prevTokenKind === TokenKind.Comma ||
+            prevTokenKind === TokenKind.Plus)
         ) {
           addToken(TokenKind.String, idx, readString(c));
           break;

tests/extract.test.ts

@@ -5,6 +5,19 @@ import { cwd } from "process";
 import { execSync } from "child_process";
 import { describe, it, expect } from "vitest";
 
+// Setup the project structure inside the temp directory
+async function setupExtractionEnv(tmpDir: string) {
+  for (const d of ["src", "scripts", "node_modules"]) {
+    await symlink(join(cwd(), d), join(tmpDir, d));
+  }
+  await writeFile(join(tmpDir, "package.json"), JSON.stringify({ name: "test", type: "commonjs" }));
+
+  await writeFile(
+    join(tmpDir, "gettext.config.js"),
+    `module.exports = { input: { path: './srctest' }, output: { path: './srctest/lang' } };`,
+  );
+}
+
 describe("Extractor Script Tests", () => {
   type WithTempDirTest = (tmpDir: string) => Promise<any>;
 
@@ -23,16 +36,7 @@ describe("Extractor Script Tests", () => {
 
   it("should correctly extract a message from a $gettext call with a trailing comma", async () => {
     await withTempDir(async (tmpDir) => {
-      // 1. Setup the project structure inside the temp directory
-      for (const d of ["src", "scripts", "node_modules"]) {
-        await symlink(join(cwd(), d), join(tmpDir, d));
-      }
-      await writeFile(join(tmpDir, "package.json"), JSON.stringify({ name: "test", type: "commonjs" }));
-
-      await writeFile(
-        join(tmpDir, "gettext.config.js"),
-        `module.exports = { input: { path: './srctest' }, output: { path: './srctest/lang' } };`,
-      );
+      await setupExtractionEnv(tmpDir);
 
       await mkdir(join(tmpDir, "srctest", "lang"), { recursive: true });
       await writeFile(
@@ -58,4 +62,32 @@ describe("Extractor Script Tests", () => {
       expect(potContent).toContain("#: srctest/MultiLineLiteralWithComma.js:2");
     });
   });
+
+  it("should correctly extract a message from a $gettext call with multiple lines", async () => {
+    await withTempDir(async (tmpDir) => {
+      await setupExtractionEnv(tmpDir);
+
+      await mkdir(join(tmpDir, "srctest", "lang"), { recursive: true });
+      await writeFile(
+        join(tmpDir, "srctest", "MultiLine.js"),
+        `
+          const myText = $gettext(
+            'This is a multiline template string that is just too long ' +
+              'to fit on one line in the code and previously wasn\\'t ' +
+              'extracted correctly.'
+          );
+        `,
+      );
+
+      execSync(`sh -c 'cd ${tmpDir}; tsx ./scripts/gettext_extract.ts'`);
+
+      // Verify that the output .pot file is correct.
+      const potContent = (await readFile(join(tmpDir, "srctest", "lang", "messages.pot"))).toString();
+      console.debug(potContent);
+      expect(potContent).toContain(
+        'msgid "This is a multiline template string that is just too long to fit on one line in the code and previously wasn\'t extracted correctly."\n',
+      );
+      expect(potContent).toContain("#: srctest/MultiLine.js:2");
+    });
+  });
 });

tests/tokenizer.spec.ts

@@ -31,6 +31,15 @@ describe("tokenizer", () => {
     ]);
   });
 
+  it("deals with string concatenation", () => {
+    expect(tokenize(keywords, `("test " + "newlines")`)).toEqual(<Token[]>[
+      { kind: TokenKind.ParenLeft, idx: 0 },
+      { kind: TokenKind.String, value: "test ", idx: 1 },
+      { kind: TokenKind.Plus, idx: 9 },
+      { kind: TokenKind.String, value: "newlines", idx: 11 },
+    ]);
+  });
+
   it("read vue file", () => {
     const src = `
 <script>