fix for 3307 (windows launcher rejects filenames with utf8 chars) (#3923)

* fix for 3307; windows launcher can run scripts with utf8 chars

* fix for macos

* corrected SAVED_CODEPAGE

* add extensive logging to scala.cli.integration.RunTestsDefault.UTF-8

* MacOS fixes in generate-native-image.sh; scalafmt; disable integration.test UTF-8 on Windows

* native image tweaks for MacOS and Linux

* RunTestDefinitions Linux/MacOS

* set utf8 registry for all windows workflows

* fix format errors not caught by ./mill -i integration.test.fix

* add BOM to custom.reg file[skip ci]

* update windows-reg-import/action.yml

* correct action.yml syntax error

* revert .github/script/generate-native-image.sh

* format with scalafmt instead of mill -i fix

* scalafmt build.mill.scala

* simplify to minimum required changes

* resolve review feedback

* remove unneeded code from UTF-8 integration test[skip ci]

* update comment and run deferred ci tests

Author: philwalk

.github/actions/windows-reg-import/action.yml

@@ -0,0 +1,39 @@
+name: windows-reg-import
+description: Import a .reg file and verify those registry values (best-effort)
+inputs:
+  reg-file:
+    description: "Path to the .reg file"
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: Attempt to import custom registry (best-effort)
+      shell: pwsh
+      run: |
+        try {
+          $regFile = Join-Path $env:GITHUB_WORKSPACE "${{ inputs.reg-file }}"
+          if (-not (Test-Path $regFile)) {
+            Write-Warning "Registry file not found (skipping): $regFile"
+          } else {
+            Write-Host "Importing registry from $regFile (attempting, non-fatal)"
+            reg import $regFile 2>&1 | ForEach-Object { Write-Host $_ }
+          }
+        } catch {
+          Write-Warning "Registry import failed (ignored): $_"
+        }
+
+    - name: Attempt to verify registry values (best-effort)
+      shell: pwsh
+      run: |
+        try {
+          $acp = (Get-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Nls\CodePage' -Name ACP -ErrorAction Stop).ACP
+          Write-Host "ACP = $acp"
+        } catch {
+          Write-Warning "Failed to read ACP (ignored): $_"
+        }
+        try {
+          $eb = (Get-ItemProperty -Path 'HKLM:\SYSTEM\CurrentControlSet\Control\Nls\MUILanguagePreferences' -Name EnableBetaUnicode -ErrorAction Stop).EnableBetaUnicode
+          Write-Host "EnableBetaUnicode = $eb"
+        } catch {
+          Write-Warning "Failed to read EnableBetaUnicode (ignored): $_"
+        }

.github/ci/windows/custom.reg

@@ -746,6 +746,10 @@ jobs:
       with:
         fetch-depth: 0
         submodules: true
+    - name: Import custom registry and verify
+      uses: ./.github/actions/windows-reg-import
+      with:
+        reg-file: .github/ci/windows/custom.reg
     - uses: VirtusLab/scala-cli-setup@v1
       with:
         jvm: "temurin:17"
@@ -778,6 +782,10 @@ jobs:
       with:
         fetch-depth: 0
         submodules: true
+    - name: Import custom registry and verify
+      uses: ./.github/actions/windows-reg-import
+      with:
+        reg-file: .github/ci/windows/custom.reg
     - name: Set up Python
       uses: actions/setup-python@v6
       with:
@@ -819,6 +827,10 @@ jobs:
       with:
         fetch-depth: 0
         submodules: true
+    - name: Import custom registry and verify
+      uses: ./.github/actions/windows-reg-import
+      with:
+        reg-file: .github/ci/windows/custom.reg
     - name: Set up Python
       uses: actions/setup-python@v6
       with:
@@ -860,6 +872,10 @@ jobs:
       with:
         fetch-depth: 0
         submodules: true
+    - name: Import custom registry and verify
+      uses: ./.github/actions/windows-reg-import
+      with:
+        reg-file: .github/ci/windows/custom.reg
     - name: Set up Python
       uses: actions/setup-python@v6
       with:
@@ -901,6 +917,10 @@ jobs:
         with:
           fetch-depth: 0
           submodules: true
+      - name: Import custom registry and verify
+        uses: ./.github/actions/windows-reg-import
+        with:
+          reg-file: .github/ci/windows/custom.reg
       - name: Set up Python
         uses: actions/setup-python@v6
         with:
@@ -942,6 +962,10 @@ jobs:
         with:
           fetch-depth: 0
           submodules: true
+      - name: Import custom registry and verify
+        uses: ./.github/actions/windows-reg-import
+        with:
+          reg-file: .github/ci/windows/custom.reg
       - name: Set up Python
         uses: actions/setup-python@v6
         with:
@@ -1518,6 +1542,10 @@ jobs:
       with:
         fetch-depth: 0
         submodules: true
+    - name: Import custom registry and verify
+      uses: ./.github/actions/windows-reg-import
+      with:
+        reg-file: .github/ci/windows/custom.reg
     - uses: VirtusLab/scala-cli-setup@v1
       with:
         jvm: "temurin:17"
@@ -1824,6 +1852,10 @@ jobs:
       with:
         fetch-depth: 0
         submodules: true
+    - name: Import custom registry and verify
+      uses: ./.github/actions/windows-reg-import
+      with:
+        reg-file: .github/ci/windows/custom.reg
     - uses: VirtusLab/scala-cli-setup@v1
       with:
         jvm: "temurin:17"

.github/workflows/ci.yml

@@ -70,7 +70,8 @@ function to_bash_syntax {
 if [[ $IS_WINDOWS ]]; then
   # needed for coursier version < 2.1.8, harmless otherwise
   IFS=$'\n'
-  eval "$(to_bash_syntax `"$cs" java --env --jvm temurin:17` || to_bash_syntax `"$cs" java --env --jvm openjdk:1.17.0`)"
+  # temurin:17 (build 17+35) doesn't support utf8 filenames, although some later 17 versions do
+  eval "$(to_bash_syntax `"$cs" java --env --jvm zulu:17` || to_bash_syntax `"$cs" java --env --jvm openjdk:1.17.0`)"
   unset IFS
 else
   eval "$("$cs" java --env --jvm temurin:17 || "$cs" java --env --jvm openjdk:1.17.0)"

mill

@@ -1077,11 +1077,11 @@ abstract class RunTestDefinitions
         fileName
       )
         .call(cwd = root)
-      if (res.out.text(Codec.default).trim != message) {
-        pprint.err.log(res.out.text(Codec.default).trim)
+      if (res.out.text(Codec.UTF8).trim != message) {
+        pprint.err.log(res.out.text(Codec.UTF8).trim)
         pprint.err.log(message)
       }
-      expect(res.out.text(Codec.default).trim == message)
+      expect(res.out.text(Codec.UTF8).trim == message)
     }
   }