refactor(macros): remove magnolia dependency for Scala 3, use native macros

Complete migration from magnolia to native Scala 3 macros in zio-json:
- Rewrite encoder with FieldEncoder abstraction, wire explicitNulls/explicitEmptyCollections
- Wire explicitEmptyCollections.decoding in decoder with missingValueDecoder pattern
- Add >64 field StringMatrix split support and ADT name distinctness validation
- Fix constructProduct for parameterless Scala 3 enum cases
- Fix encoder indent handling (indent_ for field values)
- Remove magnolia1_3 dependency from build.sbt
- Split platform-specific test files into scala-2.x/scala-3.x for cross-compilation
- Create test-data subproject for data model files (fixes cyclic macro deps)
- Bump -Xss to 8M in .jvmopts and CI (native macro inline expansion needs more stack)
- Add mima binary compat filters for removed magnolia internals (pre-1.0)
- Fix benchmark imports for relocated TestUtils

Author: 987Nabil

.github/workflows/ci.yml

@@ -1,8 +1,8 @@
 name: CI
 
 env:
-  JDK_JAVA_OPTIONS: -XX:+PrintCommandLineFlags -Xms6G -Xmx6G -Xss4M -XX:+UseG1GC -XX:ReservedCodeCacheSize=512M -XX:NonProfiledCodeHeapSize=256M # JDK_JAVA_OPTIONS is _the_ env. variable to use for modern Java
-  SBT_OPTS: -XX:+PrintCommandLineFlags -Xms6G -Xmx6G -Xss4M -XX:+UseG1GC -XX:ReservedCodeCacheSize=512M -XX:NonProfiledCodeHeapSize=256M # Needed for sbt
+  JDK_JAVA_OPTIONS: -XX:+PrintCommandLineFlags -Xms6G -Xmx6G -Xss8M -XX:+UseG1GC -XX:ReservedCodeCacheSize=512M -XX:NonProfiledCodeHeapSize=256M # JDK_JAVA_OPTIONS is _the_ env. variable to use for modern Java
+  SBT_OPTS: -XX:+PrintCommandLineFlags -Xms6G -Xmx6G -Xss8M -XX:+UseG1GC -XX:ReservedCodeCacheSize=512M -XX:NonProfiledCodeHeapSize=256M # Needed for sbt
 
 on:
   pull_request:

.jvmopts

@@ -1,5 +1,5 @@
 -Xmx6g
--Xss2m
+-Xss8m
 -XX:+UseG1GC
 -XX:InitialCodeCacheSize=512m
 -XX:ReservedCodeCacheSize=512m

README.md

@@ -171,7 +171,7 @@ High **performance** is achieved by decoding JSON directly from the input source
 
 Best in class **security** is achieved with an aggressive *early exit* strategy that avoids costly stack traces, even when parsing malformed numbers. Malicious (and badly formed) payloads are rejected before finishing reading.
 
-**Fast compilation** and **future-proofing** is possible thanks to [Magnolia](https://propensive.com/opensource/magnolia/) which allows us to generate boilerplate in a way that will survive the exodus to Scala 3. `zio-json` is internally implemented using a [`java.io.Reader`](https://docs.oracle.com/en/java/javase/14/docs/api/java.base/java/io/Reader.html) / [`java.io.Writer`](https://docs.oracle.com/en/java/javase/14/docs/api/java.base/java/io/Writer.html)-like interface, which is making a comeback to center stage in Loom.
+**Fast compilation** and **future-proofing**: For Scala 2, derivation uses [Magnolia](https://propensive.com/opensource/magnolia/). For Scala 3, derivation uses native Scala 3 macros. `zio-json` is internally implemented using a [`java.io.Reader`](https://docs.oracle.com/en/java/javase/14/docs/api/java.base/java/io/Reader.html) / [`java.io.Writer`](https://docs.oracle.com/en/java/javase/14/docs/api/java.base/java/io/Writer.html)-like interface, which is making a comeback to center stage in Loom.
 
 **Simplicity** is achieved by using well-known software patterns and avoiding bloat. The only requirement to use this library is to know about Scala's encoding of typeclasses, described in [Functional Programming for Mortals](https://leanpub.com/fpmortals/read#leanpub-auto-functionality).
 

build.sbt

@@ -1,5 +1,5 @@
 import BuildHelper.*
-import com.typesafe.tools.mima.core.Problem
+import com.typesafe.tools.mima.core.*
 import com.typesafe.tools.mima.core.ProblemFilters.exclude
 import com.typesafe.tools.mima.plugin.MimaKeys.mimaPreviousArtifacts
 import explicitdeps.ExplicitDepsPlugin.autoImport.moduleFilterRemoveValue
@@ -36,7 +36,7 @@ addCommandAlias("prepare", "fmt")
 
 addCommandAlias(
   "testJVM",
-  "zioJsonJVM/test; zioJsonYaml/test; zioJsonInteropHttp4s/test; zioJsonGolden/test; zioJsonInteropRefinedJVM/test"
+  "zioJsonTestData/compile; zioJsonJVM/test; zioJsonYaml/test; zioJsonInteropHttp4s/test; zioJsonGolden/test; zioJsonInteropRefinedJVM/test"
 )
 
 addCommandAlias(
@@ -90,7 +90,8 @@ lazy val zioJsonRoot = project
     zioJsonInteropScalaz7x.js,
     zioJsonInteropScalaz7x.jvm,
     zioJsonInteropScalaz7x.native,
-    zioJsonGolden
+    zioJsonGolden,
+    zioJsonTestData
   )
 
 val circeVersion = "0.14.15"
@@ -107,7 +108,7 @@ lazy val zioJson = crossProject(JSPlatform, JVMPlatform, NativePlatform)
     // as per @fommil, optimization slows things down.
     scalacOptions -= "-opt:l:inline",
     scalacOptions -= "-opt-inline-from:zio.internal.**",
-    Test / scalacOptions ++= {
+    scalacOptions ++= {
       if (scalaVersion.value == Scala3)
         Vector("-Yretain-trees", "-Xmax-inlines:128")
       else
@@ -130,9 +131,7 @@ lazy val zioJson = crossProject(JSPlatform, JVMPlatform, NativePlatform)
     libraryDependencies ++= {
       CrossVersion.partialVersion(scalaVersion.value) match {
         case Some((3, _)) =>
-          Seq(
-            "com.softwaremill.magnolia1_3" %%% "magnolia" % "1.3.18"
-          )
+          Seq.empty
         case _ =>
           Seq(
             "org.scala-lang"                 % "scala-reflect" % scalaVersion.value % Provided,
@@ -235,9 +234,34 @@ lazy val zioJson = crossProject(JSPlatform, JVMPlatform, NativePlatform)
     inConfig(Jmh)(org.scalafmt.sbt.ScalafmtPlugin.scalafmtConfigSettings(Jmh)),
     testFrameworks += new TestFramework("zio.test.sbt.ZTestFramework"),
     mimaBinaryIssueFilters ++= Seq(
-      exclude[Problem]("zio.json.CaseObjectDecoder.*") // FIXME: false negative reported by mima
+      exclude[Problem]("zio.json.CaseObjectDecoder.*"),
+      exclude[MissingClassProblem]("zio.json.CaseObjectDecoder"),
+      exclude[DirectMissingMethodProblem]("zio.json.DeriveJsonDecoder.join"),
+      exclude[DirectMissingMethodProblem]("zio.json.DeriveJsonDecoder.split"),
+      exclude[DirectMissingMethodProblem]("zio.json.DeriveJsonDecoder.subtypes$default$2"),
+      exclude[DirectMissingMethodProblem]("zio.json.DeriveJsonDecoder.getParams$default$4"),
+      exclude[DirectMissingMethodProblem]("zio.json.DeriveJsonDecoder.getParams_$default$5"),
+      exclude[DirectMissingMethodProblem]("zio.json.DeriveJsonDecoder.getParams__$default$6"),
+      exclude[MissingTypesProblem]("zio.json.DeriveJsonDecoder$"),
+      exclude[MissingClassProblem]("zio.json.DeriveJsonDecoder$ArraySeq"),
+      exclude[DirectMissingMethodProblem]("zio.json.DeriveJsonEncoder.join"),
+      exclude[DirectMissingMethodProblem]("zio.json.DeriveJsonEncoder.split"),
+      exclude[DirectMissingMethodProblem]("zio.json.DeriveJsonEncoder.subtypes$default$2"),
+      exclude[DirectMissingMethodProblem]("zio.json.DeriveJsonEncoder.getParams$default$4"),
+      exclude[DirectMissingMethodProblem]("zio.json.DeriveJsonEncoder.getParams_$default$5"),
+      exclude[DirectMissingMethodProblem]("zio.json.DeriveJsonEncoder.getParams__$default$6"),
+      exclude[MissingTypesProblem]("zio.json.DeriveJsonEncoder$"),
+      exclude[MissingClassProblem]("zio.json.JsonDecoderDerivation"),
+      exclude[MissingClassProblem]("zio.json.JsonDecoderDerivation$ArraySeq"),
+      exclude[MissingClassProblem]("zio.json.JsonEncoderDerivation"),
+      exclude[MissingClassProblem]("zio.json.macros$package"),
+      exclude[MissingClassProblem]("zio.json.macros$package$")
     )
   )
+  .jvmSettings(
+    Test / unmanagedClasspath += Attributed.blank((zioJsonTestData / Compile / classDirectory).value),
+    Jmh / unmanagedClasspath ++= (Test / fullClasspath).value
+  )
   .jsSettings(
     mimaBinaryIssueFilters ++= Seq(
       exclude[Problem]("zio.JsonPackagePlatformSpecific.*"),
@@ -424,3 +448,19 @@ lazy val docs = project
          |""".stripMargin
   )
   .enablePlugins(WebsitePlugin)
+
+lazy val zioJsonTestData = project
+  .in(file("zio-json-test-data/jvm"))
+  .settings(
+    name                  := "zio-json-test-data",
+    scalaVersion          := Scala3,
+    publish / skip        := true,
+    mimaPreviousArtifacts := Set(),
+    scalacOptions ++= Seq("-Yretain-trees", "-Xmax-inlines:128"),
+    Compile / unmanagedClasspath ++= (LocalProject("zioJsonJVM") / Compile / fullClasspath).value,
+    libraryDependencies ++= Seq(
+      "io.circe"                              %% "circe-core"            % circeVersion,
+      "io.circe"                              %% "circe-generic"         % circeVersion,
+      "com.github.plokhotnyuk.jsoniter-scala" %% "jsoniter-scala-macros" % "2.38.6"
+    )
+  )

docs/index.mdx

@@ -31,8 +31,8 @@ In order to use this library, we need to add the following line in our `build.sb
 libraryDependencies += "dev.zio" %% "zio-json" % "@VERSION@"
 ```
 
-For cross-platform projects with Scala.js and Scala Native need to replace `%%` operator by `%%%`, 
-and optionally when using `java.time.ZoneId` and `java.time.ZonedDateTime` types need to add 
+For cross-platform projects with Scala.js and Scala Native need to replace `%%` operator by `%%%`,
+and optionally when using `java.time.ZoneId` and `java.time.ZonedDateTime` types need to add
 the dependency on the latest version of Timezone DB:
 
 ```scala
@@ -269,7 +269,7 @@ Extreme **performance** is achieved by decoding JSON directly from the input sou
 
 Best in class **security** is achieved with an aggressive *early exit* strategy that avoids costly stack traces, even when parsing malformed numbers. Malicious (and badly formed) payloads are rejected before finishing reading.
 
-**Fast compilation** and **future-proofing** is possible thanks to [Magnolia](https://propensive.com/opensource/magnolia/) which allows us to generate boilerplate in a way that will survive the exodus to Scala 3. `zio-json` is internally implemented using a [`java.io.Reader`](https://docs.oracle.com/en/java/javase/14/docs/api/java.base/java/io/Reader.html) / [`java.io.Writer`](https://docs.oracle.com/en/java/javase/14/docs/api/java.base/java/io/Writer.html)-like interface, which is making a comeback to center stage in Loom.
+**Fast compilation** and **future-proofing**: For Scala 2, derivation uses [Magnolia](https://propensive.com/opensource/magnolia/). For Scala 3, derivation uses native Scala 3 macros. `zio-json` is internally implemented using a [`java.io.Reader`](https://docs.oracle.com/en/java/javase/14/docs/api/java.base/java/io/Reader.html) / [`java.io.Writer`](https://docs.oracle.com/en/java/javase/14/docs/api/java.base/java/io/Writer.html)-like interface, which is making a comeback to center stage in Loom.
 
 **Simplicity** is achieved by using well-known software patterns and avoiding bloat. The only requirement to use this library is to know about Scala's encoding of typeclasses, described in [Functional Programming for Mortals](https://leanpub.com/fpmortals/read#leanpub-auto-functionality).
 

…cala/zio/json/data/geojson/GeoJSON.scala …cala/zio/json/data/geojson/GeoJSON.scalazio-json/jvm/src/test/scala/zio/json/data/geojson/GeoJSON.scala renamed to zio-json-test-data/jvm/src/main/scala/zio/json/data/geojson/GeoJSON.scala zio-json/jvm/src/test/scala/zio/json/data/geojson/GeoJSON.scala renamed to zio-json-test-data/jvm/src/main/scala/zio/json/data/geojson/GeoJSON.scala

@@ -120,7 +120,7 @@ package handrolled {
     // custom decoders and is not a requirement to use the JsonDecoder[GeoJSON]
     // custom decoder (below) which is necessary to avert a DOS attack.
     implicit lazy val zioJsonJsonDecoder: JsonDecoder[Geometry] =
-      new JsonDecoder.AbstractJsonDecoder[Geometry] {
+      new JsonDecoder[Geometry] {
         import zio.json._
         import JsonDecoder.JsonError
         import internal._
@@ -270,7 +270,7 @@ package handrolled {
     // potentially complex and even skipping over it is expensive... it's a bit
     // of a corner case.
     implicit lazy val zioJsonJsonDecoder: JsonDecoder[GeoJSON] =
-      new JsonDecoder.AbstractJsonDecoder[GeoJSON] {
+      new JsonDecoder[GeoJSON] {
         import zio.json._
         import JsonDecoder.JsonError
         import internal._

…io/json/data/googlemaps/GoogleMaps.scala …io/json/data/googlemaps/GoogleMaps.scalazio-json/jvm/src/test/scala/zio/json/data/googlemaps/GoogleMaps.scala renamed to zio-json-test-data/jvm/src/main/scala/zio/json/data/googlemaps/GoogleMaps.scala zio-json/jvm/src/test/scala/zio/json/data/googlemaps/GoogleMaps.scala renamed to zio-json-test-data/jvm/src/main/scala/zio/json/data/googlemaps/GoogleMaps.scala

@@ -6,7 +6,7 @@ import com.github.plokhotnyuk.jsoniter_scala.core._
 import com.github.plokhotnyuk.jsoniter_scala.macros._
 import io.circe
 import zio.json.GeoJSONBenchmarks._
-import zio.json.TestUtils._
+import zio.jsontest.TestUtils._
 import zio.json.data.geojson.handrolled._
 import org.openjdk.jmh.annotations._
 

…cala/zio/json/data/twitter/Twitter.scala …cala/zio/json/data/twitter/Twitter.scalazio-json/jvm/src/test/scala/zio/json/data/twitter/Twitter.scala renamed to zio-json-test-data/jvm/src/main/scala/zio/json/data/twitter/Twitter.scala zio-json/jvm/src/test/scala/zio/json/data/twitter/Twitter.scala renamed to zio-json-test-data/jvm/src/main/scala/zio/json/data/twitter/Twitter.scala

@@ -5,7 +5,7 @@ import java.util.concurrent.TimeUnit
 import com.github.plokhotnyuk.jsoniter_scala.core._
 import com.github.plokhotnyuk.jsoniter_scala.macros._
 import io.circe
-import zio.json.TestUtils._
+import zio.jsontest.TestUtils._
 import zio.json.data.googlemaps._
 import org.openjdk.jmh.annotations._
 import zio.json.GoogleMapsAPIBenchmarks._