Properly fix Java 8 API compatibility (#2350)

There are 7 methods in ByteBuffer that have the problem described in #2218 and was attempted to be fixed in #2219.
Unfortunately, the fix in #2219 was incomplete as another of these 7 methods is used in the same class.
Also it could happen anytime and with similar cases, that this happens again and is only recognized very late.

4 files changed, 149 insertions, 56 deletions

diff --git a/buildSrc/src/main/kotlin/Java9Modularity.kt b/buildSrc/src/main/kotlin/Java9Modularity.kt
index 54a09016..99b17401 100644
--- a/buildSrc/src/main/kotlin/Java9Modularity.kt
+++ b/buildSrc/src/main/kotlin/Java9Modularity.kt
@@ -3,15 +3,22 @@
  */
 
 import org.gradle.api.*
+import org.gradle.api.file.*
+import org.gradle.api.provider.*
+import org.gradle.api.tasks.*
 import org.gradle.api.tasks.bundling.*
 import org.gradle.api.tasks.compile.*
+import org.gradle.jvm.toolchain.*
 import org.gradle.kotlin.dsl.*
+import org.gradle.language.base.plugins.LifecycleBasePlugin.*
+import org.gradle.process.*
 import org.jetbrains.kotlin.gradle.dsl.*
+import org.jetbrains.kotlin.gradle.plugin.*
 import org.jetbrains.kotlin.gradle.plugin.mpp.*
-import org.jetbrains.kotlin.gradle.plugin.mpp.pm20.*
 import org.jetbrains.kotlin.gradle.plugin.mpp.pm20.util.*
 import org.jetbrains.kotlin.gradle.targets.jvm.*
 import org.jetbrains.kotlin.gradle.tasks.KotlinCompile
+import org.jetbrains.kotlin.gradle.tasks.KotlinJvmCompile
 import java.io.*
 
 object Java9Modularity {
@@ -42,70 +49,154 @@ object Java9Modularity {
 
                 // derive the names of the source set and compile module task
                 val sourceSetName = defaultSourceSet.name + "Module"
-                val compileModuleTaskName = compileKotlinTask.name + "Module"
 
                 kotlin.sourceSets.create(sourceSetName) {
                     val sourceFile = this.kotlin.find { it.name == "module-info.java" }
-                    val targetFile = compileKotlinTask.destinationDirectory.file("../module-info.class").get().asFile
+                    val targetDirectory = compileKotlinTask.destinationDirectory.map {
+                        it.dir("../${it.asFile.name}Module")
+                    }
 
                     // only configure the compilation if necessary
                     if (sourceFile != null) {
-                        // the default source set depends on this new source set
-                        defaultSourceSet.dependsOn(this)
+                        // register and wire a task to verify module-info.java content
+                        //
+                        // this will compile the whole sources again with a JPMS-aware target Java version,
+                        // so that the Kotlin compiler can do the necessary verifications
+                        // while compiling with `jdk-release=1.8` those verifications are not done
+                        //
+                        // this task is only going to be executed when running with `check` or explicitly,
+                        // not during normal build operations
+                        val verifyModuleTask = registerVerifyModuleTask(
+                            compileKotlinTask,
+                            sourceFile
+                        )
+                        tasks.named("check") {
+                            dependsOn(verifyModuleTask)
+                        }
 
                         // register a new compile module task
-                        val compileModuleTask = registerCompileModuleTask(compileModuleTaskName, compileKotlinTask, sourceFile, targetFile)
+                        val compileModuleTask = registerCompileModuleTask(
+                            compileKotlinTask,
+                            sourceFile,
+                            targetDirectory
+                        )
 
                         // add the resulting module descriptor to this target's artifact
-                        artifactTask.dependsOn(compileModuleTask)
-                        artifactTask.from(targetFile) {
+                        artifactTask.from(compileModuleTask) {
                             if (multiRelease) {
                                 into("META-INF/versions/9/")
                             }
                         }
                     } else {
                         logger.info("No module-info.java file found in ${this.kotlin.srcDirs}, can't configure compilation of module-info!")
-                        // remove the source set to prevent Gradle warnings
-                        kotlin.sourceSets.remove(this)
                     }
+
+                    // remove the source set to prevent Gradle warnings
+                    kotlin.sourceSets.remove(this)
                 }
             }
         }
     }
 
-    private fun Project.registerCompileModuleTask(taskName: String, compileTask: KotlinCompile, sourceFile: File, targetFile: File) =
-        tasks.register(taskName, JavaCompile::class) {
-            // Also add the module-info.java source file to the Kotlin compile task;
-            // the Kotlin compiler will parse and check module dependencies,
-            // but it currently won't compile to a module-info.class file.
-            compileTask.source(sourceFile)
+    /**
+     * Add a Kotlin compile task that compiles `module-info.java` source file and Kotlin sources together,
+     * the Kotlin compiler will parse and check module dependencies,
+     * but it currently won't compile to a module-info.class file.
+     */
+    private fun Project.registerVerifyModuleTask(
+        compileTask: KotlinCompile,
+        sourceFile: File
+    ): TaskProvider<out KotlinJvmCompile> {
+        apply<KotlinBaseApiPlugin>()
+        val verifyModuleTaskName = "verify${compileTask.name.removePrefix("compile").capitalize()}Module"
+        // work-around for https://youtrack.jetbrains.com/issue/KT-60542
+        val verifyModuleTask = plugins
+            .findPlugin(KotlinBaseApiPlugin::class)!!
+            .registerKotlinJvmCompileTask(verifyModuleTaskName)
+        verifyModuleTask {
+            group = VERIFICATION_GROUP
+            description = "Verify Kotlin sources for JPMS problems"
+            libraries.from(compileTask.libraries)
+            source(compileTask.sources)
+            source(compileTask.javaSources)
+            // part of work-around for https://youtrack.jetbrains.com/issue/KT-60541
+            @Suppress("INVISIBLE_MEMBER")
+            source(compileTask.scriptSources)
+            source(sourceFile)
+            destinationDirectory.set(temporaryDir)
+            multiPlatformEnabled.set(compileTask.multiPlatformEnabled)
+            kotlinOptions {
+                moduleName = compileTask.kotlinOptions.moduleName
+                jvmTarget = "9"
+                freeCompilerArgs += "-Xjdk-release=9"
+            }
+            // work-around for https://youtrack.jetbrains.com/issue/KT-60583
+            inputs.files(
+                libraries.asFileTree.elements.map { libs ->
+                    libs
+                        .filter { it.asFile.exists() }
+                        .map {
+                            zipTree(it.asFile).filter { it.name == "module-info.class" }
+                        }
+                }
+            ).withPropertyName("moduleInfosOfLibraries")
+            this as KotlinCompile
+            // part of work-around for https://youtrack.jetbrains.com/issue/KT-60541
+            @Suppress("DEPRECATION")
+            ownModuleName.set(compileTask.kotlinOptions.moduleName)
+            // part of work-around for https://youtrack.jetbrains.com/issue/KT-60541
+            @Suppress("INVISIBLE_MEMBER")
+            commonSourceSet.from(compileTask.commonSourceSet)
+            // part of work-around for https://youtrack.jetbrains.com/issue/KT-60541
+            // and work-around for https://youtrack.jetbrains.com/issue/KT-60582
+            incremental = false
+        }
+        return verifyModuleTask
+    }
+
+    private fun Project.registerCompileModuleTask(
+        compileTask: KotlinCompile,
+        sourceFile: File,
+        targetDirectory: Provider<out Directory>
+    ) = tasks.register("${compileTask.name}Module", JavaCompile::class) {
+        // Configure the module compile task.
+        source(sourceFile)
+        classpath = files()
+        destinationDirectory.set(targetDirectory)
+        // use a Java 11 toolchain with release 9 option
+        // because for some OS / architecture combinations
+        // there are no Java 9 builds available
+        javaCompiler.set(
+            this@registerCompileModuleTask.the<JavaToolchainService>().compilerFor {
+                languageVersion.set(JavaLanguageVersion.of(11))
+            }
+        )
+        options.release.set(9)
 
+        options.compilerArgumentProviders.add(object : CommandLineArgumentProvider {
+            @get:CompileClasspath
+            val compileClasspath = compileTask.libraries
 
-            // Configure the module compile task.
-            dependsOn(compileTask)
-            source(sourceFile)
-            outputs.file(targetFile)
-            classpath = files()
-            destinationDirectory.set(compileTask.destinationDirectory)
-            sourceCompatibility = JavaVersion.VERSION_1_9.toString()
-            targetCompatibility = JavaVersion.VERSION_1_9.toString()
+            @get:CompileClasspath
+            val compiledClasses = compileTask.destinationDirectory
 
-            doFirst {
+            @get:Input
+            val moduleName = sourceFile
+                .readLines()
+                .single { it.contains("module ") }
+                .substringAfter("module ")
+                .substringBefore(' ')
+                .trim()
+
+            override fun asArguments() = mutableListOf(
                 // Provide the module path to the compiler instead of using a classpath.
                 // The module path should be the same as the classpath of the compiler.
-                options.compilerArgs = listOf(
-                    "--release", "9",
-                    "--module-path", compileTask.libraries.asPath,
-                    "-Xlint:-requires-transitive-automatic"
-                )
-            }
-
-            doLast {
-                // Move the compiled file out of the Kotlin compile task's destination dir,
-                // so it won't disturb Gradle's caching mechanisms.
-                val compiledFile = destinationDirectory.file(targetFile.name).get().asFile
-                targetFile.parentFile.mkdirs()
-                compiledFile.renameTo(targetFile)
-            }
-        }
+                "--module-path",
+                compileClasspath.asPath,
+                "--patch-module",
+                "$moduleName=${compiledClasses.get()}",
+                "-Xlint:-requires-transitive-automatic"
+            )
+        })
+    }
 }
diff --git a/formats/json/jvmMain/src/kotlinx/serialization/json/internal/CharsetReader.kt b/formats/json/jvmMain/src/kotlinx/serialization/json/internal/CharsetReader.kt
index d7c606bd..0f8a566c 100644
--- a/formats/json/jvmMain/src/kotlinx/serialization/json/internal/CharsetReader.kt
+++ b/formats/json/jvmMain/src/kotlinx/serialization/json/internal/CharsetReader.kt
@@ -20,20 +20,7 @@ internal class CharsetReader(
             .onMalformedInput(CodingErrorAction.REPLACE)
             .onUnmappableCharacter(CodingErrorAction.REPLACE)
         byteBuffer = ByteBuffer.wrap(ByteArrayPool8k.take())
-        // An explicit cast is needed here due to an API change in Java 9, see #2218.
-        //
-        // In Java 8 and earlier, the `flip` method was final in `Buffer`, and returned a `Buffer`.
-        // In Java 9 and later, the method was opened, and `ByteFuffer` overrides it, returning a `ByteBuffer`.
-        //
-        // You could observe this by decompiling this call with `javap`:
-        // Compiled with Java 8 it produces `INVOKEVIRTUAL java/nio/ByteBuffer.flip ()Ljava/nio/Buffer;`
-        // Compiled with Java 9+ it produces `INVOKEVIRTUAL java/nio/ByteBuffer.flip ()Ljava/nio/ByteBuffer;`
-        //
-        // This causes a `NoSuchMethodError` when running a class, compiled with a newer Java version, on Java 8.
-        //
-        // To mitigate that, `--bootclasspath` / `--release` options were introduced in `javac`, but there are no
-        // counterparts for these options in `kotlinc`, so an explicit cast is required.
-        (byteBuffer as Buffer).flip() // Make empty
+        byteBuffer.flip() // Make empty
     }
 
     @Suppress("NAME_SHADOWING")
@@ -106,7 +93,7 @@ internal class CharsetReader(
             // Method `position(I)LByteBuffer` does not exist in Java 8. For details, see comment for `flip` in `init` method
             (byteBuffer as Buffer).position(position + bytesRead)
         } finally {
-            (byteBuffer as Buffer).flip() // see the `init` block in this class for the reasoning behind the cast
+            byteBuffer.flip()
         }
         return byteBuffer.remaining()
     }
diff --git a/gradle/configure-source-sets.gradle b/gradle/configure-source-sets.gradle
index 8bba1cff..56a2239e 100644
--- a/gradle/configure-source-sets.gradle
+++ b/gradle/configure-source-sets.gradle
@@ -2,12 +2,23 @@
  * Copyright 2017-2022 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
  */
 
+java {
+    toolchain {
+        languageVersion.set(JavaLanguageVersion.of(11))
+    }
+}
+
+tasks.withType(JavaCompile).configureEach {
+    options.release = 8
+}
+
 kotlin {
     jvm {
         withJava()
-        configure([compilations.main, compilations.test]) {
+        compilations.configureEach {
             kotlinOptions {
                 jvmTarget = '1.8'
+                freeCompilerArgs += '-Xjdk-release=1.8'
             }
         }
     }
diff --git a/settings.gradle b/settings.gradle
index 01d4ea62..ed5256ee 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -2,6 +2,10 @@
  * Copyright 2017-2020 JetBrains s.r.o. Use of this source code is governed by the Apache 2.0 license.
  */
 
+plugins {
+    id 'org.gradle.toolchains.foojay-resolver-convention' version '0.5.0'
+}
+
 rootProject.name = 'kotlinx-serialization'
 
 include ':kotlinx-serialization-core'