refactor!: return JsonArray from fullQuery

gradle.properties

@@ -19,7 +19,7 @@ android.useAndroidX=true
 android.enableJetifier=false
 
 GROUP=io.github.david-allison
-VERSION_NAME=0.1.70-anki26.09.3
+VERSION_NAME=0.1.71-anki26.09.3
 
 POM_INCEPTION_YEAR=2020
 

gradle/libs.versions.toml

@@ -28,6 +28,8 @@ gradleMavenPublishPlugin = "0.37.0"
 junit = "6.1.3"
 # https://github.com/JetBrains/kotlin/releases/
 kotlin = '2.3.21'
+# https://github.com/Kotlin/kotlinx.serialization/releases
+kotlinxSerialization = "1.9.0"
 # https://github.com/protocolbuffers/protobuf/releases
 protobuf = "4.36.2"
 robolectric = "4.17"
@@ -45,6 +47,7 @@ androidx-core = { module = "androidx.test:core", version.ref = "androidxCore" }
 androidx-sqlite-framework = { module = "androidx.sqlite:sqlite-framework", version.ref = "androidxSqlite" }
 androidx-sqlite-ktx = { module = "androidx.sqlite:sqlite-ktx", version.ref = "androidxSqlite" }
 kotlin-stdlib-jdk8 = { module = "org.jetbrains.kotlin:kotlin-stdlib-jdk8", version.ref = "kotlin" }
+kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlinxSerialization" }
 protobuf-kotlin-lite = { module = "com.google.protobuf:protobuf-kotlin-lite", version.ref = "protobuf" }
 slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" }
 

rsdroid-instrumented/src/androidTest/java/net/ankiweb/rsdroid/BackendIntegrationTests.kt

@@ -17,6 +17,7 @@ package net.ankiweb.rsdroid
 
 import androidx.test.ext.junit.runners.AndroidJUnit4
 import anki.ankidroid.SchedTimingTodayLegacyRequest
+import kotlinx.serialization.json.jsonArray
 import net.ankiweb.rsdroid.ankiutil.InstrumentedTest
 import org.hamcrest.MatcherAssert.assertThat
 import org.hamcrest.core.Is
@@ -59,7 +60,9 @@ class BackendIntegrationTests : InstrumentedTest() {
     @Test
     fun fullQueryTest() {
         val backendV1 = getBackend("initial_version_2_12_1.anki2")
-        backendV1.fullQuery("select * from col")
+        val result = backendV1.fullQuery("select * from col")
+        Assert.assertEquals(1, result.size)
+        Assert.assertEquals(13, result[0].jsonArray.size)
     }
 
     @Test

rsdroid/build.gradle

@@ -93,6 +93,8 @@ afterEvaluate {
 dependencies {
     // Protobuf is part of the ABI, so include it as a compile/api dependency.
     api libs.protobuf.kotlin.lite
+    // JsonArray is returned by the public fullQuery API.
+    api libs.kotlinx.serialization.json
 
     implementation libs.androidx.annotation
     implementation libs.androidx.sqlite.ktx

rsdroid/src/main/java/net/ankiweb/rsdroid/Backend.kt

@@ -24,10 +24,12 @@ import anki.backend.GeneratedBackend
 import anki.generic.Int64
 import com.google.protobuf.ByteString
 import com.google.protobuf.InvalidProtocolBufferException
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.JsonArray
+import kotlinx.serialization.json.jsonArray
 import net.ankiweb.rsdroid.database.NotImplementedException
 import net.ankiweb.rsdroid.database.SQLHandler
 import org.json.JSONArray
-import org.json.JSONException
 import org.json.JSONObject
 import org.slf4j.LoggerFactory
 import java.io.Closeable
@@ -155,20 +157,9 @@ open class Backend(
     override fun fullQuery(
         query: String,
         bindArgs: Array<Any?>?,
-    ): JSONArray =
-        try {
-            fullQueryInternal(query, bindArgs ?: emptyArray())
-        } catch (e: JSONException) {
-            throw RuntimeException(e)
-        }
-
-    @Throws(JSONException::class)
-    private fun fullQueryInternal(
-        sql: String,
-        bindArgs: Array<Any?>,
-    ): JSONArray {
-        val output = runDbCommand(dbRequestJson(sql, bindArgs)).toStringUtf8()
-        return JSONArray(output)
+    ): JsonArray {
+        val output = runDbCommand(dbRequestJson(query, bindArgs ?: emptyArray())).toStringUtf8()
+        return Json.parseToJsonElement(output).jsonArray
     }
 
     override fun insertForId(

rsdroid/src/main/java/net/ankiweb/rsdroid/database/SQLHandler.kt

@@ -17,16 +17,25 @@ package net.ankiweb.rsdroid.database
 
 import androidx.annotation.CheckResult
 import anki.ankidroid.DbResponse
-import org.json.JSONArray
+import kotlinx.serialization.json.JsonArray
 
 interface SQLHandler {
+    /**
+     * Executes [query] and returns all rows as JSON arrays of column values.
+     *
+     * The result is fully materialized; no paging or query-cache cleanup is
+     * required. Pass `null` for [bindArgs] when there are no bind arguments.
+     *
+     * For large results, use [fullQueryProto] to retrieve the result in pages.
+     */
     @CheckResult
     fun fullQuery(
         query: String,
         bindArgs: Array<Any?>?,
-    ): JSONArray
+    ): JsonArray
 
-    fun fullQuery(query: String): JSONArray = fullQuery(query, null)
+    @CheckResult
+    fun fullQuery(query: String): JsonArray = fullQuery(query, null)
 
     fun executeGetRowsAffected(
         sql: String,

rsdroid/src/test/java/net/ankiweb/BackendCloseRaceTest.kt

@@ -1,12 +1,16 @@
 // SPDX-License-Identifier: GPL-3.0-or-later
 package net.ankiweb
 
-import android.annotation.SuppressLint
 import androidx.test.ext.junit.runners.AndroidJUnit4
+import kotlinx.serialization.json.jsonArray
+import kotlinx.serialization.json.jsonPrimitive
+import kotlinx.serialization.json.long
 import net.ankiweb.rsdroid.Backend
 import net.ankiweb.rsdroid.BackendException
 import net.ankiweb.rsdroid.BackendFactory.getBackend
 import net.ankiweb.rsdroid.testing.RustBackendLoader.ensureSetup
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertFalse
 import org.junit.Assert.assertTrue
 import org.junit.Before
 import org.junit.Test
@@ -34,15 +38,17 @@ class BackendCloseRaceTest {
     fun closeDoesNotInterruptInFlightCalls() {
         val backend = getBackend()
         backend.openCollection(":memory:")
-        @SuppressLint("CheckResult")
-        backend.fullQuery(longQuery(rows = 1_000), null) // warm up the query path
+        val warmup = backend.fullQuery(longQuery(rows = 1_000))
+        assertEquals(1_000L, warmup[0].jsonArray[0].jsonPrimitive.long)
 
         var queryError: Exception? = null
+        var queryCount: Long? = null
         val queryThread =
             thread(name = "backend-slow-query") {
                 try {
                     // keeps the backend busy inside a single native call for over a second
-                    backend.fullQuery(longQuery(rows = 50_000_000), null)
+                    val result = backend.fullQuery(longQuery(rows = 50_000_000))
+                    queryCount = result[0].jsonArray[0].jsonPrimitive.long
                 } catch (e: Exception) {
                     queryError = e
                 }
@@ -51,12 +57,16 @@ class BackendCloseRaceTest {
         sleep(500.milliseconds) // let the query enter native code
         backend.close()
         queryThread.join(1.minutes)
+        assertFalse("query thread did not finish", queryThread.isAlive)
 
         // Acceptable outcomes:
         // * close() waited for the in-flight call: the query succeeds.
         // * the call lost the race and was cleanly rejected.
         if (queryError != null) {
             assertTrue("unexpected query error: $queryError", queryError is BackendException)
+            assertEquals("Backend has been closed", queryError.message)
+        } else {
+            assertEquals(50_000_000L, queryCount)
         }
     }
 

rsdroid/src/test/java/net/ankiweb/FullQueryTest.kt

@@ -0,0 +1,114 @@
+// SPDX-License-Identifier: GPL-3.0-or-later
+package net.ankiweb
+
+import androidx.test.ext.junit.runners.AndroidJUnit4
+import kotlinx.serialization.json.JsonNull
+import kotlinx.serialization.json.double
+import kotlinx.serialization.json.int
+import kotlinx.serialization.json.jsonArray
+import kotlinx.serialization.json.jsonPrimitive
+import kotlinx.serialization.json.long
+import net.ankiweb.rsdroid.Backend
+import net.ankiweb.rsdroid.BackendException
+import net.ankiweb.rsdroid.BackendFactory.getBackend
+import net.ankiweb.rsdroid.database.SQLHandler
+import net.ankiweb.rsdroid.testing.RustBackendLoader.ensureSetup
+import org.junit.Assert.assertEquals
+import org.junit.Assert.assertFalse
+import org.junit.Assert.assertThrows
+import org.junit.Assert.assertTrue
+import org.junit.Before
+import org.junit.Test
+import org.junit.runner.RunWith
+
+@RunWith(AndroidJUnit4::class)
+class FullQueryTest {
+    @Before
+    fun loadLibrary() {
+        ensureSetup()
+    }
+
+    @Test
+    fun sqlValuesAreReturnedWithoutLosingPrecision() {
+        withBackend { backend ->
+            val text = "quote\" slash/line\n🎴"
+            val rows =
+                backend.fullQuery(
+                    "SELECT ?, ?, ?, ?, ?, x'0080ff'",
+                    arrayOf(Long.MIN_VALUE, Long.MAX_VALUE, 1.25, text, null),
+                )
+            assertEquals(1, rows.size)
+            val row = rows[0].jsonArray
+            assertEquals(6, row.size)
+            // Numeric accessors also accept quoted numbers, so check JSON types separately.
+            assertFalse(row[0].jsonPrimitive.isString)
+            assertFalse(row[1].jsonPrimitive.isString)
+            assertFalse(row[2].jsonPrimitive.isString)
+            assertTrue(row[3].jsonPrimitive.isString)
+            assertEquals(Long.MIN_VALUE, row[0].jsonPrimitive.long)
+            assertEquals(Long.MAX_VALUE, row[1].jsonPrimitive.long)
+            assertEquals(1.25, row[2].jsonPrimitive.double, 0.0)
+            assertEquals(text, row[3].jsonPrimitive.content)
+            assertEquals(JsonNull, row[4])
+            assertEquals(listOf(0, 128, 255), row[5].jsonArray.map { it.jsonPrimitive.int })
+            assertTrue(row[5].jsonArray.all { !it.jsonPrimitive.isString })
+        }
+    }
+
+    @Test
+    fun noArgumentOverloadsAndEmptyResultsWork() {
+        withBackend { backend ->
+            val database: SQLHandler = backend
+            val expected = backend.fullQuery("SELECT 42", emptyArray())
+            assertEquals(1, expected.size)
+            assertEquals(42, expected[0].jsonArray[0].jsonPrimitive.int)
+            assertEquals(expected, database.fullQuery("SELECT 42"))
+            assertEquals(expected, database.fullQuery("SELECT 42", null))
+            assertTrue(database.fullQuery("SELECT 42 WHERE 0").isEmpty())
+        }
+    }
+
+    @Test
+    fun resultsRemainReadableAfterClosingBackend() {
+        val rows =
+            getBackend().use { backend ->
+                backend.openCollection(":memory:")
+                backend.fullQuery("SELECT 1 AS value UNION ALL SELECT 2 ORDER BY value")
+            }
+        assertEquals(listOf(1, 2), rows.map { it.jsonArray[0].jsonPrimitive.int })
+    }
+
+    @Test
+    fun fullResultDoesNotDependOnPageSizeOrLeaveCachedPages() {
+        withBackend { backend ->
+            backend.setPageSize(9)
+            try {
+                val rows =
+                    backend.fullQuery(
+                        "SELECT 1 AS value UNION ALL SELECT 2 UNION ALL SELECT 3 ORDER BY value",
+                    )
+                assertEquals(listOf(1, 2, 3), rows.map { it.jsonArray[0].jsonPrimitive.int })
+                assertTrue(backend.getActiveSequenceNumbers().isEmpty())
+            } finally {
+                // Page size is shared by all backend instances; restore the default.
+                backend.setPageSize(2 * 1024 * 1024)
+            }
+        }
+    }
+
+    @Test
+    fun databaseErrorsRemainBackendExceptions() {
+        withBackend { backend ->
+            assertThrows(BackendException::class.java) {
+                assertTrue(backend.fullQuery("SELECT * FROM missing_table").isEmpty())
+            }
+        }
+    }
+
+    private fun withBackend(block: (Backend) -> Unit) {
+        getBackend().use { backend ->
+            backend.openCollection(":memory:")
+            block(backend)
+        }
+    }
+}