diff --git a/gradle.properties b/gradle.properties index ed2049323..ab7fc150b 100644 --- a/gradle.properties +++ b/gradle.properties @@ -19,7 +19,7 @@ android.useAndroidX=true android.enableJetifier=false GROUP=io.github.david-allison -VERSION_NAME=0.1.70-anki26.09.3 +VERSION_NAME=0.1.71-anki26.09.3 POM_INCEPTION_YEAR=2020 diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index fa9f6a0d4..85a87027c 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -28,6 +28,8 @@ gradleMavenPublishPlugin = "0.37.0" junit = "6.1.3" # https://github.com/JetBrains/kotlin/releases/ kotlin = '2.3.21' +# https://github.com/Kotlin/kotlinx.serialization/releases +kotlinxSerialization = "1.9.0" # https://github.com/protocolbuffers/protobuf/releases protobuf = "4.36.2" robolectric = "4.17" @@ -45,6 +47,7 @@ androidx-core = { module = "androidx.test:core", version.ref = "androidxCore" } androidx-sqlite-framework = { module = "androidx.sqlite:sqlite-framework", version.ref = "androidxSqlite" } androidx-sqlite-ktx = { module = "androidx.sqlite:sqlite-ktx", version.ref = "androidxSqlite" } kotlin-stdlib-jdk8 = { module = "org.jetbrains.kotlin:kotlin-stdlib-jdk8", version.ref = "kotlin" } +kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "kotlinxSerialization" } protobuf-kotlin-lite = { module = "com.google.protobuf:protobuf-kotlin-lite", version.ref = "protobuf" } slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" } diff --git a/rsdroid-instrumented/src/androidTest/java/net/ankiweb/rsdroid/BackendIntegrationTests.kt b/rsdroid-instrumented/src/androidTest/java/net/ankiweb/rsdroid/BackendIntegrationTests.kt index 47c954151..9a4567205 100644 --- a/rsdroid-instrumented/src/androidTest/java/net/ankiweb/rsdroid/BackendIntegrationTests.kt +++ b/rsdroid-instrumented/src/androidTest/java/net/ankiweb/rsdroid/BackendIntegrationTests.kt @@ -17,6 +17,7 @@ package net.ankiweb.rsdroid import androidx.test.ext.junit.runners.AndroidJUnit4 import anki.ankidroid.SchedTimingTodayLegacyRequest +import kotlinx.serialization.json.jsonArray import net.ankiweb.rsdroid.ankiutil.InstrumentedTest import org.hamcrest.MatcherAssert.assertThat import org.hamcrest.core.Is @@ -59,7 +60,9 @@ class BackendIntegrationTests : InstrumentedTest() { @Test fun fullQueryTest() { val backendV1 = getBackend("initial_version_2_12_1.anki2") - backendV1.fullQuery("select * from col") + val result = backendV1.fullQuery("select * from col") + Assert.assertEquals(1, result.size) + Assert.assertEquals(13, result[0].jsonArray.size) } @Test diff --git a/rsdroid/build.gradle b/rsdroid/build.gradle index 85fb21a6d..791b74564 100644 --- a/rsdroid/build.gradle +++ b/rsdroid/build.gradle @@ -93,6 +93,8 @@ afterEvaluate { dependencies { // Protobuf is part of the ABI, so include it as a compile/api dependency. api libs.protobuf.kotlin.lite + // JsonArray is returned by the public fullQuery API. + api libs.kotlinx.serialization.json implementation libs.androidx.annotation implementation libs.androidx.sqlite.ktx diff --git a/rsdroid/src/main/java/net/ankiweb/rsdroid/Backend.kt b/rsdroid/src/main/java/net/ankiweb/rsdroid/Backend.kt index 947a460ef..6deead738 100644 --- a/rsdroid/src/main/java/net/ankiweb/rsdroid/Backend.kt +++ b/rsdroid/src/main/java/net/ankiweb/rsdroid/Backend.kt @@ -24,10 +24,12 @@ import anki.backend.GeneratedBackend import anki.generic.Int64 import com.google.protobuf.ByteString import com.google.protobuf.InvalidProtocolBufferException +import kotlinx.serialization.json.Json +import kotlinx.serialization.json.JsonArray +import kotlinx.serialization.json.jsonArray import net.ankiweb.rsdroid.database.NotImplementedException import net.ankiweb.rsdroid.database.SQLHandler import org.json.JSONArray -import org.json.JSONException import org.json.JSONObject import org.slf4j.LoggerFactory import java.io.Closeable @@ -155,20 +157,9 @@ open class Backend( override fun fullQuery( query: String, bindArgs: Array?, - ): JSONArray = - try { - fullQueryInternal(query, bindArgs ?: emptyArray()) - } catch (e: JSONException) { - throw RuntimeException(e) - } - - @Throws(JSONException::class) - private fun fullQueryInternal( - sql: String, - bindArgs: Array, - ): JSONArray { - val output = runDbCommand(dbRequestJson(sql, bindArgs)).toStringUtf8() - return JSONArray(output) + ): JsonArray { + val output = runDbCommand(dbRequestJson(query, bindArgs ?: emptyArray())).toStringUtf8() + return Json.parseToJsonElement(output).jsonArray } override fun insertForId( diff --git a/rsdroid/src/main/java/net/ankiweb/rsdroid/database/SQLHandler.kt b/rsdroid/src/main/java/net/ankiweb/rsdroid/database/SQLHandler.kt index c01e26bbd..bf717632a 100644 --- a/rsdroid/src/main/java/net/ankiweb/rsdroid/database/SQLHandler.kt +++ b/rsdroid/src/main/java/net/ankiweb/rsdroid/database/SQLHandler.kt @@ -17,16 +17,25 @@ package net.ankiweb.rsdroid.database import androidx.annotation.CheckResult import anki.ankidroid.DbResponse -import org.json.JSONArray +import kotlinx.serialization.json.JsonArray interface SQLHandler { + /** + * Executes [query] and returns all rows as JSON arrays of column values. + * + * The result is fully materialized; no paging or query-cache cleanup is + * required. Pass `null` for [bindArgs] when there are no bind arguments. + * + * For large results, use [fullQueryProto] to retrieve the result in pages. + */ @CheckResult fun fullQuery( query: String, bindArgs: Array?, - ): JSONArray + ): JsonArray - fun fullQuery(query: String): JSONArray = fullQuery(query, null) + @CheckResult + fun fullQuery(query: String): JsonArray = fullQuery(query, null) fun executeGetRowsAffected( sql: String, diff --git a/rsdroid/src/test/java/net/ankiweb/BackendCloseRaceTest.kt b/rsdroid/src/test/java/net/ankiweb/BackendCloseRaceTest.kt index f7be1f1c4..769eeab4e 100644 --- a/rsdroid/src/test/java/net/ankiweb/BackendCloseRaceTest.kt +++ b/rsdroid/src/test/java/net/ankiweb/BackendCloseRaceTest.kt @@ -1,12 +1,16 @@ // SPDX-License-Identifier: GPL-3.0-or-later package net.ankiweb -import android.annotation.SuppressLint import androidx.test.ext.junit.runners.AndroidJUnit4 +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonPrimitive +import kotlinx.serialization.json.long import net.ankiweb.rsdroid.Backend import net.ankiweb.rsdroid.BackendException import net.ankiweb.rsdroid.BackendFactory.getBackend import net.ankiweb.rsdroid.testing.RustBackendLoader.ensureSetup +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse import org.junit.Assert.assertTrue import org.junit.Before import org.junit.Test @@ -34,15 +38,17 @@ class BackendCloseRaceTest { fun closeDoesNotInterruptInFlightCalls() { val backend = getBackend() backend.openCollection(":memory:") - @SuppressLint("CheckResult") - backend.fullQuery(longQuery(rows = 1_000), null) // warm up the query path + val warmup = backend.fullQuery(longQuery(rows = 1_000)) + assertEquals(1_000L, warmup[0].jsonArray[0].jsonPrimitive.long) var queryError: Exception? = null + var queryCount: Long? = null val queryThread = thread(name = "backend-slow-query") { try { // keeps the backend busy inside a single native call for over a second - backend.fullQuery(longQuery(rows = 50_000_000), null) + val result = backend.fullQuery(longQuery(rows = 50_000_000)) + queryCount = result[0].jsonArray[0].jsonPrimitive.long } catch (e: Exception) { queryError = e } @@ -51,12 +57,16 @@ class BackendCloseRaceTest { sleep(500.milliseconds) // let the query enter native code backend.close() queryThread.join(1.minutes) + assertFalse("query thread did not finish", queryThread.isAlive) // Acceptable outcomes: // * close() waited for the in-flight call: the query succeeds. // * the call lost the race and was cleanly rejected. if (queryError != null) { assertTrue("unexpected query error: $queryError", queryError is BackendException) + assertEquals("Backend has been closed", queryError.message) + } else { + assertEquals(50_000_000L, queryCount) } } diff --git a/rsdroid/src/test/java/net/ankiweb/FullQueryTest.kt b/rsdroid/src/test/java/net/ankiweb/FullQueryTest.kt new file mode 100644 index 000000000..a1cd9df5d --- /dev/null +++ b/rsdroid/src/test/java/net/ankiweb/FullQueryTest.kt @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: GPL-3.0-or-later +package net.ankiweb + +import androidx.test.ext.junit.runners.AndroidJUnit4 +import kotlinx.serialization.json.JsonNull +import kotlinx.serialization.json.double +import kotlinx.serialization.json.int +import kotlinx.serialization.json.jsonArray +import kotlinx.serialization.json.jsonPrimitive +import kotlinx.serialization.json.long +import net.ankiweb.rsdroid.Backend +import net.ankiweb.rsdroid.BackendException +import net.ankiweb.rsdroid.BackendFactory.getBackend +import net.ankiweb.rsdroid.database.SQLHandler +import net.ankiweb.rsdroid.testing.RustBackendLoader.ensureSetup +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertThrows +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Test +import org.junit.runner.RunWith + +@RunWith(AndroidJUnit4::class) +class FullQueryTest { + @Before + fun loadLibrary() { + ensureSetup() + } + + @Test + fun sqlValuesAreReturnedWithoutLosingPrecision() { + withBackend { backend -> + val text = "quote\" slash/line\n🎴" + val rows = + backend.fullQuery( + "SELECT ?, ?, ?, ?, ?, x'0080ff'", + arrayOf(Long.MIN_VALUE, Long.MAX_VALUE, 1.25, text, null), + ) + assertEquals(1, rows.size) + val row = rows[0].jsonArray + assertEquals(6, row.size) + // Numeric accessors also accept quoted numbers, so check JSON types separately. + assertFalse(row[0].jsonPrimitive.isString) + assertFalse(row[1].jsonPrimitive.isString) + assertFalse(row[2].jsonPrimitive.isString) + assertTrue(row[3].jsonPrimitive.isString) + assertEquals(Long.MIN_VALUE, row[0].jsonPrimitive.long) + assertEquals(Long.MAX_VALUE, row[1].jsonPrimitive.long) + assertEquals(1.25, row[2].jsonPrimitive.double, 0.0) + assertEquals(text, row[3].jsonPrimitive.content) + assertEquals(JsonNull, row[4]) + assertEquals(listOf(0, 128, 255), row[5].jsonArray.map { it.jsonPrimitive.int }) + assertTrue(row[5].jsonArray.all { !it.jsonPrimitive.isString }) + } + } + + @Test + fun noArgumentOverloadsAndEmptyResultsWork() { + withBackend { backend -> + val database: SQLHandler = backend + val expected = backend.fullQuery("SELECT 42", emptyArray()) + assertEquals(1, expected.size) + assertEquals(42, expected[0].jsonArray[0].jsonPrimitive.int) + assertEquals(expected, database.fullQuery("SELECT 42")) + assertEquals(expected, database.fullQuery("SELECT 42", null)) + assertTrue(database.fullQuery("SELECT 42 WHERE 0").isEmpty()) + } + } + + @Test + fun resultsRemainReadableAfterClosingBackend() { + val rows = + getBackend().use { backend -> + backend.openCollection(":memory:") + backend.fullQuery("SELECT 1 AS value UNION ALL SELECT 2 ORDER BY value") + } + assertEquals(listOf(1, 2), rows.map { it.jsonArray[0].jsonPrimitive.int }) + } + + @Test + fun fullResultDoesNotDependOnPageSizeOrLeaveCachedPages() { + withBackend { backend -> + backend.setPageSize(9) + try { + val rows = + backend.fullQuery( + "SELECT 1 AS value UNION ALL SELECT 2 UNION ALL SELECT 3 ORDER BY value", + ) + assertEquals(listOf(1, 2, 3), rows.map { it.jsonArray[0].jsonPrimitive.int }) + assertTrue(backend.getActiveSequenceNumbers().isEmpty()) + } finally { + // Page size is shared by all backend instances; restore the default. + backend.setPageSize(2 * 1024 * 1024) + } + } + } + + @Test + fun databaseErrorsRemainBackendExceptions() { + withBackend { backend -> + assertThrows(BackendException::class.java) { + assertTrue(backend.fullQuery("SELECT * FROM missing_table").isEmpty()) + } + } + } + + private fun withBackend(block: (Backend) -> Unit) { + getBackend().use { backend -> + backend.openCollection(":memory:") + block(backend) + } + } +}