diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 419fb99..2219fde 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -39,7 +39,9 @@ jobs: - name: Scan tracked files for secrets run: scripts/check-secrets.sh - name: Run tests without production credentials - run: ./gradlew test + env: + CHROME_BIN: /usr/bin/google-chrome + run: ./gradlew test :app:shared:jsBrowserTest :app:shared:wasmJsBrowserTest changes: if: github.event_name == 'push' && github.ref == 'refs/heads/main' @@ -195,6 +197,12 @@ jobs: --region "${{ env.REGION }}" \ --project "${{ env.PROJECT_ID }}" \ --allow-unauthenticated \ + --use-http2 \ + --memory 4Gi \ + --cpu 2 \ + --concurrency 2 \ + --max-instances 5 \ + --timeout 3600 \ --remove-env-vars "B2_KEY_ID,B2_APPLICATION_KEY" \ --update-env-vars "PUBLIC_BASE_URL=${{ env.PUBLIC_BASE_URL }},STRIPE_RETURN_URL=${{ env.STRIPE_RETURN_URL }},STRIPE_CHECKOUT_SUCCESS_URL=${{ env.STRIPE_CHECKOUT_SUCCESS_URL }},STRIPE_CHECKOUT_CANCEL_URL=${{ env.STRIPE_CHECKOUT_CANCEL_URL }},STRIPE_PORTAL_RETURN_URL=${{ env.STRIPE_PORTAL_RETURN_URL }},STRIPE_PRO_PRICE_ID=$STRIPE_PRO_PRICE_ID,B2_BUCKET=${{ env.B2_BUCKET }},B2_ENDPOINT=${{ env.B2_ENDPOINT }},B2_REGION=${{ env.B2_REGION }},FIREBASE_STORAGE_BUCKET=${{ env.FIREBASE_STORAGE_BUCKET }},DREBIN451_FIREBASE_WEB_API_KEY=$DREBIN451_FIREBASE_WEB_API_KEY" \ --update-secrets "FIREBASE_SERVICE_ACCOUNT_BASE64=drebin451-firebase-admin-json-base64:latest,B2_KEY_ID=drebin451-b2-key-id:latest,B2_APPLICATION_KEY=drebin451-b2-application-key:latest,STRIPE_SECRET_KEY=drebin451-stripe-secret-key:latest,STRIPE_WEBHOOK_SECRET=drebin451-stripe-webhook-secret:latest,DREBIN451_CRON_SECRET=drebin451-cron-secret:latest" @@ -227,7 +235,7 @@ jobs: run: ./gradlew :app:androidApp:assembleRelease - name: Publish APK to Drebin451 - uses: Commit451/drebin451-release@6bf98ca1786d7ed4d8ba123f6f89a297b1f86c4b # v1 + uses: Commit451/drebin451-release@6bd9c5cc88b9f2b00d9bd6e63131fbc75ce5a18f # v1.0.2 with: api-key: ${{ secrets.DREBIN_API_KEY }} apk-path: app/androidApp/build/outputs/apk/release/androidApp-release.apk diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5603000..9a8e662 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,4 +23,6 @@ jobs: - name: Scan tracked files for secrets run: scripts/check-secrets.sh - name: Run tests without production credentials - run: ./gradlew test + env: + CHROME_BIN: /usr/bin/google-chrome + run: ./gradlew test :app:shared:jsBrowserTest :app:shared:wasmJsBrowserTest diff --git a/README.md b/README.md index 5abab19..e71df01 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,10 @@ and distribute your apps while getting them ready for prime time. ## Uploading -There are a few ways you can upload your app to Drebin451: +There are a few ways you can upload your app to Drebin451. APKs up to **1 GiB** are supported; the +browser, Android app, and official release action stream the file instead of loading the entire APK +into memory. The production API uses end-to-end HTTP/2 so Cloud Run does not apply its 32 MiB +HTTP/1 request limit. ### Manual upload diff --git a/app/shared/src/androidMain/kotlin/com/commit451/drebin451/file/ApkPicker.android.kt b/app/shared/src/androidMain/kotlin/com/commit451/drebin451/file/ApkPicker.android.kt index b5e812d..bd50bf3 100644 --- a/app/shared/src/androidMain/kotlin/com/commit451/drebin451/file/ApkPicker.android.kt +++ b/app/shared/src/androidMain/kotlin/com/commit451/drebin451/file/ApkPicker.android.kt @@ -18,9 +18,15 @@ actual fun rememberApkPicker(onResult: (PickedApk?) -> Unit): () -> Unit { onResult(null) return@rememberLauncherForActivityResult } - val name = context.displayName(uri) ?: "app.apk" - val bytes = context.contentResolver.openInputStream(uri)?.use { it.readBytes() } - onResult(if (bytes == null) null else PickedApk(name, bytes)) + val metadata = context.apkMetadata(uri) + initializeApkUpload(context.applicationContext.contentResolver) + onResult( + PickedApk( + fileName = metadata.fileName, + sizeBytes = metadata.sizeBytes, + sourceId = uri.toString(), + ), + ) } return { // Some providers report APKs as octet-stream/zip, so accept those too. @@ -34,8 +40,27 @@ actual fun rememberApkPicker(onResult: (PickedApk?) -> Unit): () -> Unit { } } -private fun Context.displayName(uri: Uri): String? = - contentResolver.query(uri, null, null, null, null)?.use { cursor -> - val index = cursor.getColumnIndex(OpenableColumns.DISPLAY_NAME) - if (index >= 0 && cursor.moveToFirst()) cursor.getString(index) else null +private data class ApkMetadata(val fileName: String, val sizeBytes: Long) + +private fun Context.apkMetadata(uri: Uri): ApkMetadata { + var fileName = "app.apk" + var sizeBytes = -1L + contentResolver.query( + uri, + arrayOf(OpenableColumns.DISPLAY_NAME, OpenableColumns.SIZE), + null, + null, + null, + )?.use { cursor -> + if (cursor.moveToFirst()) { + val nameIndex = cursor.getColumnIndex(OpenableColumns.DISPLAY_NAME) + if (nameIndex >= 0) fileName = cursor.getString(nameIndex)?.ifBlank { "app.apk" } ?: "app.apk" + val sizeIndex = cursor.getColumnIndex(OpenableColumns.SIZE) + if (sizeIndex >= 0 && !cursor.isNull(sizeIndex)) sizeBytes = cursor.getLong(sizeIndex) + } + } + if (sizeBytes < 0) { + sizeBytes = contentResolver.openAssetFileDescriptor(uri, "r")?.use { it.length } ?: -1L } + return ApkMetadata(fileName, sizeBytes) +} diff --git a/app/shared/src/androidMain/kotlin/com/commit451/drebin451/file/PlatformApkUpload.android.kt b/app/shared/src/androidMain/kotlin/com/commit451/drebin451/file/PlatformApkUpload.android.kt new file mode 100644 index 0000000..8afacfb --- /dev/null +++ b/app/shared/src/androidMain/kotlin/com/commit451/drebin451/file/PlatformApkUpload.android.kt @@ -0,0 +1,101 @@ +package com.commit451.drebin451.file + +import android.content.ContentResolver +import android.net.Uri +import android.util.Base64 +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.awaitCancellation +import kotlinx.coroutines.cancelAndJoin +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.currentCoroutineContext +import kotlinx.coroutines.ensureActive +import kotlinx.coroutines.launch +import kotlinx.coroutines.withContext +import java.io.ByteArrayOutputStream +import java.io.InputStream +import java.net.HttpURLConnection +import java.net.URL + +private const val MAX_RESPONSE_BYTES = 1024 * 1024 +private lateinit var apkContentResolver: ContentResolver + +internal fun initializeApkUpload(contentResolver: ContentResolver) { + apkContentResolver = contentResolver +} + +actual suspend fun uploadPickedApk( + picked: PickedApk, + uploadUrl: String, + bearerToken: String, + retainOnUnauthorized: Boolean, +): PlatformUploadResponse = coroutineScope { + val connection = URL(uploadUrl).openConnection() as HttpURLConnection + // This sibling reacts to parent cancellation even while blocking URLConnection I/O is in progress. + val cancellationWatcher = launch(Dispatchers.IO) { + try { + awaitCancellation() + } finally { + connection.disconnect() + } + } + + try { + withContext(Dispatchers.IO) { + connection.requestMethod = "POST" + connection.doOutput = true + connection.instanceFollowRedirects = false + connection.connectTimeout = 30_000 + connection.readTimeout = 3_600_000 + if (picked.sizeBytes > 0) { + connection.setFixedLengthStreamingMode(picked.sizeBytes) + } else { + connection.setChunkedStreamingMode(256 * 1024) + } + connection.setRequestProperty("Authorization", "Bearer $bearerToken") + connection.setRequestProperty("Content-Type", "application/vnd.android.package-archive") + connection.setRequestProperty( + "X-Apk-File-Name-Base64", + Base64.encodeToString(picked.fileName.encodeToByteArray(), Base64.NO_WRAP), + ) + + connection.outputStream.buffered(256 * 1024).use { output -> + val uri = Uri.parse(picked.sourceId) + val input = apkContentResolver.openInputStream(uri) + ?: throw IllegalArgumentException("Could not open the selected APK.") + input.use { + val buffer = ByteArray(256 * 1024) + while (true) { + currentCoroutineContext().ensureActive() + val read = input.read(buffer) + if (read < 0) break + output.write(buffer, 0, read) + } + } + } + + val statusCode = connection.responseCode + val responseStream = if (statusCode >= 400) connection.errorStream else connection.inputStream + PlatformUploadResponse( + statusCode = statusCode, + body = responseStream?.use { it.readUtf8Limited(MAX_RESPONSE_BYTES) }.orEmpty(), + ) + } + } finally { + cancellationWatcher.cancelAndJoin() + connection.disconnect() + } +} + +actual suspend fun discardPickedApk(picked: PickedApk) = Unit + +private fun InputStream.readUtf8Limited(maxBytes: Int): String { + val output = ByteArrayOutputStream() + val buffer = ByteArray(8192) + while (true) { + val read = read(buffer) + if (read < 0) break + require(output.size() + read <= maxBytes) { "Drebin451 response is too large." } + output.write(buffer, 0, read) + } + return output.toByteArray().decodeToString() +} diff --git a/app/shared/src/commonMain/kotlin/com/commit451/drebin451/api/Api.kt b/app/shared/src/commonMain/kotlin/com/commit451/drebin451/api/Api.kt index 66e5ad4..adb468b 100644 --- a/app/shared/src/commonMain/kotlin/com/commit451/drebin451/api/Api.kt +++ b/app/shared/src/commonMain/kotlin/com/commit451/drebin451/api/Api.kt @@ -1,5 +1,10 @@ package com.commit451.drebin451.api +import com.commit451.drebin451.auth.firebaseIdToken +import com.commit451.drebin451.file.PickedApk +import com.commit451.drebin451.file.PlatformUploadResponse +import com.commit451.drebin451.file.discardPickedApk +import com.commit451.drebin451.file.uploadPickedApk import com.commit451.drebin451.model.ApiKey import com.commit451.drebin451.model.ApiKeyCreated import com.commit451.drebin451.model.App @@ -14,8 +19,6 @@ import com.commit451.drebin451.model.User import com.commit451.drebin451.model.VersionNote import io.ktor.client.call.body import io.ktor.client.request.delete -import io.ktor.client.request.forms.MultiPartFormDataContent -import io.ktor.client.request.forms.formData import io.ktor.client.request.get import io.ktor.client.request.parameter import io.ktor.client.request.patch @@ -24,8 +27,6 @@ import io.ktor.client.request.setBody import io.ktor.client.statement.HttpResponse import io.ktor.client.statement.bodyAsText import io.ktor.http.ContentType -import io.ktor.http.Headers -import io.ktor.http.HttpHeaders import io.ktor.http.HttpStatusCode import io.ktor.http.contentType import io.ktor.http.isSuccess @@ -41,6 +42,14 @@ internal fun httpErrorMessage(status: HttpStatusCode, body: String): String { ?: fallback } +internal fun appVersionFromUploadResponse(response: PlatformUploadResponse): AppVersion { + val status = HttpStatusCode.fromValue(response.statusCode) + if (!status.isSuccess()) { + throw HttpException(status, httpErrorMessage(status, response.body)) + } + return errorJson.decodeFromString(response.body) +} + /** * The single client entry point to the Ktor backend. The Firebase bearer token is * attached automatically by the configured [HttpClient] (see createHttpClient()). @@ -129,28 +138,39 @@ object Api { client.get("$baseUrl/apps/$appId/versions/$versionId").bodyOrThrow() /** - * Uploads an APK. The server reads its applicationId/version/label from the bytes and - * either creates a new app or appends a version to the existing one, returning the - * created [AppVersion]. + * Streams an APK from its platform file handle. The server reads its applicationId/version/label + * and either creates a new app or appends a version to the existing one. */ - suspend fun uploadApp(fileName: String, bytes: ByteArray): AppVersion { - val response = client.post("$baseUrl/apps") { - setBody( - MultiPartFormDataContent( - formData { - append( - key = "apk", - value = bytes, - headers = Headers.build { - append(HttpHeaders.ContentType, AppVersion.CONTENT_TYPE_APK) - append(HttpHeaders.ContentDisposition, "filename=\"$fileName\"") - }, - ) - }, - ), + suspend fun uploadApp(picked: PickedApk): AppVersion { + if (picked.sizeBytes == 0L || picked.sizeBytes > AppVersion.MAX_FILE_SIZE_BYTES) { + discardPickedApk(picked) + require(picked.sizeBytes != 0L) { "The APK file is empty." } + throw IllegalArgumentException("APK files may not exceed 1 GiB.") + } + + suspend fun upload(forceRefresh: Boolean): PlatformUploadResponse { + val token = try { + firebaseIdToken(forceRefresh) + } catch (t: Throwable) { + discardPickedApk(picked) + throw t + } ?: run { + discardPickedApk(picked) + throw HttpException(HttpStatusCode.Unauthorized, "Sign in before uploading an APK.") + } + return uploadPickedApk( + picked = picked, + uploadUrl = "$baseUrl/apps", + bearerToken = token, + retainOnUnauthorized = !forceRefresh, ) } - return response.bodyOrThrow() + + var response = upload(forceRefresh = false) + if (response.statusCode == HttpStatusCode.Unauthorized.value) { + response = upload(forceRefresh = true) + } + return appVersionFromUploadResponse(response) } /** Deletes an app and every version under it. */ diff --git a/app/shared/src/commonMain/kotlin/com/commit451/drebin451/file/PickedApk.kt b/app/shared/src/commonMain/kotlin/com/commit451/drebin451/file/PickedApk.kt index 0db94f4..7342cc5 100644 --- a/app/shared/src/commonMain/kotlin/com/commit451/drebin451/file/PickedApk.kt +++ b/app/shared/src/commonMain/kotlin/com/commit451/drebin451/file/PickedApk.kt @@ -1,4 +1,11 @@ package com.commit451.drebin451.file -/** A user-picked APK: its display name and raw bytes. */ -class PickedApk(val fileName: String, val bytes: ByteArray) +/** + * A user-picked APK represented by a platform file handle rather than raw bytes. Keeping the source + * handle lets Android and web stream files as large as 1 GiB without first duplicating them in RAM. + */ +class PickedApk( + val fileName: String, + val sizeBytes: Long, + internal val sourceId: String, +) diff --git a/app/shared/src/commonMain/kotlin/com/commit451/drebin451/file/PlatformApkUpload.kt b/app/shared/src/commonMain/kotlin/com/commit451/drebin451/file/PlatformApkUpload.kt new file mode 100644 index 0000000..9d965a0 --- /dev/null +++ b/app/shared/src/commonMain/kotlin/com/commit451/drebin451/file/PlatformApkUpload.kt @@ -0,0 +1,20 @@ +package com.commit451.drebin451.file + +import kotlinx.serialization.Serializable + +@Serializable +data class PlatformUploadResponse( + val statusCode: Int, + val body: String, +) + +/** Uploads the selected platform file as multipart form data without loading it all into memory. */ +expect suspend fun uploadPickedApk( + picked: PickedApk, + uploadUrl: String, + bearerToken: String, + retainOnUnauthorized: Boolean, +): PlatformUploadResponse + +/** Releases platform resources when validation fails or an upload coroutine is cancelled. */ +expect suspend fun discardPickedApk(picked: PickedApk) diff --git a/app/shared/src/commonMain/kotlin/com/commit451/drebin451/ui/AppDetailViewModel.kt b/app/shared/src/commonMain/kotlin/com/commit451/drebin451/ui/AppDetailViewModel.kt index 36a5cae..081b345 100644 --- a/app/shared/src/commonMain/kotlin/com/commit451/drebin451/ui/AppDetailViewModel.kt +++ b/app/shared/src/commonMain/kotlin/com/commit451/drebin451/ui/AppDetailViewModel.kt @@ -119,7 +119,7 @@ class AppDetailViewModel(initialApp: App) : ViewModel() { viewModelScope.launch { _state.update { it.copy(uploading = true, storageLimitUploadDialogMessage = null) } try { - val version = Api.uploadApp(fileName = picked.fileName, bytes = picked.bytes) + val version = Api.uploadApp(picked) val label = version.versionName.ifBlank { picked.fileName } _state.update { it.copy(uploading = false, message = "Uploaded $label") } load() diff --git a/app/shared/src/commonMain/kotlin/com/commit451/drebin451/ui/HomeViewModel.kt b/app/shared/src/commonMain/kotlin/com/commit451/drebin451/ui/HomeViewModel.kt index ed98e80..bc30379 100644 --- a/app/shared/src/commonMain/kotlin/com/commit451/drebin451/ui/HomeViewModel.kt +++ b/app/shared/src/commonMain/kotlin/com/commit451/drebin451/ui/HomeViewModel.kt @@ -211,7 +211,7 @@ class HomeViewModel : ViewModel() { viewModelScope.launch { _state.update { it.copy(uploading = true, storageLimitUploadDialogMessage = null) } try { - val version = Api.uploadApp(fileName = picked.fileName, bytes = picked.bytes) + val version = Api.uploadApp(picked) val label = version.versionName.ifBlank { picked.fileName } val promptApp = notificationPromptAppFor(version) _state.update { diff --git a/app/shared/src/commonTest/kotlin/com/commit451/drebin451/api/ApiErrorTest.kt b/app/shared/src/commonTest/kotlin/com/commit451/drebin451/api/ApiErrorTest.kt index 2155186..1e023cf 100644 --- a/app/shared/src/commonTest/kotlin/com/commit451/drebin451/api/ApiErrorTest.kt +++ b/app/shared/src/commonTest/kotlin/com/commit451/drebin451/api/ApiErrorTest.kt @@ -1,8 +1,10 @@ package com.commit451.drebin451.api +import com.commit451.drebin451.file.PlatformUploadResponse import io.ktor.http.HttpStatusCode import kotlin.test.Test import kotlin.test.assertEquals +import kotlin.test.assertFailsWith class ApiErrorTest { @@ -38,4 +40,33 @@ class ApiErrorTest { ), ) } + + @Test + fun uploadResponse_decodesCreatedVersion() { + val version = appVersionFromUploadResponse( + PlatformUploadResponse( + statusCode = 201, + body = "{\"id\":\"version-1\",\"fileName\":\"large.apk\",\"fileSizeBytes\":1073741824}", + ), + ) + + assertEquals("version-1", version.id) + assertEquals("large.apk", version.fileName) + assertEquals(1L shl 30, version.fileSizeBytes) + } + + @Test + fun uploadResponse_preservesStructuredApiErrors() { + val error = assertFailsWith { + appVersionFromUploadResponse( + PlatformUploadResponse( + statusCode = 402, + body = "{\"errorMessage\":\"Storage limit exceeded\"}", + ), + ) + } + + assertEquals(HttpStatusCode.PaymentRequired, error.statusCode) + assertEquals("Storage limit exceeded", error.message) + } } diff --git a/app/shared/src/jsMain/kotlin/com/commit451/drebin451/file/ApkPicker.js.kt b/app/shared/src/jsMain/kotlin/com/commit451/drebin451/file/ApkPicker.js.kt index 1f7fabf..ae3e503 100644 --- a/app/shared/src/jsMain/kotlin/com/commit451/drebin451/file/ApkPicker.js.kt +++ b/app/shared/src/jsMain/kotlin/com/commit451/drebin451/file/ApkPicker.js.kt @@ -3,10 +3,25 @@ package com.commit451.drebin451.file import kotlinx.coroutines.await import kotlin.js.Promise -/** - * `window.drebinPickApk`, installed in index.html — opens the file dialog, reads the chosen APK, - * and resolves with "\n" (or "" if the dialog was dismissed). - */ +/** Browser bridges installed by index.html for retaining and streaming the selected File object. */ private external fun drebinPickApk(): Promise +private external fun drebinUploadPickedApk( + sourceId: String, + uploadUrl: String, + bearerToken: String, + retainOnUnauthorized: Boolean, +): Promise +private external fun drebinDiscardPickedApk(sourceId: String) internal actual suspend fun pickApkRaw(): String = drebinPickApk().await() + +internal actual suspend fun uploadPickedApkRaw( + sourceId: String, + uploadUrl: String, + bearerToken: String, + retainOnUnauthorized: Boolean, +): String = drebinUploadPickedApk(sourceId, uploadUrl, bearerToken, retainOnUnauthorized).await() + +internal actual suspend fun discardPickedApkRaw(sourceId: String) { + drebinDiscardPickedApk(sourceId) +} diff --git a/app/shared/src/wasmJsMain/kotlin/com/commit451/drebin451/file/ApkPicker.wasmJs.kt b/app/shared/src/wasmJsMain/kotlin/com/commit451/drebin451/file/ApkPicker.wasmJs.kt index e6efd66..26ffa65 100644 --- a/app/shared/src/wasmJsMain/kotlin/com/commit451/drebin451/file/ApkPicker.wasmJs.kt +++ b/app/shared/src/wasmJsMain/kotlin/com/commit451/drebin451/file/ApkPicker.wasmJs.kt @@ -1,14 +1,38 @@ +@file:OptIn(kotlin.js.ExperimentalWasmJsInterop::class) + package com.commit451.drebin451.file import kotlinx.coroutines.await import kotlin.js.Promise -/** - * Bridges to `window.drebinPickApk` (index.html), which opens the file dialog and resolves with - * "\n" (or "" on cancel). Kotlin/Wasm can't reference JS globals directly, - * hence the [JsFun] shim. - */ @JsFun("() => drebinPickApk()") private external fun drebinPickApk(): Promise +@JsFun("(sourceId, uploadUrl, bearerToken, retainOnUnauthorized) => drebinUploadPickedApk(sourceId, uploadUrl, bearerToken, retainOnUnauthorized)") +private external fun drebinUploadPickedApk( + sourceId: JsString, + uploadUrl: JsString, + bearerToken: JsString, + retainOnUnauthorized: Boolean, +): Promise + +@JsFun("(sourceId) => drebinDiscardPickedApk(sourceId)") +private external fun drebinDiscardPickedApk(sourceId: JsString) + internal actual suspend fun pickApkRaw(): String = drebinPickApk().await().toString() + +internal actual suspend fun uploadPickedApkRaw( + sourceId: String, + uploadUrl: String, + bearerToken: String, + retainOnUnauthorized: Boolean, +): String = drebinUploadPickedApk( + sourceId.toJsString(), + uploadUrl.toJsString(), + bearerToken.toJsString(), + retainOnUnauthorized, +).await().toString() + +internal actual suspend fun discardPickedApkRaw(sourceId: String) { + drebinDiscardPickedApk(sourceId.toJsString()) +} diff --git a/app/shared/src/webMain/kotlin/com/commit451/drebin451/file/ApkPicker.web.kt b/app/shared/src/webMain/kotlin/com/commit451/drebin451/file/ApkPicker.web.kt index 6891897..1ed9666 100644 --- a/app/shared/src/webMain/kotlin/com/commit451/drebin451/file/ApkPicker.web.kt +++ b/app/shared/src/webMain/kotlin/com/commit451/drebin451/file/ApkPicker.web.kt @@ -3,43 +3,68 @@ package com.commit451.drebin451.file import androidx.compose.runtime.Composable import androidx.compose.runtime.rememberCoroutineScope import kotlinx.coroutines.launch -import kotlin.io.encoding.Base64 +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json + +private val pickerJson = Json { ignoreUnknownKeys = true } + +@Serializable +private data class WebPickedApk( + val token: String, + val name: String, + val size: Long, +) /** - * Web APK picker, shared across js + wasmJs. The actual file dialog and byte read live in plain JS - * (`window.drebinPickApk` in index.html) because FileReader interop differs awkwardly between the - * two web backends; only the thin token-style bridge [pickApkRaw] is platform-specific, mirroring - * how Google sign-in is wired (see fetchGoogleIdToken). The JS side hands back the filename, a - * newline, then the base64 bytes — or an empty string if the user cancelled. + * Web APK picker shared across js + wasmJs. Plain JavaScript keeps the browser File object in a + * short-lived token map; Kotlin receives metadata only, so selecting a large APK does not base64 or + * duplicate the file in memory. */ @Composable actual fun rememberApkPicker(onResult: (PickedApk?) -> Unit): () -> Unit { val scope = rememberCoroutineScope() return { scope.launch { - // A read failure (or cancel) yields null, matching Android — the caller just no-ops. val picked = runCatching { parsePickedApk(pickApkRaw()) }.getOrNull() onResult(picked) } } } -/** - * Opens the browser file picker and suspends until the user chooses a file or dismisses the dialog. - * Returns "\n", or an empty string on cancel. Implemented per web - * backend (js vs wasmJs) since the JS interop differs between Kotlin/JS and Kotlin/Wasm. - */ internal expect suspend fun pickApkRaw(): String +internal expect suspend fun uploadPickedApkRaw( + sourceId: String, + uploadUrl: String, + bearerToken: String, + retainOnUnauthorized: Boolean, +): String +internal expect suspend fun discardPickedApkRaw(sourceId: String) -/** - * Splits the "\n" payload from JS back into a [PickedApk], or null if the user - * cancelled (empty / delimiter-less string). The newline can't appear in a filename, so the first - * one cleanly separates the name from the base64 body. - */ private fun parsePickedApk(raw: String): PickedApk? { - val newline = raw.indexOf('\n') - if (newline < 0) return null - val name = raw.substring(0, newline) - val bytes = Base64.Default.decode(raw.substring(newline + 1)) - return PickedApk(name.ifBlank { "app.apk" }, bytes) + if (raw.isBlank()) return null + val metadata = pickerJson.decodeFromString(raw) + return PickedApk( + fileName = metadata.name.ifBlank { "app.apk" }, + sizeBytes = metadata.size, + sourceId = metadata.token, + ) +} + +actual suspend fun uploadPickedApk( + picked: PickedApk, + uploadUrl: String, + bearerToken: String, + retainOnUnauthorized: Boolean, +): PlatformUploadResponse = try { + pickerJson.decodeFromString( + uploadPickedApkRaw(picked.sourceId, uploadUrl, bearerToken, retainOnUnauthorized), + ) +} catch (t: Throwable) { + // Promise cancellation/network failure must abort fetch and release the retained browser File. + discardPickedApkRaw(picked.sourceId) + throw t +} + +actual suspend fun discardPickedApk(picked: PickedApk) { + discardPickedApkRaw(picked.sourceId) } diff --git a/app/webApp/src/webMain/resources/index.html b/app/webApp/src/webMain/resources/index.html index 4a64b84..1d4e974 100644 --- a/app/webApp/src/webMain/resources/index.html +++ b/app/webApp/src/webMain/resources/index.html @@ -187,44 +187,35 @@ }; })(); - // Opens the browser file dialog (filtered to .apk), reads the chosen file, and hands it back - // to the Kotlin web app (js + wasmJs) as the filename, a newline, then the base64 bytes — or - // an empty string if the dialog was dismissed. FileReader.readAsDataURL lets the browser do - // the base64 natively (fast for large APKs); the Kotlin side splits on the first newline - // (never present in a filename) and decodes. Kept in plain JS so the FileReader bits live in - // one readable place for both targets. + // Keeps the browser File object behind a short-lived token. Kotlin receives only metadata; + // the upload bridge below gives the original File directly to fetch/FormData, so even a 1 GiB + // APK is never converted to base64 or copied into the Wasm/JS heap. + window.drebinPickedApks = window.drebinPickedApks || Object.create(null); + window.drebinApkUploadControllers = window.drebinApkUploadControllers || Object.create(null); window.drebinPickApk = function () { - return new Promise(function (resolve, reject) { + return new Promise(function (resolve) { var input = document.createElement('input'); input.type = 'file'; input.accept = '.apk,application/vnd.android.package-archive'; input.style.display = 'none'; var settled = false; - function finish(fn, value) { + function finish(value) { if (settled) return; settled = true; if (input.parentNode) input.parentNode.removeChild(input); - fn(value); + resolve(value); } - // 'cancel' fires (modern browsers) when the dialog is closed with nothing chosen. - input.addEventListener('cancel', function () { finish(resolve, ''); }); + input.addEventListener('cancel', function () { finish(''); }); input.addEventListener('change', function () { var file = input.files && input.files[0]; - if (!file) { finish(resolve, ''); return; } - var reader = new FileReader(); - reader.onload = function () { - // result is a data: URL — strip the "data:;base64," prefix. - var result = String(reader.result); - var comma = result.indexOf(','); - var base64 = comma >= 0 ? result.substring(comma + 1) : ''; - finish(resolve, file.name + '\n' + base64); - }; - reader.onerror = function () { - finish(reject, reader.error || new Error("Could not read the selected file.")); - }; - reader.readAsDataURL(file); + if (!file) { finish(''); return; } + var token = window.crypto && typeof window.crypto.randomUUID === 'function' + ? window.crypto.randomUUID() + : String(Date.now()) + '-' + Math.random().toString(36).slice(2); + window.drebinPickedApks[token] = file; + finish(JSON.stringify({ token: token, name: file.name, size: file.size })); }); // Some browsers only fire 'click' reliably when the input is in the DOM. @@ -233,6 +224,50 @@ }); }; + window.drebinDiscardPickedApk = function (sourceId) { + var key = String(sourceId); + var controller = window.drebinApkUploadControllers[key]; + if (controller) controller.abort(); + delete window.drebinApkUploadControllers[key]; + delete window.drebinPickedApks[key]; + }; + + window.drebinUploadPickedApk = async function (sourceId, uploadUrl, bearerToken, retainOnUnauthorized) { + var key = String(sourceId); + var file = window.drebinPickedApks[key]; + if (!file) throw new Error('The selected APK is no longer available. Please select it again.'); + + var controller = new AbortController(); + window.drebinApkUploadControllers[key] = controller; + try { + var fileNameBytes = new TextEncoder().encode(file.name || 'app.apk'); + var fileNameBinary = ''; + for (var i = 0; i < fileNameBytes.length; i += 1) { + fileNameBinary += String.fromCharCode(fileNameBytes[i]); + } + var response = await fetch(String(uploadUrl), { + method: 'POST', + headers: { + 'Authorization': 'Bearer ' + String(bearerToken), + 'Content-Type': 'application/vnd.android.package-archive', + 'X-Apk-File-Name-Base64': btoa(fileNameBinary) + }, + body: file, + signal: controller.signal + }); + var body = await response.text(); + // Preserve the File only for the first 401 so Firebase can refresh and retry once. + if (response.status !== 401 || !retainOnUnauthorized) { + delete window.drebinPickedApks[key]; + } + return JSON.stringify({ statusCode: response.status, body: body }); + } finally { + if (window.drebinApkUploadControllers[key] === controller) { + delete window.drebinApkUploadControllers[key]; + } + } + }; + // Starts a browser download for APK bytes that the Kotlin app already fetched through the // authenticated API. Kept in plain JS so both Kotlin/JS and Kotlin/Wasm can share it. window.drebinDownloadApk = function (fileName, contentType, base64) { diff --git a/core/src/commonMain/kotlin/com/commit451/drebin451/model/AppVersion.kt b/core/src/commonMain/kotlin/com/commit451/drebin451/model/AppVersion.kt index 073213d..19274e4 100644 --- a/core/src/commonMain/kotlin/com/commit451/drebin451/model/AppVersion.kt +++ b/core/src/commonMain/kotlin/com/commit451/drebin451/model/AppVersion.kt @@ -34,5 +34,7 @@ data class AppVersion( ) { companion object { const val CONTENT_TYPE_APK = "application/vnd.android.package-archive" + /** Largest APK accepted by every Drebin451 upload client and the backend: exactly 1 GiB. */ + const val MAX_FILE_SIZE_BYTES: Long = 1L shl 30 } } diff --git a/server/src/main/kotlin/com/commit451/drebin451/Application.kt b/server/src/main/kotlin/com/commit451/drebin451/Application.kt index e60e627..8b2d0a8 100644 --- a/server/src/main/kotlin/com/commit451/drebin451/Application.kt +++ b/server/src/main/kotlin/com/commit451/drebin451/Application.kt @@ -31,12 +31,15 @@ import io.ktor.server.application.Application import io.ktor.server.application.ApplicationCall import io.ktor.server.application.install import io.ktor.server.application.log +import io.ktor.server.engine.connector import io.ktor.server.engine.embeddedServer import io.ktor.server.netty.Netty +import io.ktor.server.netty.NettyApplicationEngine import io.ktor.server.plugins.contentnegotiation.ContentNegotiation import io.ktor.server.plugins.cors.routing.CORS import io.ktor.server.plugins.origin import io.ktor.server.plugins.statuspages.StatusPages +import io.ktor.server.request.contentType import io.ktor.server.request.receive import io.ktor.server.request.receiveChannel import io.ktor.server.request.receiveMultipart @@ -51,13 +54,22 @@ import io.ktor.server.routing.patch import io.ktor.server.routing.post import io.ktor.server.routing.routing import io.ktor.utils.io.readRemaining +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.NonCancellable +import kotlinx.coroutines.withContext import kotlinx.io.readByteArray import kotlinx.serialization.json.Json +import java.nio.file.Files import java.security.MessageDigest +import java.util.Base64 import java.util.UUID /** Upper bound on the free-text note stored per upload, to keep version rows small. */ internal const val MAX_NOTE_LENGTH = 2000 +internal const val MAX_MULTIPART_PARTS = 2 +internal const val LEGACY_MULTIPART_MAX_REQUEST_BYTES = 40L * 1024 * 1024 +internal const val ApkFileNameBase64Header = "X-Apk-File-Name-Base64" +internal const val UploadNoteBase64Header = "X-Upload-Note-Base64" internal const val STORAGE_QUOTA_EXCEEDED_ERROR_MESSAGE = "Storage limit exceeded, please update your plan to continue uploading" @@ -90,6 +102,25 @@ internal fun isAuthorizedCronSecret(presented: String?, configured: String?): Bo internal fun normalizeVersionNote(note: String): String = note.trim().take(MAX_NOTE_LENGTH) +internal fun decodeUploadHeader(value: String?, maxBytes: Int): String { + if (value.isNullOrBlank()) return "" + val bytes = try { + Base64.getDecoder().decode(value) + } catch (t: IllegalArgumentException) { + throw IllegalArgumentException("Invalid base64 upload header", t) + } + require(bytes.size <= maxBytes) { "Upload header is too large" } + return bytes.toString(Charsets.UTF_8) +} + +internal fun safeUploadFileName(value: String): String = value + .substringAfterLast('/') + .substringAfterLast('\\') + .map { if (it.isISOControl()) '_' else it } + .joinToString("") + .trim() + .ifBlank { "app.apk" } + internal fun newVersionPushBody(versionName: String, versionCode: Long, note: String): String { val versionLabel = if (versionName.isNotBlank()) { "Version $versionName ($versionCode)" @@ -110,8 +141,21 @@ internal fun newVersionPushDeepLink(shareId: String, versionId: String): String fun main() { Firebasis.initialize() val port = System.getenv("PORT")?.toIntOrNull() ?: 8080 - embeddedServer(Netty, port = port, host = "0.0.0.0", module = Application::module) - .start(wait = true) + embeddedServer( + Netty, + configure = { configureCloudRunEngine(port) }, + module = Application::module, + ).start(wait = true) +} + +/** Cloud Run forwards end-to-end HTTP/2 as cleartext h2c after terminating public TLS. */ +internal fun NettyApplicationEngine.Configuration.configureCloudRunEngine(port: Int) { + connector { + host = "0.0.0.0" + this.port = port + } + enableHttp2 = true + enableH2c = true } fun Application.module() { @@ -123,6 +167,8 @@ fun Application.module() { allowHeader(HttpHeaders.Authorization) allowHeader(HttpHeaders.ContentType) allowHeader(ApiKeyHeader) + allowHeader(ApkFileNameBase64Header) + allowHeader(UploadNoteBase64Header) allowNonSimpleContentTypes = true allowMethod(HttpMethod.Options) allowMethod(HttpMethod.Get) @@ -341,113 +387,197 @@ fun Application.module() { call.respond(HttpStatusCode.NoContent) } - // Upload an APK (multipart): part "apk" = file. The applicationId, version and label - // are read from the APK itself — this creates the app on the first upload of an - // applicationId, or adds another version to the existing app otherwise. + // Upload an APK. Current clients stream a raw APK body; bounded multipart remains supported + // for older action/app versions. Metadata is read from the APK itself. post("/$prefix/apps") { // Accepts a full-access API key (CI/scripts) via X-API-Key, or a Firebase session; // either resolves to the owning user. val user = Firebasis.refreshPlanIfStale(requireUploader() ?: return@post) + val storageStatus = user.storageStatus() + val requestContentType = call.request.contentType() + val legacyMultipart = requestContentType.match(ContentType.MultiPart.FormData) + val declaredBytes = call.request.headers[HttpHeaders.ContentLength]?.toLongOrNull() + val estimatedApkBytes = if (legacyMultipart) { + val legacyRequestBytes = requireNotNull(declaredBytes) { + "Legacy multipart uploads require Content-Length." + } + require(legacyRequestBytes <= LEGACY_MULTIPART_MAX_REQUEST_BYTES) { + "Legacy multipart uploads may not exceed 40 MiB; use the raw HTTP/2 upload protocol." + } + (legacyRequestBytes - 64L * 1024).coerceAtLeast(1) + } else { + require( + requestContentType.match(ContentType.parse(AppVersion.CONTENT_TYPE_APK)) || + requestContentType.match(ContentType.Application.OctetStream), + ) { "APK uploads require content type ${AppVersion.CONTENT_TYPE_APK}." } + declaredBytes?.also { + require(it in 1..AppVersion.MAX_FILE_SIZE_BYTES) { "APK files may not exceed 1 GiB." } + } + } + if (estimatedApkBytes != null && estimatedApkBytes > storageStatus.remainingBytes) { + throw StorageQuotaExceededException( + plan = storageStatus.plan, + usedBytes = storageStatus.usedBytes, + attemptedBytes = estimatedApkBytes, + limitBytes = storageStatus.limitBytes, + ) + } - var fileName: String? = null - var bytes: ByteArray? = null - var note = "" - - call.receiveMultipart().forEachPart { part -> - when (part) { - is PartData.FileItem -> { - fileName = part.originalFileName?.takeIf { it.isNotBlank() } ?: "app.apk" - bytes = part.provider().readRemaining().readByteArray() - } - // Optional free-text annotation (CI sends the commit message); trimmed and - // capped. Any other form field is ignored. - is PartData.FormItem -> { - if (part.name == "note") note = normalizeVersionNote(part.value) + val uploadFile = Files.createTempFile("drebin451-upload-", ".apk").toFile() + try { + val (name, uploadSize, note) = if (legacyMultipart) { + var fileName: String? = null + var uploadSizeBytes: Long? = null + var legacyNote = "" + var noteSeen = false + var partCount = 0 + + call.receiveMultipart().forEachPart { part -> + try { + when (part) { + is PartData.FileItem -> { + val source = part.provider() + try { + partCount++ + require(partCount <= MAX_MULTIPART_PARTS) { "Too many multipart parts." } + require(part.name == "apk") { "Unexpected file part '${part.name}'." } + require(uploadSizeBytes == null) { "Only one 'apk' file part is allowed." } + fileName = part.originalFileName?.takeIf { it.isNotBlank() } ?: "app.apk" + uploadSizeBytes = copyUploadToFile(source, uploadFile) + } catch (t: Throwable) { + source.cancel(t) + throw t + } + } + is PartData.FormItem -> { + partCount++ + require(partCount <= MAX_MULTIPART_PARTS) { "Too many multipart parts." } + require(part.name == "note") { "Unexpected form part '${part.name}'." } + require(!noteSeen) { "Only one 'note' form part is allowed." } + require(part.value.length <= MAX_NOTE_LENGTH) { "Upload note is too long." } + noteSeen = true + legacyNote = normalizeVersionNote(part.value) + } + + else -> throw IllegalArgumentException("Unsupported multipart part.") + } + } finally { + if (part is PartData.FileItem || part is PartData.FormItem) part.release() + } } - else -> {} + Triple( + safeUploadFileName(fileName ?: "app.apk"), + uploadSizeBytes ?: throw IllegalArgumentException("Missing 'apk' file part"), + legacyNote, + ) + } else { + val rawFileName = safeUploadFileName( + decodeUploadHeader(call.request.headers[ApkFileNameBase64Header], maxBytes = 1024), + ) + val rawNote = normalizeVersionNote( + decodeUploadHeader(call.request.headers[UploadNoteBase64Header], maxBytes = 8192), + ) + Triple(rawFileName, copyUploadToFile(call.receiveChannel(), uploadFile), rawNote) } - part.release() - } + val info = withContext(Dispatchers.IO) { parseApk(uploadFile) } + + val appId = Firebasis.appDocId(user.uid, info.applicationId) + val versionId = UUID.randomUUID().toString() + val storagePath = apkStoragePath(user.uid, info.applicationId, versionId, name) + + var storageReserved = false + var iconUploaded = false + val iconStoragePath = Firebasis.iconStoragePath(user.uid, info.applicationId) + + val (version, app) = try { + Firebasis.reserveStorage(user.uid, uploadSize) + storageReserved = true + + Firebasis.uploadFile(storagePath, uploadFile, AppVersion.CONTENT_TYPE_APK) + + // Capture the launcher icon once: store it (and set the app's imageUrl) only while the + // app has no icon yet — the first upload that yields a raster one. Served publicly by + // GET /apps/{id}/icon. + val existingImageUrl = Firebasis.getApp(appId)?.imageUrl ?: "" + val imageUrl = if (existingImageUrl.isBlank() && info.icon != null) { + Firebasis.uploadBytes( + iconStoragePath, + info.icon.bytes, + info.icon.contentType, + ) + iconUploaded = true + "$publicBaseUrl/$prefix/apps/$appId/icon" + } else { + existingImageUrl + } - val data = bytes ?: throw IllegalArgumentException("Missing 'apk' file part") - val name = fileName ?: "app.apk" - val info = parseApk(data) - - val appId = Firebasis.appDocId(user.uid, info.applicationId) - val versionId = UUID.randomUUID().toString() - val storagePath = apkStoragePath(user.uid, info.applicationId, versionId, name) - val uploadSizeBytes = data.size.toLong() - - var storageReserved = false - var iconUploaded = false - val iconStoragePath = Firebasis.iconStoragePath(user.uid, info.applicationId) - - val (version, app) = try { - Firebasis.reserveStorage(user.uid, uploadSizeBytes) - storageReserved = true - - Firebasis.uploadBytes(storagePath, data, AppVersion.CONTENT_TYPE_APK) - - // Capture the launcher icon once: store it (and set the app's imageUrl) only while the - // app has no icon yet — the first upload that yields a raster one. Served publicly by - // GET /apps/{id}/icon. - val existingImageUrl = Firebasis.getApp(appId)?.imageUrl ?: "" - val imageUrl = if (existingImageUrl.isBlank() && info.icon != null) { - Firebasis.uploadBytes( - iconStoragePath, - info.icon.bytes, - info.icon.contentType, + val now = System.currentTimeMillis() + val version = AppVersion( + id = versionId, + appId = appId, + applicationId = info.applicationId, + ownerUserId = user.uid, + versionName = info.versionName, + versionCode = info.versionCode, + fileName = name, + fileSizeBytes = uploadSize, + contentType = AppVersion.CONTENT_TYPE_APK, + createdAt = now, + updatedAt = now, + note = note, + storagePath = storagePath, ) - iconUploaded = true - "$publicBaseUrl/$prefix/apps/$appId/icon" - } else { - existingImageUrl + val app = Firebasis.addVersion( + version, + appLabel = info.label, + ownerName = user.displayName, + imageUrl = imageUrl + ) + storageReserved = false + version to app + } catch (t: Throwable) { + // A disconnected request cancels the call coroutine. Cleanup must still reconcile quota + // and object storage, and each operation must run even if a previous cleanup fails. + withContext(NonCancellable) { + if (storageReserved) { + try { + Firebasis.releaseStorage(user.uid, uploadSize) + } catch (cleanupFailure: Throwable) { + call.application.log.error("Failed to release reserved upload storage", cleanupFailure) + } + } + try { + Firebasis.deleteBlob(storagePath) + } catch (cleanupFailure: Throwable) { + call.application.log.error("Failed to delete rejected APK object", cleanupFailure) + } + if (iconUploaded) { + try { + Firebasis.deleteBlob(iconStoragePath) + } catch (cleanupFailure: Throwable) { + call.application.log.error("Failed to delete rejected APK icon", cleanupFailure) + } + } + } + throw t } - val now = System.currentTimeMillis() - val version = AppVersion( - id = versionId, + // Notify devices that follow this app (subscribed to its update topic). Best-effort — + // never fails the upload. The first upload of an app has no subscribers yet, so this is + // naturally a no-op then. + Messenger.sendNewVersion( appId = appId, - applicationId = info.applicationId, - ownerUserId = user.uid, - versionName = info.versionName, - versionCode = info.versionCode, - fileName = name, - fileSizeBytes = uploadSizeBytes, - contentType = AppVersion.CONTENT_TYPE_APK, - createdAt = now, - updatedAt = now, - note = note, - storagePath = storagePath, + versionId = version.id, + title = info.label.ifBlank { info.applicationId }, + body = newVersionPushBody(version.versionName, version.versionCode, version.note), + deepLink = newVersionPushDeepLink(app.shareId, version.id), ) - val app = Firebasis.addVersion( - version, - appLabel = info.label, - ownerName = user.displayName, - imageUrl = imageUrl - ) - storageReserved = false - version to app - } catch (t: Throwable) { - if (storageReserved) Firebasis.releaseStorage(user.uid, uploadSizeBytes) - Firebasis.deleteBlob(storagePath) - if (iconUploaded) Firebasis.deleteBlob(iconStoragePath) - throw t - } - // Notify devices that follow this app (subscribed to its update topic). Best-effort — - // never fails the upload. The first upload of an app has no subscribers yet, so this is - // naturally a no-op then. - Messenger.sendNewVersion( - appId = appId, - versionId = version.id, - title = info.label.ifBlank { info.applicationId }, - body = newVersionPushBody(version.versionName, version.versionCode, version.note), - deepLink = newVersionPushDeepLink(app.shareId, version.id), - ) - - call.respond(version) + call.respond(version) + } finally { + if (!uploadFile.delete()) uploadFile.deleteOnExit() + } } // Single app listing (auth; owner or added-to-Shared only). diff --git a/server/src/main/kotlin/com/commit451/drebin451/UploadStreaming.kt b/server/src/main/kotlin/com/commit451/drebin451/UploadStreaming.kt new file mode 100644 index 0000000..4672047 --- /dev/null +++ b/server/src/main/kotlin/com/commit451/drebin451/UploadStreaming.kt @@ -0,0 +1,39 @@ +package com.commit451.drebin451 + +import com.commit451.drebin451.model.AppVersion +import io.ktor.utils.io.ByteReadChannel +import io.ktor.utils.io.copyTo +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import java.io.File +import java.io.FileOutputStream + +/** Largest APK accepted by Drebin451: exactly 1 GiB. */ +internal const val MAX_APK_UPLOAD_BYTES = AppVersion.MAX_FILE_SIZE_BYTES + +internal fun requireApkUploadSize(byteCount: Long) { + require(byteCount > 0) { "The APK file is empty." } + require(byteCount <= MAX_APK_UPLOAD_BYTES) { "APK files may not exceed 1 GiB." } +} + +/** + * Streams one multipart file part to the container's temporary filesystem. Reading one byte past + * [maxBytes] makes an oversized upload fail without buffering the request in the JVM heap. + */ +internal suspend fun copyUploadToFile( + source: ByteReadChannel, + target: File, + maxBytes: Long = MAX_APK_UPLOAD_BYTES, +): Long = withContext(Dispatchers.IO) { + require(maxBytes > 0) { "Maximum upload size must be positive." } + val copied = FileOutputStream(target, false).channel.use { output -> + source.copyTo(output, limit = maxBytes + 1) + } + require(copied > 0) { "The APK file is empty." } + if (copied > maxBytes) { + val cause = IllegalArgumentException("APK files may not exceed 1 GiB.") + source.cancel(cause) + throw cause + } + copied +} diff --git a/server/src/main/kotlin/com/commit451/drebin451/apk/ApkInfo.kt b/server/src/main/kotlin/com/commit451/drebin451/apk/ApkInfo.kt index 49c70b4..115c4bd 100644 --- a/server/src/main/kotlin/com/commit451/drebin451/apk/ApkInfo.kt +++ b/server/src/main/kotlin/com/commit451/drebin451/apk/ApkInfo.kt @@ -1,6 +1,19 @@ package com.commit451.drebin451.apk +import net.dongliu.apk.parser.AbstractApkFile +import net.dongliu.apk.parser.ApkFile import net.dongliu.apk.parser.ByteArrayApkFile +import net.dongliu.apk.parser.bean.IconPath +import java.io.ByteArrayOutputStream +import java.io.File +import java.io.InputStream +import java.util.zip.ZipFile + +private const val MAX_APK_ENTRY_COUNT = 100_000 +private const val MAX_MANIFEST_BYTES = 16L * 1024 * 1024 +private const val MAX_RESOURCE_TABLE_BYTES = 64L * 1024 * 1024 +private const val MAX_RESOURCE_ENTRY_BYTES = 16L * 1024 * 1024 +private const val MAX_RESOURCE_TOTAL_BYTES = 256L * 1024 * 1024 /** Identity + version metadata pulled from an uploaded APK's manifest/resources. */ data class ApkInfo( @@ -21,9 +34,26 @@ data class ApkInfo( * [label] is the app's display name resolved from resources; it falls back to the * [applicationId] when absent (some APKs use a non-default locale or a stripped label). */ -fun parseApk(bytes: ByteArray): ApkInfo { +fun parseApk(bytes: ByteArray): ApkInfo = parseApk( + openApk = { ByteArrayApkFile(bytes) }, + iconExtractor = ::extractInflatedIcon, +) + +/** Parses an APK from disk so large uploads never need to be duplicated in the JVM heap. */ +fun parseApk(file: File): ApkInfo { + validateApkArchive(file) + return parseApk( + openApk = { ApkFile(file) }, + iconExtractor = { apk -> extractFileIcon(file, apk.iconPaths.orEmpty()) }, + ) +} + +private inline fun parseApk( + openApk: () -> AbstractApkFile, + iconExtractor: (AbstractApkFile) -> ApkIcon?, +): ApkInfo { return try { - ByteArrayApkFile(bytes).use { apk -> + openApk().use { apk -> val meta = apk.apkMeta val applicationId = meta.packageName?.takeIf { it.isNotBlank() } ?: throw IllegalArgumentException("APK is missing an applicationId (package name)") @@ -32,7 +62,7 @@ fun parseApk(bytes: ByteArray): ApkInfo { versionName = meta.versionName ?: "", versionCode = meta.versionCode ?: 0L, label = meta.label?.takeIf { it.isNotBlank() } ?: applicationId, - icon = extractIcon(apk), + icon = iconExtractor(apk), ) } } catch (e: IllegalArgumentException) { @@ -43,6 +73,47 @@ fun parseApk(bytes: ByteArray): ApkInfo { } } +/** + * apk-parser inflates manifest/resource/icon ZIP entries into byte arrays. Bound the entries it can + * reach before invoking it so a small compressed ZIP cannot exhaust the server heap. + */ +private fun validateApkArchive(file: File) { + try { + ZipFile(file).use { zip -> + var entryCount = 0 + var resourceBytes = 0L + var hasManifest = false + val entries = zip.entries() + while (entries.hasMoreElements()) { + val entry = entries.nextElement() + if (entry.isDirectory) continue + entryCount++ + require(entryCount <= MAX_APK_ENTRY_COUNT) { "APK contains too many ZIP entries" } + val size = entry.size + require(size >= 0) { "APK contains an entry with an unknown size" } + when { + entry.name == "AndroidManifest.xml" -> { + hasManifest = true + require(size <= MAX_MANIFEST_BYTES) { "APK manifest is too large" } + } + entry.name == "resources.arsc" -> + require(size <= MAX_RESOURCE_TABLE_BYTES) { "APK resource table is too large" } + entry.name.startsWith("res/") -> { + require(size <= MAX_RESOURCE_ENTRY_BYTES) { "APK resource entry is too large" } + resourceBytes += size + require(resourceBytes <= MAX_RESOURCE_TOTAL_BYTES) { "APK resources are too large" } + } + } + } + require(hasManifest) { "APK is missing AndroidManifest.xml" } + } + } catch (e: IllegalArgumentException) { + throw e + } catch (t: Throwable) { + throw IllegalArgumentException("Not a valid APK", t) + } +} + /** Image content types we recognise from an icon's path; null for anything we shouldn't serve. */ private fun contentTypeForIcon(path: String?): String? = when (path?.substringAfterLast('.', "")?.lowercase()) { @@ -53,13 +124,47 @@ private fun contentTypeForIcon(path: String?): String? = } /** - * Picks the best raster launcher icon from an APK, or null when it has none we can serve (e.g. a - * purely adaptive/vector icon). Considers both the dedicated raster icon files and the general icon - * list, choosing the largest image — i.e. the highest-density variant. Never throws: icon - * extraction is best-effort and must not fail an otherwise-valid upload. + * File-backed uploads read only one bounded raster icon selected from manifest icon paths. This + * avoids apk-parser eagerly inflating every icon candidate from an untrusted ZIP. */ -private fun extractIcon(apk: ByteArrayApkFile): ApkIcon? = try { - (apk.iconFiles.orEmpty() + apk.allIcons.orEmpty()) +private fun extractFileIcon(file: File, iconPaths: List): ApkIcon? = try { + ZipFile(file).use { zip -> + val candidate = iconPaths.asSequence() + .distinctBy { it.path } + .take(512) + .mapNotNull { iconPath -> + val contentType = contentTypeForIcon(iconPath.path) ?: return@mapNotNull null + val entry = zip.getEntry(iconPath.path) ?: return@mapNotNull null + if (entry.isDirectory || entry.size !in 1..MAX_RESOURCE_ENTRY_BYTES) return@mapNotNull null + Triple(iconPath, entry, contentType) + } + .maxWithOrNull(compareBy> { it.first.density } + .thenBy { it.second.size }) + ?: return null + val bytes = zip.getInputStream(candidate.second).use { + it.readBytesBounded(MAX_RESOURCE_ENTRY_BYTES.toInt()) + } + ApkIcon(bytes, candidate.third) + } +} catch (t: Throwable) { + null +} + +private fun InputStream.readBytesBounded(maxBytes: Int): ByteArray { + val output = ByteArrayOutputStream() + val buffer = ByteArray(8192) + while (true) { + val read = read(buffer) + if (read < 0) break + require(output.size() + read <= maxBytes) { "APK icon expands beyond the safe limit" } + output.write(buffer, 0, read) + } + return output.toByteArray() +} + +/** Byte-array compatibility path used by existing focused parser tests. */ +private fun extractInflatedIcon(apk: AbstractApkFile): ApkIcon? = try { + apk.iconFiles.orEmpty().ifEmpty { apk.allIcons.orEmpty() } .mapNotNull { face -> val data = face.data val contentType = contentTypeForIcon(face.path) diff --git a/server/src/main/kotlin/com/commit451/drebin451/firebase/Firebasis.kt b/server/src/main/kotlin/com/commit451/drebin451/firebase/Firebasis.kt index a90d9ca..e490246 100644 --- a/server/src/main/kotlin/com/commit451/drebin451/firebase/Firebasis.kt +++ b/server/src/main/kotlin/com/commit451/drebin451/firebase/Firebasis.kt @@ -28,6 +28,7 @@ import com.google.firebase.FirebaseOptions import com.google.firebase.auth.FirebaseAuth import com.google.firebase.cloud.FirestoreClient import org.slf4j.LoggerFactory +import java.io.File import java.security.SecureRandom import java.util.Base64 @@ -931,6 +932,10 @@ object Firebasis { objectStorage.put(path, bytes, contentType) } + suspend fun uploadFile(path: String, file: File, contentType: String) { + objectStorage.put(path, file, contentType) + } + suspend fun getBlob(path: String): StoredObject? = objectStorage.get(path) diff --git a/server/src/main/kotlin/com/commit451/drebin451/storage/B2ObjectStorage.kt b/server/src/main/kotlin/com/commit451/drebin451/storage/B2ObjectStorage.kt index f42c5d8..eb49db7 100644 --- a/server/src/main/kotlin/com/commit451/drebin451/storage/B2ObjectStorage.kt +++ b/server/src/main/kotlin/com/commit451/drebin451/storage/B2ObjectStorage.kt @@ -8,9 +8,11 @@ import aws.sdk.kotlin.services.s3.model.ListObjectsV2Request import aws.sdk.kotlin.services.s3.model.NoSuchKey import aws.sdk.kotlin.services.s3.model.PutObjectRequest import aws.smithy.kotlin.runtime.content.ByteStream +import aws.smithy.kotlin.runtime.content.fromFile import aws.smithy.kotlin.runtime.content.toByteArray import aws.smithy.kotlin.runtime.net.url.Url import org.slf4j.LoggerFactory +import java.io.File /** Thin wrapper around the AWS S3 Kotlin SDK for Backblaze B2 object storage. */ class B2ObjectStorage( @@ -39,6 +41,19 @@ class B2ObjectStorage( ) } + /** Uploads from disk without materializing the complete object in the JVM heap. */ + suspend fun put(path: String, file: File, contentType: String) { + client.putObject( + PutObjectRequest { + bucket = config.bucket + key = path + this.contentType = contentType + contentLength = file.length() + body = ByteStream.fromFile(file) + }, + ) + } + suspend fun get(path: String): StoredObject? = try { client.getObject( GetObjectRequest { diff --git a/server/src/test/kotlin/com/commit451/drebin451/Http2ServerTest.kt b/server/src/test/kotlin/com/commit451/drebin451/Http2ServerTest.kt new file mode 100644 index 0000000..bb64326 --- /dev/null +++ b/server/src/test/kotlin/com/commit451/drebin451/Http2ServerTest.kt @@ -0,0 +1,20 @@ +package com.commit451.drebin451 + +import io.ktor.server.netty.NettyApplicationEngine +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertTrue + +class Http2ServerTest { + + @Test + fun `cloud run connector accepts cleartext HTTP2`() { + val configuration = NettyApplicationEngine.Configuration() + + configuration.configureCloudRunEngine(port = 9191) + + assertTrue(configuration.enableHttp2) + assertTrue(configuration.enableH2c) + assertEquals(9191, configuration.connectors.single().port) + } +} diff --git a/server/src/test/kotlin/com/commit451/drebin451/UploadStreamingTest.kt b/server/src/test/kotlin/com/commit451/drebin451/UploadStreamingTest.kt new file mode 100644 index 0000000..cbd753a --- /dev/null +++ b/server/src/test/kotlin/com/commit451/drebin451/UploadStreamingTest.kt @@ -0,0 +1,62 @@ +package com.commit451.drebin451 + +import io.ktor.utils.io.ByteReadChannel +import kotlinx.coroutines.runBlocking +import java.io.File +import kotlin.test.Test +import kotlin.test.assertContentEquals +import kotlin.test.assertEquals +import kotlin.test.assertFailsWith + +class UploadStreamingTest { + + @Test + fun `maximum APK upload size is one gibibyte`() { + assertEquals(1L shl 30, MAX_APK_UPLOAD_BYTES) + } + + @Test + fun `upload size accepts one gibibyte and rejects anything larger`() { + requireApkUploadSize(MAX_APK_UPLOAD_BYTES) + + assertFailsWith { + requireApkUploadSize(MAX_APK_UPLOAD_BYTES + 1) + } + } + + @Test + fun `upload size rejects an empty file`() { + assertFailsWith { requireApkUploadSize(0) } + } + + @Test + fun `upload channel is copied to disk at the configured boundary`() = runBlocking { + val bytes = "12345678".encodeToByteArray() + val file = File.createTempFile("drebin451-upload-test-", ".apk") + try { + val copied = copyUploadToFile(ByteReadChannel(bytes), file, maxBytes = bytes.size.toLong()) + + assertEquals(bytes.size.toLong(), copied) + assertContentEquals(bytes, file.readBytes()) + } finally { + file.delete() + } + } + + @Test + fun `upload channel stops after one byte beyond the configured boundary`() = runBlocking { + val file = File.createTempFile("drebin451-upload-test-", ".apk") + try { + assertFailsWith { + copyUploadToFile( + ByteReadChannel("123456789".encodeToByteArray()), + file, + maxBytes = 8, + ) + } + assertEquals(9, file.length()) + } finally { + file.delete() + } + } +} diff --git a/server/src/test/kotlin/com/commit451/drebin451/VersionNoteTest.kt b/server/src/test/kotlin/com/commit451/drebin451/VersionNoteTest.kt index 404f2eb..6a70616 100644 --- a/server/src/test/kotlin/com/commit451/drebin451/VersionNoteTest.kt +++ b/server/src/test/kotlin/com/commit451/drebin451/VersionNoteTest.kt @@ -1,6 +1,7 @@ package com.commit451.drebin451 import com.commit451.drebin451.push.PushData +import java.util.Base64 import kotlin.test.Test import kotlin.test.assertEquals @@ -23,6 +24,19 @@ class VersionNoteTest { assertEquals("", normalizeVersionNote(" ")) } + @Test + fun `raw upload headers preserve utf8 metadata`() { + val value = "Unicode 🚀 release" + val encoded = Base64.getEncoder().encodeToString(value.encodeToByteArray()) + + assertEquals(value, decodeUploadHeader(encoded, maxBytes = 1024)) + } + + @Test + fun `raw upload filenames discard path and control characters`() { + assertEquals("release_.apk", safeUploadFileName("../../folder/release\n.apk")) + } + @Test fun `new version push body includes note when present`() { assertEquals( diff --git a/server/src/test/kotlin/com/commit451/drebin451/apk/ApkInfoTest.kt b/server/src/test/kotlin/com/commit451/drebin451/apk/ApkInfoTest.kt index 82940b2..c4fac1f 100644 --- a/server/src/test/kotlin/com/commit451/drebin451/apk/ApkInfoTest.kt +++ b/server/src/test/kotlin/com/commit451/drebin451/apk/ApkInfoTest.kt @@ -1,9 +1,11 @@ package com.commit451.drebin451.apk import java.io.ByteArrayOutputStream +import java.io.File import java.util.zip.ZipEntry import java.util.zip.ZipOutputStream import kotlin.test.Test +import kotlin.test.assertContains import kotlin.test.assertFailsWith class ApkInfoTest { @@ -16,6 +18,18 @@ class ApkInfoTest { } } + @Test + fun `rejects a file that is not an apk without loading it into a byte array`() { + val file = File.createTempFile("drebin451-invalid-", ".apk") + try { + file.writeText("definitely not an apk") + + assertFailsWith { parseApk(file) } + } finally { + file.delete() + } + } + @Test fun `rejects a zip without an android manifest`() { val zip = ByteArrayOutputStream().use { bos -> @@ -28,4 +42,26 @@ class ApkInfoTest { } assertFailsWith { parseApk(zip) } } + + @Test + fun `rejects a compressed apk resource that would inflate beyond the parser limit`() { + val file = File.createTempFile("drebin451-zip-bomb-", ".apk") + try { + ZipOutputStream(file.outputStream().buffered()).use { zip -> + zip.putNextEntry(ZipEntry("AndroidManifest.xml")) + zip.write("placeholder".encodeToByteArray()) + zip.closeEntry() + zip.putNextEntry(ZipEntry("res/drawable/oversized.png")) + val block = ByteArray(64 * 1024) + repeat(256) { zip.write(block) } + zip.write(0) + zip.closeEntry() + } + + val error = assertFailsWith { parseApk(file) } + assertContains(error.message.orEmpty(), "resource entry is too large") + } finally { + file.delete() + } + } }