Support streamed APK uploads up to 1 GiB

.github/workflows/deploy.yml

@@ -39,7 +39,9 @@ jobs:
       - name: Scan tracked files for secrets
         run: scripts/check-secrets.sh
       - name: Run tests without production credentials
-        run: ./gradlew test
+        env:
+          CHROME_BIN: /usr/bin/google-chrome
+        run: ./gradlew test :app:shared:jsBrowserTest :app:shared:wasmJsBrowserTest
 
   changes:
     if: github.event_name == 'push' && github.ref == 'refs/heads/main'
@@ -195,6 +197,12 @@ jobs:
             --region "${{ env.REGION }}" \
             --project "${{ env.PROJECT_ID }}" \
             --allow-unauthenticated \
+            --use-http2 \
+            --memory 4Gi \
+            --cpu 2 \
+            --concurrency 2 \
+            --max-instances 5 \
+            --timeout 3600 \
             --remove-env-vars "B2_KEY_ID,B2_APPLICATION_KEY" \
             --update-env-vars "PUBLIC_BASE_URL=${{ env.PUBLIC_BASE_URL }},STRIPE_RETURN_URL=${{ env.STRIPE_RETURN_URL }},STRIPE_CHECKOUT_SUCCESS_URL=${{ env.STRIPE_CHECKOUT_SUCCESS_URL }},STRIPE_CHECKOUT_CANCEL_URL=${{ env.STRIPE_CHECKOUT_CANCEL_URL }},STRIPE_PORTAL_RETURN_URL=${{ env.STRIPE_PORTAL_RETURN_URL }},STRIPE_PRO_PRICE_ID=$STRIPE_PRO_PRICE_ID,B2_BUCKET=${{ env.B2_BUCKET }},B2_ENDPOINT=${{ env.B2_ENDPOINT }},B2_REGION=${{ env.B2_REGION }},FIREBASE_STORAGE_BUCKET=${{ env.FIREBASE_STORAGE_BUCKET }},DREBIN451_FIREBASE_WEB_API_KEY=$DREBIN451_FIREBASE_WEB_API_KEY" \
             --update-secrets "FIREBASE_SERVICE_ACCOUNT_BASE64=drebin451-firebase-admin-json-base64:latest,B2_KEY_ID=drebin451-b2-key-id:latest,B2_APPLICATION_KEY=drebin451-b2-application-key:latest,STRIPE_SECRET_KEY=drebin451-stripe-secret-key:latest,STRIPE_WEBHOOK_SECRET=drebin451-stripe-webhook-secret:latest,DREBIN451_CRON_SECRET=drebin451-cron-secret:latest"
@@ -227,7 +235,7 @@ jobs:
         run: ./gradlew :app:androidApp:assembleRelease
 
       - name: Publish APK to Drebin451
-        uses: Commit451/drebin451-release@6bf98ca1786d7ed4d8ba123f6f89a297b1f86c4b # v1
+        uses: Commit451/drebin451-release@6bd9c5cc88b9f2b00d9bd6e63131fbc75ce5a18f # v1.0.2
         with:
           api-key: ${{ secrets.DREBIN_API_KEY }}
           apk-path: app/androidApp/build/outputs/apk/release/androidApp-release.apk

.github/workflows/test.yml

@@ -23,4 +23,6 @@ jobs:
       - name: Scan tracked files for secrets
         run: scripts/check-secrets.sh
       - name: Run tests without production credentials
-        run: ./gradlew test
+        env:
+          CHROME_BIN: /usr/bin/google-chrome
+        run: ./gradlew test :app:shared:jsBrowserTest :app:shared:wasmJsBrowserTest

README.md

@@ -8,7 +8,10 @@ and distribute your apps while getting them ready for prime time.
 
 ## Uploading
 
-There are a few ways you can upload your app to Drebin451:
+There are a few ways you can upload your app to Drebin451. APKs up to **1 GiB** are supported; the
+browser, Android app, and official release action stream the file instead of loading the entire APK
+into memory. The production API uses end-to-end HTTP/2 so Cloud Run does not apply its 32 MiB
+HTTP/1 request limit.
 
 ### Manual upload
 

app/shared/src/androidMain/kotlin/com/commit451/drebin451/file/ApkPicker.android.kt

@@ -18,9 +18,15 @@ actual fun rememberApkPicker(onResult: (PickedApk?) -> Unit): () -> Unit {
             onResult(null)
             return@rememberLauncherForActivityResult
         }
-        val name = context.displayName(uri) ?: "app.apk"
-        val bytes = context.contentResolver.openInputStream(uri)?.use { it.readBytes() }
-        onResult(if (bytes == null) null else PickedApk(name, bytes))
+        val metadata = context.apkMetadata(uri)
+        initializeApkUpload(context.applicationContext.contentResolver)
+        onResult(
+            PickedApk(
+                fileName = metadata.fileName,
+                sizeBytes = metadata.sizeBytes,
+                sourceId = uri.toString(),
+            ),
+        )
     }
     return {
         // Some providers report APKs as octet-stream/zip, so accept those too.
@@ -34,8 +40,27 @@ actual fun rememberApkPicker(onResult: (PickedApk?) -> Unit): () -> Unit {
     }
 }
 
-private fun Context.displayName(uri: Uri): String? =
-    contentResolver.query(uri, null, null, null, null)?.use { cursor ->
-        val index = cursor.getColumnIndex(OpenableColumns.DISPLAY_NAME)
-        if (index >= 0 && cursor.moveToFirst()) cursor.getString(index) else null
+private data class ApkMetadata(val fileName: String, val sizeBytes: Long)
+
+private fun Context.apkMetadata(uri: Uri): ApkMetadata {
+    var fileName = "app.apk"
+    var sizeBytes = -1L
+    contentResolver.query(
+        uri,
+        arrayOf(OpenableColumns.DISPLAY_NAME, OpenableColumns.SIZE),
+        null,
+        null,
+        null,
+    )?.use { cursor ->
+        if (cursor.moveToFirst()) {
+            val nameIndex = cursor.getColumnIndex(OpenableColumns.DISPLAY_NAME)
+            if (nameIndex >= 0) fileName = cursor.getString(nameIndex)?.ifBlank { "app.apk" } ?: "app.apk"
+            val sizeIndex = cursor.getColumnIndex(OpenableColumns.SIZE)
+            if (sizeIndex >= 0 && !cursor.isNull(sizeIndex)) sizeBytes = cursor.getLong(sizeIndex)
+        }
+    }
+    if (sizeBytes < 0) {
+        sizeBytes = contentResolver.openAssetFileDescriptor(uri, "r")?.use { it.length } ?: -1L
     }
+    return ApkMetadata(fileName, sizeBytes)
+}

app/shared/src/androidMain/kotlin/com/commit451/drebin451/file/PlatformApkUpload.android.kt

@@ -0,0 +1,101 @@
+package com.commit451.drebin451.file
+
+import android.content.ContentResolver
+import android.net.Uri
+import android.util.Base64
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.awaitCancellation
+import kotlinx.coroutines.cancelAndJoin
+import kotlinx.coroutines.coroutineScope
+import kotlinx.coroutines.currentCoroutineContext
+import kotlinx.coroutines.ensureActive
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.withContext
+import java.io.ByteArrayOutputStream
+import java.io.InputStream
+import java.net.HttpURLConnection
+import java.net.URL
+
+private const val MAX_RESPONSE_BYTES = 1024 * 1024
+private lateinit var apkContentResolver: ContentResolver
+
+internal fun initializeApkUpload(contentResolver: ContentResolver) {
+    apkContentResolver = contentResolver
+}
+
+actual suspend fun uploadPickedApk(
+    picked: PickedApk,
+    uploadUrl: String,
+    bearerToken: String,
+    retainOnUnauthorized: Boolean,
+): PlatformUploadResponse = coroutineScope {
+    val connection = URL(uploadUrl).openConnection() as HttpURLConnection
+    // This sibling reacts to parent cancellation even while blocking URLConnection I/O is in progress.
+    val cancellationWatcher = launch(Dispatchers.IO) {
+        try {
+            awaitCancellation()
+        } finally {
+            connection.disconnect()
+        }
+    }
+
+    try {
+        withContext(Dispatchers.IO) {
+            connection.requestMethod = "POST"
+            connection.doOutput = true
+            connection.instanceFollowRedirects = false
+            connection.connectTimeout = 30_000
+            connection.readTimeout = 3_600_000
+            if (picked.sizeBytes > 0) {
+                connection.setFixedLengthStreamingMode(picked.sizeBytes)
+            } else {
+                connection.setChunkedStreamingMode(256 * 1024)
+            }
+            connection.setRequestProperty("Authorization", "Bearer $bearerToken")
+            connection.setRequestProperty("Content-Type", "application/vnd.android.package-archive")
+            connection.setRequestProperty(
+                "X-Apk-File-Name-Base64",
+                Base64.encodeToString(picked.fileName.encodeToByteArray(), Base64.NO_WRAP),
+            )
+
+            connection.outputStream.buffered(256 * 1024).use { output ->
+                val uri = Uri.parse(picked.sourceId)
+                val input = apkContentResolver.openInputStream(uri)
+                    ?: throw IllegalArgumentException("Could not open the selected APK.")
+                input.use {
+                    val buffer = ByteArray(256 * 1024)
+                    while (true) {
+                        currentCoroutineContext().ensureActive()
+                        val read = input.read(buffer)
+                        if (read < 0) break
+                        output.write(buffer, 0, read)
+                    }
+                }
+            }
+
+            val statusCode = connection.responseCode
+            val responseStream = if (statusCode >= 400) connection.errorStream else connection.inputStream
+            PlatformUploadResponse(
+                statusCode = statusCode,
+                body = responseStream?.use { it.readUtf8Limited(MAX_RESPONSE_BYTES) }.orEmpty(),
+            )
+        }
+    } finally {
+        cancellationWatcher.cancelAndJoin()
+        connection.disconnect()
+    }
+}
+
+actual suspend fun discardPickedApk(picked: PickedApk) = Unit
+
+private fun InputStream.readUtf8Limited(maxBytes: Int): String {
+    val output = ByteArrayOutputStream()
+    val buffer = ByteArray(8192)
+    while (true) {
+        val read = read(buffer)
+        if (read < 0) break
+        require(output.size() + read <= maxBytes) { "Drebin451 response is too large." }
+        output.write(buffer, 0, read)
+    }
+    return output.toByteArray().decodeToString()
+}

app/shared/src/commonMain/kotlin/com/commit451/drebin451/api/Api.kt

@@ -1,5 +1,10 @@
 package com.commit451.drebin451.api
 
+import com.commit451.drebin451.auth.firebaseIdToken
+import com.commit451.drebin451.file.PickedApk
+import com.commit451.drebin451.file.PlatformUploadResponse
+import com.commit451.drebin451.file.discardPickedApk
+import com.commit451.drebin451.file.uploadPickedApk
 import com.commit451.drebin451.model.ApiKey
 import com.commit451.drebin451.model.ApiKeyCreated
 import com.commit451.drebin451.model.App
@@ -14,8 +19,6 @@ import com.commit451.drebin451.model.User
 import com.commit451.drebin451.model.VersionNote
 import io.ktor.client.call.body
 import io.ktor.client.request.delete
-import io.ktor.client.request.forms.MultiPartFormDataContent
-import io.ktor.client.request.forms.formData
 import io.ktor.client.request.get
 import io.ktor.client.request.parameter
 import io.ktor.client.request.patch
@@ -24,8 +27,6 @@ import io.ktor.client.request.setBody
 import io.ktor.client.statement.HttpResponse
 import io.ktor.client.statement.bodyAsText
 import io.ktor.http.ContentType
-import io.ktor.http.Headers
-import io.ktor.http.HttpHeaders
 import io.ktor.http.HttpStatusCode
 import io.ktor.http.contentType
 import io.ktor.http.isSuccess
@@ -41,6 +42,14 @@ internal fun httpErrorMessage(status: HttpStatusCode, body: String): String {
         ?: fallback
 }
 
+internal fun appVersionFromUploadResponse(response: PlatformUploadResponse): AppVersion {
+    val status = HttpStatusCode.fromValue(response.statusCode)
+    if (!status.isSuccess()) {
+        throw HttpException(status, httpErrorMessage(status, response.body))
+    }
+    return errorJson.decodeFromString<AppVersion>(response.body)
+}
+
 /**
  * The single client entry point to the Ktor backend. The Firebase bearer token is
  * attached automatically by the configured [HttpClient] (see createHttpClient()).
@@ -129,28 +138,39 @@ object Api {
         client.get("$baseUrl/apps/$appId/versions/$versionId").bodyOrThrow()
 
     /**
-     * Uploads an APK. The server reads its applicationId/version/label from the bytes and
-     * either creates a new app or appends a version to the existing one, returning the
-     * created [AppVersion].
+     * Streams an APK from its platform file handle. The server reads its applicationId/version/label
+     * and either creates a new app or appends a version to the existing one.
      */
-    suspend fun uploadApp(fileName: String, bytes: ByteArray): AppVersion {
-        val response = client.post("$baseUrl/apps") {
-            setBody(
-                MultiPartFormDataContent(
-                    formData {
-                        append(
-                            key = "apk",
-                            value = bytes,
-                            headers = Headers.build {
-                                append(HttpHeaders.ContentType, AppVersion.CONTENT_TYPE_APK)
-                                append(HttpHeaders.ContentDisposition, "filename=\"$fileName\"")
-                            },
-                        )
-                    },
-                ),
+    suspend fun uploadApp(picked: PickedApk): AppVersion {
+        if (picked.sizeBytes == 0L || picked.sizeBytes > AppVersion.MAX_FILE_SIZE_BYTES) {
+            discardPickedApk(picked)
+            require(picked.sizeBytes != 0L) { "The APK file is empty." }
+            throw IllegalArgumentException("APK files may not exceed 1 GiB.")
+        }
+
+        suspend fun upload(forceRefresh: Boolean): PlatformUploadResponse {
+            val token = try {
+                firebaseIdToken(forceRefresh)
+            } catch (t: Throwable) {
+                discardPickedApk(picked)
+                throw t
+            } ?: run {
+                discardPickedApk(picked)
+                throw HttpException(HttpStatusCode.Unauthorized, "Sign in before uploading an APK.")
+            }
+            return uploadPickedApk(
+                picked = picked,
+                uploadUrl = "$baseUrl/apps",
+                bearerToken = token,
+                retainOnUnauthorized = !forceRefresh,
             )
         }
-        return response.bodyOrThrow()
+
+        var response = upload(forceRefresh = false)
+        if (response.statusCode == HttpStatusCode.Unauthorized.value) {
+            response = upload(forceRefresh = true)
+        }
+        return appVersionFromUploadResponse(response)
     }
 
     /** Deletes an app and every version under it. */

app/shared/src/commonMain/kotlin/com/commit451/drebin451/file/PickedApk.kt

@@ -1,4 +1,11 @@
 package com.commit451.drebin451.file
 
-/** A user-picked APK: its display name and raw bytes. */
-class PickedApk(val fileName: String, val bytes: ByteArray)
+/**
+ * A user-picked APK represented by a platform file handle rather than raw bytes. Keeping the source
+ * handle lets Android and web stream files as large as 1 GiB without first duplicating them in RAM.
+ */
+class PickedApk(
+    val fileName: String,
+    val sizeBytes: Long,
+    internal val sourceId: String,
+)

app/shared/src/commonMain/kotlin/com/commit451/drebin451/file/PlatformApkUpload.kt

@@ -0,0 +1,20 @@
+package com.commit451.drebin451.file
+
+import kotlinx.serialization.Serializable
+
+@Serializable
+data class PlatformUploadResponse(
+    val statusCode: Int,
+    val body: String,
+)
+
+/** Uploads the selected platform file as multipart form data without loading it all into memory. */
+expect suspend fun uploadPickedApk(
+    picked: PickedApk,
+    uploadUrl: String,
+    bearerToken: String,
+    retainOnUnauthorized: Boolean,
+): PlatformUploadResponse
+
+/** Releases platform resources when validation fails or an upload coroutine is cancelled. */
+expect suspend fun discardPickedApk(picked: PickedApk)

app/shared/src/commonMain/kotlin/com/commit451/drebin451/ui/AppDetailViewModel.kt

@@ -119,7 +119,7 @@ class AppDetailViewModel(initialApp: App) : ViewModel() {
         viewModelScope.launch {
             _state.update { it.copy(uploading = true, storageLimitUploadDialogMessage = null) }
             try {
-                val version = Api.uploadApp(fileName = picked.fileName, bytes = picked.bytes)
+                val version = Api.uploadApp(picked)
                 val label = version.versionName.ifBlank { picked.fileName }
                 _state.update { it.copy(uploading = false, message = "Uploaded $label") }
                 load()

app/shared/src/commonMain/kotlin/com/commit451/drebin451/ui/HomeViewModel.kt

@@ -211,7 +211,7 @@ class HomeViewModel : ViewModel() {
         viewModelScope.launch {
             _state.update { it.copy(uploading = true, storageLimitUploadDialogMessage = null) }
             try {
-                val version = Api.uploadApp(fileName = picked.fileName, bytes = picked.bytes)
+                val version = Api.uploadApp(picked)
                 val label = version.versionName.ifBlank { picked.fileName }
                 val promptApp = notificationPromptAppFor(version)
                 _state.update {