优化大文件稳定并发传输与打包流程
This commit is contained in:
@@ -20,12 +20,14 @@ import java.util.concurrent.Executors
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.UUID
|
||||
|
||||
|
||||
data class SelectedFile(
|
||||
val uri: Uri,
|
||||
val name: String,
|
||||
val size: Long,
|
||||
val uploadKey: String = UUID.randomUUID().toString(),
|
||||
)
|
||||
|
||||
object UploadClient {
|
||||
@@ -42,7 +44,7 @@ object UploadClient {
|
||||
// QFT Direct can briefly reset a TCP flow while the Android network route
|
||||
// settles. Chunk PUTs and the init/finish control calls are safe to retry;
|
||||
// the Windows receiver treats a repeated chunk as an idempotent success.
|
||||
private const val LOCAL_REQUEST_ATTEMPTS = 3
|
||||
private const val LOCAL_REQUEST_ATTEMPTS = 8
|
||||
|
||||
fun describeFile(resolver: ContentResolver, uri: Uri): SelectedFile? {
|
||||
var name = "qft-upload.bin"
|
||||
@@ -220,16 +222,20 @@ object UploadClient {
|
||||
|
||||
fun upload(
|
||||
resolver: ContentResolver,
|
||||
network: Network?,
|
||||
networkProvider: () -> Network?,
|
||||
baseUrl: String,
|
||||
pairing: PairingPayload,
|
||||
deviceName: String,
|
||||
file: SelectedFile,
|
||||
abortReason: () -> String? = { null },
|
||||
onProgress: (sent: Long, total: Long) -> Unit,
|
||||
) {
|
||||
if (pairing.version >= 2 && file.size >= PARALLEL_MIN_FILE_SIZE && isSeekable(resolver, file)) {
|
||||
if (pairing.version >= 2 && file.size >= PARALLEL_MIN_FILE_SIZE) {
|
||||
require(isSeekable(resolver, file)) {
|
||||
"大文件必须使用可随机读取的本地文件才能并行直传;请先把云盘文件下载到手机本地后再选择"
|
||||
}
|
||||
try {
|
||||
uploadParallel(resolver, network, baseUrl, pairing, deviceName, file, onProgress)
|
||||
uploadParallel(resolver, networkProvider, baseUrl, pairing, deviceName, file, abortReason, onProgress)
|
||||
return
|
||||
} catch (e: Exception) {
|
||||
// Never mask a transient v2 failure by immediately switching to a
|
||||
@@ -242,14 +248,20 @@ object UploadClient {
|
||||
}
|
||||
}
|
||||
Log.i(TAG, "using single-stream upload: size=${file.size}, endpoint=$baseUrl")
|
||||
uploadLegacy(resolver, network, baseUrl, pairing, deviceName, file, onProgress)
|
||||
uploadLegacy(resolver, networkProvider(), baseUrl, pairing, deviceName, file, abortReason, onProgress)
|
||||
}
|
||||
|
||||
private data class InitReply(val uploadId: String, val chunkSize: Long, val totalChunks: Int)
|
||||
private data class InitReply(
|
||||
val uploadId: String,
|
||||
val chunkSize: Long,
|
||||
val totalChunks: Int,
|
||||
val pendingChunks: IntArray,
|
||||
)
|
||||
|
||||
private fun <T> retryLocalRequest(label: String, block: () -> T): T {
|
||||
private fun <T> retryLocalRequest(label: String, abortReason: () -> String?, block: () -> T): T {
|
||||
var last: Throwable? = null
|
||||
repeat(LOCAL_REQUEST_ATTEMPTS) { attempt ->
|
||||
abortReason()?.let { throw IllegalStateException(it, last) }
|
||||
try {
|
||||
return block()
|
||||
} catch (e: Throwable) {
|
||||
@@ -260,7 +272,7 @@ object UploadClient {
|
||||
e,
|
||||
)
|
||||
if (attempt + 1 < LOCAL_REQUEST_ATTEMPTS) {
|
||||
try { Thread.sleep(120L * (attempt + 1)) } catch (_: InterruptedException) {
|
||||
try { Thread.sleep((250L shl attempt).coerceAtMost(3_000L)) } catch (_: InterruptedException) {
|
||||
Thread.currentThread().interrupt()
|
||||
throw e
|
||||
}
|
||||
@@ -288,27 +300,37 @@ object UploadClient {
|
||||
|
||||
private fun uploadParallel(
|
||||
resolver: ContentResolver,
|
||||
network: Network?,
|
||||
networkProvider: () -> Network?,
|
||||
baseUrl: String,
|
||||
pairing: PairingPayload,
|
||||
deviceName: String,
|
||||
file: SelectedFile,
|
||||
abortReason: () -> String?,
|
||||
onProgress: (sent: Long, total: Long) -> Unit,
|
||||
) {
|
||||
Log.i(TAG, "parallel init: size=${file.size}, endpoint=$baseUrl")
|
||||
val init = initUpload(network, baseUrl, pairing, deviceName, file)
|
||||
Log.i(TAG, "parallel init complete: chunks=${init.totalChunks}, chunkSize=${init.chunkSize}")
|
||||
val init = initUpload(networkProvider, baseUrl, pairing, deviceName, file, file.uploadKey, abortReason)
|
||||
Log.i(TAG, "parallel init complete: chunks=${init.totalChunks}, pending=${init.pendingChunks.size}, chunkSize=${init.chunkSize}")
|
||||
if (init.totalChunks == 0) {
|
||||
finishUpload(network, baseUrl, pairing, deviceName, init.uploadId)
|
||||
finishUpload(networkProvider, baseUrl, pairing, deviceName, init.uploadId, abortReason)
|
||||
onProgress(0, 0)
|
||||
return
|
||||
}
|
||||
|
||||
val cpuWorkers = Runtime.getRuntime().availableProcessors().coerceIn(2, MAX_WORKERS)
|
||||
val workerCount = init.totalChunks.coerceAtMost(cpuWorkers).coerceAtLeast(1)
|
||||
// Upload workers are network/disk bound, not CPU bound. Large files
|
||||
// therefore use all eight lanes even on a 4-core phone.
|
||||
if (init.pendingChunks.isEmpty()) {
|
||||
finishUpload(networkProvider, baseUrl, pairing, deviceName, init.uploadId, abortReason)
|
||||
onProgress(file.size, file.size)
|
||||
return
|
||||
}
|
||||
val workerCount = init.pendingChunks.size.coerceAtMost(MAX_WORKERS).coerceAtLeast(1)
|
||||
val pool = Executors.newFixedThreadPool(workerCount)
|
||||
val next = AtomicInteger(0)
|
||||
val sent = AtomicLong(0)
|
||||
val pendingBytes = init.pendingChunks.sumOf { index ->
|
||||
minOf(init.chunkSize, file.size - index.toLong() * init.chunkSize)
|
||||
}
|
||||
val sent = AtomicLong((file.size - pendingBytes).coerceAtLeast(0L))
|
||||
val lastReport = AtomicLong(0)
|
||||
val failures = mutableListOf<Throwable>()
|
||||
val futures = (0 until workerCount).map {
|
||||
@@ -318,13 +340,15 @@ object UploadClient {
|
||||
val channel = input.channel
|
||||
val buffer = ByteBuffer.allocate(STREAM_BUFFER)
|
||||
while (!Thread.currentThread().isInterrupted) {
|
||||
val index = next.getAndIncrement()
|
||||
if (index >= init.totalChunks) break
|
||||
abortReason()?.let { error(it) }
|
||||
val pendingIndex = next.getAndIncrement()
|
||||
if (pendingIndex >= init.pendingChunks.size) break
|
||||
val index = init.pendingChunks[pendingIndex]
|
||||
val offset = index.toLong() * init.chunkSize
|
||||
val length = minOf(init.chunkSize, file.size - offset)
|
||||
sendChunk(
|
||||
network, baseUrl, pairing, deviceName,
|
||||
init.uploadId, index, channel, offset, length, buffer,
|
||||
networkProvider, baseUrl, pairing, deviceName,
|
||||
init.uploadId, index, channel, offset, length, buffer, abortReason,
|
||||
)
|
||||
val totalSent = sent.addAndGet(length)
|
||||
val now = android.os.SystemClock.elapsedRealtime()
|
||||
@@ -345,7 +369,7 @@ object UploadClient {
|
||||
pool.shutdownNow()
|
||||
}
|
||||
if (failures.isNotEmpty()) throw failures.first()
|
||||
finishUpload(network, baseUrl, pairing, deviceName, init.uploadId)
|
||||
finishUpload(networkProvider, baseUrl, pairing, deviceName, init.uploadId, abortReason)
|
||||
onProgress(file.size, file.size)
|
||||
}
|
||||
|
||||
@@ -357,10 +381,10 @@ object UploadClient {
|
||||
}
|
||||
|
||||
private fun initUpload(
|
||||
network: Network?, baseUrl: String, pairing: PairingPayload,
|
||||
deviceName: String, file: SelectedFile,
|
||||
): InitReply = retryLocalRequest("本地上传初始化失败") {
|
||||
val conn = open(network, URL("$baseUrl/qft/upload/init"))
|
||||
networkProvider: () -> Network?, baseUrl: String, pairing: PairingPayload,
|
||||
deviceName: String, file: SelectedFile, requestKey: String, abortReason: () -> String?,
|
||||
): InitReply = retryLocalRequest("本地上传初始化失败", abortReason) {
|
||||
val conn = open(networkProvider(), URL("$baseUrl/qft/upload/init"))
|
||||
try {
|
||||
conn.requestMethod = "POST"
|
||||
conn.doOutput = true
|
||||
@@ -371,15 +395,27 @@ object UploadClient {
|
||||
conn.setRequestProperty("X-QFT-Filename", b64(file.name))
|
||||
conn.setRequestProperty("X-QFT-Size", file.size.toString())
|
||||
conn.setRequestProperty("X-QFT-Chunk-Size", desiredChunkSize(file.size).toString())
|
||||
conn.setRequestProperty("X-QFT-Upload-Key", requestKey)
|
||||
conn.outputStream.close()
|
||||
val code = conn.responseCode
|
||||
if (code != 200) error("初始化并行上传失败 HTTP $code")
|
||||
val body = conn.inputStream.bufferedReader().use { it.readText() }
|
||||
val obj = JSONObject(body)
|
||||
val totalChunks = obj.getInt("total_chunks")
|
||||
val missing = obj.optJSONArray("missing")
|
||||
val pending = if (missing == null) {
|
||||
IntArray(totalChunks) { it }
|
||||
} else {
|
||||
IntArray(missing.length()) { missing.getInt(it) }
|
||||
}
|
||||
if (pending.any { it !in 0 until totalChunks } || pending.distinct().size != pending.size) {
|
||||
error("电脑返回了无效的缺块列表")
|
||||
}
|
||||
InitReply(
|
||||
uploadId = obj.getString("upload_id"),
|
||||
chunkSize = obj.getLong("chunk_size"),
|
||||
totalChunks = obj.getInt("total_chunks"),
|
||||
totalChunks = totalChunks,
|
||||
pendingChunks = pending,
|
||||
)
|
||||
} finally {
|
||||
conn.disconnect()
|
||||
@@ -387,17 +423,17 @@ object UploadClient {
|
||||
}
|
||||
|
||||
private fun sendChunk(
|
||||
network: Network?, baseUrl: String, pairing: PairingPayload, deviceName: String,
|
||||
networkProvider: () -> Network?, baseUrl: String, pairing: PairingPayload, deviceName: String,
|
||||
uploadId: String, index: Int, channel: FileChannel, offset: Long, length: Long,
|
||||
buffer: ByteBuffer,
|
||||
) = retryLocalRequest("分块 $index 上传失败") {
|
||||
buffer: ByteBuffer, abortReason: () -> String?,
|
||||
) = retryLocalRequest("分块 $index 上传失败", abortReason) {
|
||||
Log.d(TAG, "chunk start: index=$index, bytes=$length, endpoint=$baseUrl")
|
||||
val conn = open(network, URL("$baseUrl/qft/upload/chunk/$uploadId/$index"))
|
||||
val conn = open(networkProvider(), URL("$baseUrl/qft/upload/chunk/$uploadId/$index"))
|
||||
try {
|
||||
conn.requestMethod = "POST"
|
||||
conn.doOutput = true
|
||||
conn.connectTimeout = 4_000
|
||||
conn.readTimeout = 15_000
|
||||
conn.connectTimeout = 8_000
|
||||
conn.readTimeout = 90_000
|
||||
conn.setFixedLengthStreamingMode(length)
|
||||
conn.setRequestProperty("Content-Type", "application/octet-stream")
|
||||
commonHeaders(conn, pairing, deviceName)
|
||||
@@ -406,6 +442,7 @@ object UploadClient {
|
||||
var remaining = length
|
||||
val array = buffer.array()
|
||||
while (remaining > 0) {
|
||||
abortReason()?.let { error(it) }
|
||||
buffer.clear()
|
||||
buffer.limit(minOf(array.size.toLong(), remaining).toInt())
|
||||
val n = channel.read(buffer)
|
||||
@@ -428,10 +465,10 @@ object UploadClient {
|
||||
}
|
||||
|
||||
private fun finishUpload(
|
||||
network: Network?, baseUrl: String, pairing: PairingPayload,
|
||||
deviceName: String, uploadId: String,
|
||||
) = retryLocalRequest("本地上传完成确认失败") {
|
||||
val conn = open(network, URL("$baseUrl/qft/upload/finish/$uploadId"))
|
||||
networkProvider: () -> Network?, baseUrl: String, pairing: PairingPayload,
|
||||
deviceName: String, uploadId: String, abortReason: () -> String?,
|
||||
) = retryLocalRequest("本地上传完成确认失败", abortReason) {
|
||||
val conn = open(networkProvider(), URL("$baseUrl/qft/upload/finish/$uploadId"))
|
||||
try {
|
||||
conn.requestMethod = "POST"
|
||||
conn.doOutput = true
|
||||
@@ -458,6 +495,7 @@ object UploadClient {
|
||||
pairing: PairingPayload,
|
||||
deviceName: String,
|
||||
file: SelectedFile,
|
||||
abortReason: () -> String?,
|
||||
onProgress: (sent: Long, total: Long) -> Unit,
|
||||
) {
|
||||
val conn = open(network, URL("$baseUrl/qft/upload"))
|
||||
@@ -478,6 +516,7 @@ object UploadClient {
|
||||
var sent = 0L
|
||||
var lastReport = 0L
|
||||
while (true) {
|
||||
abortReason()?.let { error(it) }
|
||||
val n = input.read(buf)
|
||||
if (n <= 0) break
|
||||
output.write(buf, 0, n)
|
||||
@@ -520,7 +559,7 @@ object UploadClient {
|
||||
|
||||
private fun commonHeaders(conn: HttpURLConnection, pairing: PairingPayload, deviceName: String) {
|
||||
conn.useCaches = false
|
||||
conn.setRequestProperty("Connection", "close")
|
||||
conn.setRequestProperty("Connection", "keep-alive")
|
||||
conn.setRequestProperty("X-QFT-Token", pairing.session)
|
||||
conn.setRequestProperty("X-QFT-Device", b64(deviceName))
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user