Files
QFT/scripts/parallel_upload_selftest.py
T
2026-08-10 15:45:04 +08:00

61 lines
2.0 KiB
Python

#!/usr/bin/env python3
import hashlib, os, tempfile
from concurrent.futures import ThreadPoolExecutor
SIZE = 64 * 1024 * 1024 + 12345
CHUNK = 8 * 1024 * 1024
WORKERS = 8
IO = 1024 * 1024
def sha(path):
h = hashlib.sha256()
with open(path, "rb", buffering=0) as f:
while True:
b = f.read(4 * 1024 * 1024)
if not b: break
h.update(b)
return h.digest()
def main():
with tempfile.TemporaryDirectory() as d:
src = os.path.join(d, "source.bin")
dst = os.path.join(d, "received.qftpart")
# deterministic 1 MiB pattern, streamed to avoid holding the file in RAM
pattern = bytes((i * 131 + 17) & 0xFF for i in range(IO))
remaining = SIZE
with open(src, "wb", buffering=0) as f:
while remaining:
n = min(remaining, len(pattern))
f.write(pattern[:n]); remaining -= n
with open(dst, "wb") as f: f.truncate(SIZE)
src_fd = os.open(src, os.O_RDONLY)
dst_fd = os.open(dst, os.O_RDWR)
try:
total = (SIZE + CHUNK - 1) // CHUNK
def worker(index):
offset = index * CHUNK
remaining = min(CHUNK, SIZE - offset)
pos = 0
while remaining:
n = min(IO, remaining)
block = os.pread(src_fd, n, offset + pos)
assert len(block) == n
wrote = os.pwrite(dst_fd, block, offset + pos)
assert wrote == n
remaining -= n; pos += n
return index
with ThreadPoolExecutor(max_workers=WORKERS) as pool:
completed = sorted(pool.map(worker, reversed(range(total))))
assert completed == list(range(total))
finally:
os.close(src_fd); os.close(dst_fd)
assert sha(src) == sha(dst)
print(f"PASS parallel upload: {SIZE} bytes, {CHUNK//(1024*1024)} MiB chunks, {WORKERS} workers, 1 MiB worker I/O, SHA-256 identical")
if __name__ == "__main__": main()