#!/usr/bin/env python3 import hashlib, os, tempfile from concurrent.futures import ThreadPoolExecutor SIZE = 64 * 1024 * 1024 + 12345 CHUNK = 8 * 1024 * 1024 WORKERS = 8 IO = 1024 * 1024 def sha(path): h = hashlib.sha256() with open(path, "rb", buffering=0) as f: while True: b = f.read(4 * 1024 * 1024) if not b: break h.update(b) return h.digest() def main(): with tempfile.TemporaryDirectory() as d: src = os.path.join(d, "source.bin") dst = os.path.join(d, "received.qftpart") # deterministic 1 MiB pattern, streamed to avoid holding the file in RAM pattern = bytes((i * 131 + 17) & 0xFF for i in range(IO)) remaining = SIZE with open(src, "wb", buffering=0) as f: while remaining: n = min(remaining, len(pattern)) f.write(pattern[:n]); remaining -= n with open(dst, "wb") as f: f.truncate(SIZE) src_fd = os.open(src, os.O_RDONLY) dst_fd = os.open(dst, os.O_RDWR) try: total = (SIZE + CHUNK - 1) // CHUNK def worker(index): offset = index * CHUNK remaining = min(CHUNK, SIZE - offset) pos = 0 while remaining: n = min(IO, remaining) block = os.pread(src_fd, n, offset + pos) assert len(block) == n wrote = os.pwrite(dst_fd, block, offset + pos) assert wrote == n remaining -= n; pos += n return index with ThreadPoolExecutor(max_workers=WORKERS) as pool: completed = sorted(pool.map(worker, reversed(range(total)))) assert completed == list(range(total)) finally: os.close(src_fd); os.close(dst_fd) assert sha(src) == sha(dst) print(f"PASS parallel upload: {SIZE} bytes, {CHUNK//(1024*1024)} MiB chunks, {WORKERS} workers, 1 MiB worker I/O, SHA-256 identical") if __name__ == "__main__": main()