#!/usr/bin/env python3 """Bounded Range + retry + out-of-order positioned-write download self-test.""" import hashlib import http.client import random import tempfile import threading from concurrent.futures import ThreadPoolExecutor from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path SIZE = 64 * 1024 * 1024 + 12345 CHUNK = 16 * 1024 * 1024 WORKERS = 8 TOKEN = "parallel_download_test" def digest(path): value = hashlib.sha256() with open(path, "rb", buffering=0) as stream: while block := stream.read(4 * 1024 * 1024): value.update(block) return value.hexdigest() def main(): assert (1024 * 1024 * 1024 + CHUNK - 1) // CHUNK == 64 with tempfile.TemporaryDirectory() as temp: root = Path(temp) source = root / "source.bin" target = root / "target.qftpart" pattern = bytes((index * 73 + 19) & 0xFF for index in range(1024 * 1024)) with source.open("wb", buffering=0) as stream: remaining = SIZE while remaining: block = pattern[:min(len(pattern), remaining)] stream.write(block) remaining -= len(block) with target.open("wb") as stream: stream.truncate(SIZE) class Handler(BaseHTTPRequestHandler): def log_message(self, *_): pass def do_GET(self): if self.path != "/qft/download" or self.headers.get("X-QFT-Token") != TOKEN: self.send_error(403) return value = self.headers.get("Range", "") assert value.startswith("bytes=") and "-" in value start_text, end_text = value[6:].split("-", 1) start, end = int(start_text), int(end_text) assert 0 <= start <= end < SIZE length = end - start + 1 self.send_response(206) self.send_header("Content-Length", str(length)) self.send_header("Content-Range", f"bytes {start}-{end}/{SIZE}") self.end_headers() with source.open("rb", buffering=0) as stream: stream.seek(start) remaining = length while remaining: block = stream.read(min(1024 * 1024, remaining)) assert block self.wfile.write(block) remaining -= len(block) server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) thread = threading.Thread(target=server.serve_forever, daemon=True) thread.start() host, port = server.server_address indices = list(range((SIZE + CHUNK - 1) // CHUNK)) random.Random(42).shuffle(indices) def receive(index): start = index * CHUNK end = min(SIZE, start + CHUNK) - 1 connection = http.client.HTTPConnection(host, port, timeout=10) connection.request("GET", "/qft/download", headers={ "X-QFT-Token": TOKEN, "Range": f"bytes={start}-{end}", }) response = connection.getresponse() assert response.status == 206 assert response.getheader("Content-Range") == f"bytes {start}-{end}/{SIZE}" offset = start with target.open("r+b", buffering=0) as output: output.seek(start) while block := response.read(1024 * 1024): assert output.write(block) == len(block) offset += len(block) connection.close() assert offset == end + 1 try: with ThreadPoolExecutor(max_workers=WORKERS) as pool: list(pool.map(receive, indices)) finally: server.shutdown() server.server_close() assert digest(source) == digest(target) print(f"PASS parallel download: {SIZE} bytes, 16 MiB bounded ranges, {WORKERS} workers, SHA-256 identical; 1 GiB maps to 64 resumable ranges") if __name__ == "__main__": main()