Files
QFT/scripts/parallel_download_selftest.py
2026-08-11 11:27:40 +08:00

108 lines
4.1 KiB
Python

#!/usr/bin/env python3
"""Bounded Range + retry + out-of-order positioned-write download self-test."""
import hashlib
import http.client
import random
import tempfile
import threading
from concurrent.futures import ThreadPoolExecutor
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
SIZE = 64 * 1024 * 1024 + 12345
CHUNK = 16 * 1024 * 1024
WORKERS = 8
TOKEN = "parallel_download_test"
def digest(path):
value = hashlib.sha256()
with open(path, "rb", buffering=0) as stream:
while block := stream.read(4 * 1024 * 1024):
value.update(block)
return value.hexdigest()
def main():
assert (1024 * 1024 * 1024 + CHUNK - 1) // CHUNK == 64
with tempfile.TemporaryDirectory() as temp:
root = Path(temp)
source = root / "source.bin"
target = root / "target.qftpart"
pattern = bytes((index * 73 + 19) & 0xFF for index in range(1024 * 1024))
with source.open("wb", buffering=0) as stream:
remaining = SIZE
while remaining:
block = pattern[:min(len(pattern), remaining)]
stream.write(block)
remaining -= len(block)
with target.open("wb") as stream:
stream.truncate(SIZE)
class Handler(BaseHTTPRequestHandler):
def log_message(self, *_):
pass
def do_GET(self):
if self.path != "/qft/download" or self.headers.get("X-QFT-Token") != TOKEN:
self.send_error(403)
return
value = self.headers.get("Range", "")
assert value.startswith("bytes=") and "-" in value
start_text, end_text = value[6:].split("-", 1)
start, end = int(start_text), int(end_text)
assert 0 <= start <= end < SIZE
length = end - start + 1
self.send_response(206)
self.send_header("Content-Length", str(length))
self.send_header("Content-Range", f"bytes {start}-{end}/{SIZE}")
self.end_headers()
with source.open("rb", buffering=0) as stream:
stream.seek(start)
remaining = length
while remaining:
block = stream.read(min(1024 * 1024, remaining))
assert block
self.wfile.write(block)
remaining -= len(block)
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
host, port = server.server_address
indices = list(range((SIZE + CHUNK - 1) // CHUNK))
random.Random(42).shuffle(indices)
def receive(index):
start = index * CHUNK
end = min(SIZE, start + CHUNK) - 1
connection = http.client.HTTPConnection(host, port, timeout=10)
connection.request("GET", "/qft/download", headers={
"X-QFT-Token": TOKEN,
"Range": f"bytes={start}-{end}",
})
response = connection.getresponse()
assert response.status == 206
assert response.getheader("Content-Range") == f"bytes {start}-{end}/{SIZE}"
offset = start
with target.open("r+b", buffering=0) as output:
output.seek(start)
while block := response.read(1024 * 1024):
assert output.write(block) == len(block)
offset += len(block)
connection.close()
assert offset == end + 1
try:
with ThreadPoolExecutor(max_workers=WORKERS) as pool:
list(pool.map(receive, indices))
finally:
server.shutdown()
server.server_close()
assert digest(source) == digest(target)
print(f"PASS parallel download: {SIZE} bytes, 16 MiB bounded ranges, {WORKERS} workers, SHA-256 identical; 1 GiB maps to 64 resumable ranges")
if __name__ == "__main__":
main()