V0.3.0版本优化传输速度和提示

This commit is contained in:
zhy
2026-08-10 15:45:04 +08:00
commit d885259e2b
58 changed files with 10284 additions and 0 deletions
@@ -0,0 +1,96 @@
#!/usr/bin/env python3
"""Loopback integration test for QFTDISC2 + Parallel v2 init/chunk/finish semantics."""
import hashlib, http.client, json, os, random, socket, tempfile, threading, time
from concurrent.futures import ThreadPoolExecutor
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
TOKEN = "integration_session_128bit_ABC"
DISCOVERY_MAGIC = f"QFTDISC2:{TOKEN}".encode()
CHUNK = 8 * 1024 * 1024
with tempfile.TemporaryDirectory() as td:
root = Path(td)
source = root / "src.bin"
source.write_bytes(os.urandom(40 * 1024 * 1024 + 12345))
source_sha = hashlib.sha256(source.read_bytes()).hexdigest()
part = root / "recv.qftpart"
part.write_bytes(b"")
part_fp = open(part, "r+b", buffering=0)
part_fp.truncate(source.stat().st_size)
lock = threading.Lock()
received = set()
upload_id = "up_test_01"
total_chunks = (source.stat().st_size + CHUNK - 1) // CHUNK
class Handler(BaseHTTPRequestHandler):
def log_message(self, *_): pass
def _auth(self): return self.headers.get("X-QFT-Token") == TOKEN
def _reply(self, code, body=b"{}"):
self.send_response(code); self.send_header("Content-Length", str(len(body))); self.end_headers(); self.wfile.write(body)
def do_GET(self):
if not self._auth(): return self._reply(403)
if self.path == "/qft/ping": return self._reply(200, b'{"ok":true,"protocol":2,"parallel":true}')
self._reply(404)
def do_POST(self):
if not self._auth(): return self._reply(403)
if self.path == "/qft/upload/init":
body = json.dumps({"ok":True,"upload_id":upload_id,"chunk_size":CHUNK,"total_chunks":total_chunks}).encode()
return self._reply(200, body)
prefix = f"/qft/upload/chunk/{upload_id}/"
if self.path.startswith(prefix):
idx = int(self.path[len(prefix):]); n = int(self.headers["Content-Length"]); data = self.rfile.read(n)
offset = idx * CHUNK
assert n == min(CHUNK, source.stat().st_size - offset)
with lock:
part_fp.seek(offset); part_fp.write(data); received.add(idx)
return self._reply(200)
if self.path == f"/qft/upload/finish/{upload_id}":
if len(received) != total_chunks: return self._reply(409)
with lock: part_fp.flush(); os.fsync(part_fp.fileno())
return self._reply(200, json.dumps({"ok":True}).encode())
self._reply(404)
httpd = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
port = httpd.server_address[1]
threading.Thread(target=httpd.serve_forever, daemon=True).start()
udp = socket.socket(socket.AF_INET, socket.SOCK_DGRAM); udp.bind(("127.0.0.1", 0)); udp_port = udp.getsockname()[1]
stop = threading.Event()
def udp_server():
while not stop.is_set():
udp.settimeout(.2)
try: data, peer = udp.recvfrom(512)
except socket.timeout: continue
except OSError:
if stop.is_set(): break
raise
if data == DISCOVERY_MAGIC: udp.sendto(f"QFTHERE2:{TOKEN}:{port}".encode(), peer)
threading.Thread(target=udp_server, daemon=True).start()
# UDP discovery client.
c = socket.socket(socket.AF_INET, socket.SOCK_DGRAM); c.settimeout(1); c.sendto(DISCOVERY_MAGIC, ("127.0.0.1", udp_port))
resp, _ = c.recvfrom(512); assert resp == f"QFTHERE2:{TOKEN}:{port}".encode(); c.close()
def request(method, path, body=b"", headers=None):
conn = http.client.HTTPConnection("127.0.0.1", port, timeout=5)
h = {"X-QFT-Token": TOKEN}; h.update(headers or {})
conn.request(method, path, body=body, headers=h); r = conn.getresponse(); payload = r.read(); status = r.status; conn.close(); return status, payload
assert request("GET", "/qft/ping")[0] == 200
status, body = request("POST", "/qft/upload/init", headers={"Content-Length":"0"}); assert status == 200
init = json.loads(body); assert init["chunk_size"] == CHUNK and init["total_chunks"] == total_chunks
indices = list(range(total_chunks)); random.Random(42).shuffle(indices)
def send(idx):
with open(source, "rb") as f:
f.seek(idx * CHUNK); data = f.read(min(CHUNK, source.stat().st_size - idx * CHUNK))
status, _ = request("POST", f"/qft/upload/chunk/{upload_id}/{idx}", data, {"Content-Length":str(len(data))})
assert status == 200
started = time.perf_counter()
with ThreadPoolExecutor(max_workers=8) as ex: list(ex.map(send, indices))
assert request("POST", f"/qft/upload/finish/{upload_id}", headers={"Content-Length":"0"})[0] == 200
elapsed = time.perf_counter() - started
part_fp.close(); recv_sha = hashlib.sha256(part.read_bytes()).hexdigest(); assert recv_sha == source_sha
stop.set(); udp.close(); httpd.shutdown(); httpd.server_close()
print(f"PASS loopback Parallel v2 integration: UDP discovery + HTTP auth + {total_chunks} out-of-order chunks / 8 workers; "
f"{source.stat().st_size} bytes SHA-256 identical in {elapsed:.2f}s")