v1.8.73: reliable SFTP upload — chunked resume, SHA256 verification, adaptive retry
- CLI (ssh.py): chunked resume upload for files >10MB with .part atomic rename - CLI: SHA256 verification (sha256sum on Linux, Get-FileHash on Windows) - CLI: adaptive retry count based on file size (up to 30 for large files) - CLI: SSH keepalive 15s + window_size 4MB for stable transfers - CLI: path injection fix in SHA256 shell commands - CLI: Windows SFTP path fix for PowerShell Get-FileHash - GUI (ssh_client.py): chunked upload with resume in SFTPSession - GUI: retry up to 3 attempts with SHA256 readback in SSHClientWrapper - GUI: keepalive 15s + window_size 4MB in both auth paths - Tested: 5MB, 15MB, 200MB uploads to Windows SSH server (116) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
chrome-storm-c442
@@ -7,6 +7,7 @@ import platform
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
import hashlib
|
||||
import paramiko
|
||||
from core.logger import log
|
||||
|
||||
@@ -47,7 +48,8 @@ def _connect_client(server: dict, key_path: str, timeout: int = 15) -> paramiko.
|
||||
client.connect(**kwargs)
|
||||
transport = client.get_transport()
|
||||
if transport is not None:
|
||||
transport.set_keepalive(30)
|
||||
transport.set_keepalive(15)
|
||||
transport.default_window_size = 4 * 1024 * 1024
|
||||
return client
|
||||
except paramiko.AuthenticationException:
|
||||
log.debug(f"Key auth failed for {server.get('alias', '?')}, trying password")
|
||||
@@ -73,7 +75,8 @@ def _connect_client(server: dict, key_path: str, timeout: int = 15) -> paramiko.
|
||||
client.connect(**kwargs)
|
||||
transport = client.get_transport()
|
||||
if transport is not None:
|
||||
transport.set_keepalive(30)
|
||||
transport.set_keepalive(15)
|
||||
transport.default_window_size = 4 * 1024 * 1024
|
||||
return client
|
||||
|
||||
raise Exception(f"No auth method for {server.get('alias', 'unknown')}")
|
||||
@@ -238,21 +241,94 @@ class SSHClientWrapper:
|
||||
pass
|
||||
client.close()
|
||||
|
||||
def upload(self, local_path: str, remote_path: str, progress_cb=None):
|
||||
client = self.connect()
|
||||
try:
|
||||
sftp = client.open_sftp()
|
||||
if progress_cb:
|
||||
sftp.put(local_path, remote_path, callback=progress_cb)
|
||||
else:
|
||||
sftp.put(local_path, remote_path)
|
||||
def upload(self, local_path: str, remote_path: str, progress_cb=None,
|
||||
max_retries=3):
|
||||
file_size = os.path.getsize(local_path)
|
||||
use_resume = file_size > 10 * 1024 * 1024
|
||||
tmp_path = (remote_path + ".part") if use_resume else remote_path
|
||||
|
||||
for attempt in range(1, max_retries + 1):
|
||||
client = self.connect()
|
||||
try:
|
||||
sftp.chmod(remote_path, 0o664)
|
||||
except OSError:
|
||||
pass # Windows OpenSSH doesn't support chmod
|
||||
sftp.close()
|
||||
finally:
|
||||
client.close()
|
||||
sftp = client.open_sftp()
|
||||
|
||||
if use_resume:
|
||||
remote_offset = 0
|
||||
try:
|
||||
remote_offset = sftp.stat(tmp_path).st_size
|
||||
if remote_offset > file_size:
|
||||
sftp.remove(tmp_path)
|
||||
remote_offset = 0
|
||||
except (FileNotFoundError, IOError):
|
||||
remote_offset = 0
|
||||
|
||||
with open(local_path, 'rb') as f:
|
||||
f.seek(remote_offset)
|
||||
if remote_offset > 0:
|
||||
rf = sftp.open(tmp_path, 'r+b')
|
||||
rf.seek(remote_offset)
|
||||
else:
|
||||
rf = sftp.open(tmp_path, 'wb')
|
||||
rf.set_pipelined(True)
|
||||
try:
|
||||
transferred = remote_offset
|
||||
while transferred < file_size:
|
||||
data = f.read(256 * 1024)
|
||||
if not data:
|
||||
break
|
||||
rf.write(data)
|
||||
transferred += len(data)
|
||||
if progress_cb:
|
||||
progress_cb(transferred, file_size)
|
||||
finally:
|
||||
rf.close()
|
||||
|
||||
if sftp.stat(tmp_path).st_size != file_size:
|
||||
raise IOError("Upload size mismatch")
|
||||
|
||||
# SHA256 verification via SFTP readback
|
||||
local_hash = hashlib.sha256()
|
||||
with open(local_path, 'rb') as f:
|
||||
for chunk in iter(lambda: f.read(1024 * 1024), b''):
|
||||
local_hash.update(chunk)
|
||||
remote_hash = hashlib.sha256()
|
||||
with sftp.open(tmp_path, 'rb') as rf:
|
||||
while True:
|
||||
chunk = rf.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
remote_hash.update(chunk)
|
||||
if local_hash.hexdigest() != remote_hash.hexdigest():
|
||||
sftp.remove(tmp_path)
|
||||
raise IOError("SHA256 mismatch after upload")
|
||||
|
||||
try:
|
||||
sftp.remove(remote_path)
|
||||
except (FileNotFoundError, IOError):
|
||||
pass
|
||||
sftp.rename(tmp_path, remote_path)
|
||||
else:
|
||||
if progress_cb:
|
||||
sftp.put(local_path, remote_path, callback=progress_cb)
|
||||
else:
|
||||
sftp.put(local_path, remote_path)
|
||||
|
||||
try:
|
||||
sftp.chmod(remote_path, 0o664)
|
||||
except OSError:
|
||||
pass
|
||||
sftp.close()
|
||||
return # Success
|
||||
|
||||
except (EOFError, TimeoutError, OSError,
|
||||
paramiko.SSHException) as e:
|
||||
log.warning(f"Upload attempt {attempt}/{max_retries}: {e}")
|
||||
if attempt < max_retries:
|
||||
time.sleep(2 ** attempt)
|
||||
else:
|
||||
raise
|
||||
finally:
|
||||
client.close()
|
||||
|
||||
def download(self, remote_path: str, local_path: str, progress_cb=None):
|
||||
client = self.connect()
|
||||
@@ -385,11 +461,83 @@ class SFTPSession:
|
||||
self._sftp.rename(old, new)
|
||||
|
||||
def upload(self, local_path: str, remote_path: str, progress_cb=None):
|
||||
if progress_cb:
|
||||
file_size = os.path.getsize(local_path)
|
||||
if file_size > 10 * 1024 * 1024: # >10MB: chunked
|
||||
self._upload_chunked(local_path, remote_path, file_size, progress_cb)
|
||||
elif progress_cb:
|
||||
self._sftp.put(local_path, remote_path, callback=progress_cb)
|
||||
else:
|
||||
self._sftp.put(local_path, remote_path)
|
||||
|
||||
def _upload_chunked(self, local_path, remote_path, file_size, progress_cb):
|
||||
"""Chunked upload with resume, .part, atomic rename and SHA256 verification."""
|
||||
tmp_path = remote_path + ".part"
|
||||
remote_offset = 0
|
||||
try:
|
||||
remote_offset = self._sftp.stat(tmp_path).st_size
|
||||
if remote_offset > file_size:
|
||||
self._sftp.remove(tmp_path)
|
||||
remote_offset = 0
|
||||
except (FileNotFoundError, IOError):
|
||||
remote_offset = 0
|
||||
|
||||
with open(local_path, 'rb') as f:
|
||||
f.seek(remote_offset)
|
||||
if remote_offset > 0:
|
||||
rf = self._sftp.open(tmp_path, 'r+b')
|
||||
rf.seek(remote_offset)
|
||||
else:
|
||||
rf = self._sftp.open(tmp_path, 'wb')
|
||||
rf.set_pipelined(True)
|
||||
try:
|
||||
transferred = remote_offset
|
||||
while transferred < file_size:
|
||||
data = f.read(256 * 1024)
|
||||
if not data:
|
||||
break
|
||||
rf.write(data)
|
||||
transferred += len(data)
|
||||
if progress_cb:
|
||||
progress_cb(transferred, file_size)
|
||||
finally:
|
||||
rf.close()
|
||||
|
||||
# Validate: size
|
||||
if self._sftp.stat(tmp_path).st_size != file_size:
|
||||
raise IOError("Upload size mismatch")
|
||||
|
||||
# Validate: SHA256 via SFTP readback
|
||||
local_hash = hashlib.sha256()
|
||||
with open(local_path, 'rb') as f:
|
||||
for chunk in iter(lambda: f.read(1024 * 1024), b''):
|
||||
local_hash.update(chunk)
|
||||
|
||||
remote_hash = hashlib.sha256()
|
||||
with self._sftp.open(tmp_path, 'rb') as rf:
|
||||
while True:
|
||||
chunk = rf.read(1024 * 1024)
|
||||
if not chunk:
|
||||
break
|
||||
remote_hash.update(chunk)
|
||||
|
||||
if local_hash.hexdigest() != remote_hash.hexdigest():
|
||||
self._sftp.remove(tmp_path)
|
||||
raise IOError(
|
||||
f"SHA256 mismatch! local={local_hash.hexdigest()[:16]}... "
|
||||
f"remote={remote_hash.hexdigest()[:16]}..."
|
||||
)
|
||||
|
||||
# Atomic rename
|
||||
try:
|
||||
self._sftp.remove(remote_path)
|
||||
except (FileNotFoundError, IOError):
|
||||
pass
|
||||
self._sftp.rename(tmp_path, remote_path)
|
||||
try:
|
||||
self._sftp.chmod(remote_path, 0o664)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def download(self, remote_path: str, local_path: str, progress_cb=None):
|
||||
if progress_cb:
|
||||
self._sftp.get(remote_path, local_path, callback=progress_cb)
|
||||
|
||||
Reference in New Issue
Block a user