mirror of
https://github.com/MyLuxy/overdrive.git
synced 2026-10-17 17:01:15 +02:00
507 lines
34 KiB
Python
507 lines
34 KiB
Python
"""Real local archive extraction with the same native worker used on PS5."""
|
|
from pathlib import Path
|
|
import binascii,hashlib,io,json,lzma,os,random,struct,subprocess,tarfile,tempfile,time,zipfile
|
|
from rar_worker import build_host
|
|
ROOT=Path(__file__).resolve().parents[1]
|
|
DATA=bytes(range(256))*1024
|
|
TOO_BIG="This archive needs more memory to extract than the console app can use."
|
|
NO_LAST="Incomplete RAR archive or missing volume."
|
|
NO_FIRST="Missing first RAR volume."
|
|
|
|
def number7z(v):
|
|
for n in range(8):
|
|
if v<1<<(8*n+7-n):return bytes([(0xFF00>>n)&0xFF|v>>(8*n)])+(v&((1<<8*n)-1)).to_bytes(n,"little")
|
|
return b"\xff"+v.to_bytes(8,"little")
|
|
|
|
def encode_7z_header(plain):
|
|
"""Rewrites a bsdtar 7z with the LZMA-compressed header 7-Zip writes by default."""
|
|
offset,size=struct.unpack("<QQ",plain[12:28]);header=plain[32+offset:32+offset+size]
|
|
packed=lzma.compress(header,format=lzma.FORMAT_RAW,filters=[{"id":lzma.FILTER_LZMA1,"dict_size":1<<16}])
|
|
coder=b"\x23\x03\x01\x01"+number7z(5)+b"\x5d"+(1<<16).to_bytes(4,"little")
|
|
encoded=(b"\x17\x06"+number7z(offset)+number7z(1)+b"\x09"+number7z(len(packed))+b"\x00"
|
|
+b"\x07\x0b"+number7z(1)+b"\x00"+number7z(1)+coder+b"\x0c"+number7z(len(header))+b"\x00\x00")
|
|
start=struct.pack("<QQI",offset+len(packed),len(encoded),binascii.crc32(encoded))
|
|
return plain[:8]+struct.pack("<I",binascii.crc32(start))+start+plain[32:32+offset]+packed+encoded
|
|
|
|
def vint(v):
|
|
out=b""
|
|
while v>=0x80:out+=bytes([v&0x7f|0x80]);v>>=7
|
|
return out+bytes([v])
|
|
|
|
def rar5_block(header,data=b""):
|
|
size=bytes([len(header)])
|
|
return struct.pack("<I",binascii.crc32(size+header))+size+header+data
|
|
|
|
def rar5_with_window(*shifts):
|
|
"""A RAR5 archive whose files, in order, declare 128 KiB << shift dictionaries."""
|
|
data=bytes(64)
|
|
files=b"".join(rar5_block(bytes([2,2,len(data),0,len(data),0])+vint(3<<7|shift<<10)+bytes([0,1])+b"a",data) for shift in shifts)
|
|
return b"Rar!\x1a\x07\x01\x00"+rar5_block(bytes([1,0,0]))+files+rar5_block(bytes([5,0,0]))
|
|
|
|
def plain_rar5(name,data,link=None):
|
|
"""An unencrypted RAR5 archive storing one file, or a symbolic link to `link`."""
|
|
extra=b""
|
|
if link is not None:
|
|
record=vint(5)+vint(1)+vint(0)+vint(len(link))+link.encode();extra=vint(len(record))+record
|
|
fields=vint(4)+vint(len(data))+vint(0)+struct.pack("<I",binascii.crc32(data))+vint(0)+vint(1)+vint(len(name.encode()))+name.encode()
|
|
file=vint(2)+vint(3 if extra else 2)+(vint(len(extra)) if extra else b"")+vint(len(data))+fields+extra
|
|
return b"Rar!\x1a\x07\x01\x00"+rar5_block(bytes([1,0,0]))+rar5_block(file,data)+rar5_block(bytes([5,0,0]))
|
|
|
|
def plain_rar4(name,data,end=True):
|
|
"""A RAR 1.5-4 archive storing one file, with or without the end block RAR 2.x and "-en" leave out."""
|
|
def block(kind,flags,fields=b"",data=b""):
|
|
body=bytes([kind])+struct.pack("<HH",flags,7+len(fields))+fields
|
|
return struct.pack("<H",binascii.crc32(body)&0xffff)+body+data
|
|
fields=struct.pack("<IIBIIBBHI",len(data),len(data),3,binascii.crc32(data),0,29,0x30,len(name),0o100644)+name.encode()
|
|
return b"Rar!\x1a\x07\x00"+block(0x73,0,bytes(6))+block(0x74,0x8000,fields,data)+(block(0x7b,0x4000) if end else b"")
|
|
|
|
def encrypted_rar5(data,password="DLPSGAME.COM",name="nested/game.ffpfsc",headers=True,volumes=1,compression=0,crypto_flags=1,quick_open=False):
|
|
"""Independent RAR5 fixture with AES-CBC data/headers and spec password checks."""
|
|
salt=bytes(range(16));power=15;iterations=1<<power
|
|
key=hashlib.pbkdf2_hmac("sha256",password.encode(),salt,iterations)
|
|
check=hashlib.pbkdf2_hmac("sha256",password.encode(),salt,iterations+32)
|
|
folded=bytes(check[i]^check[i+8]^check[i+16]^check[i+24] for i in range(8))
|
|
password_check=folded+hashlib.sha256(folded).digest()[:4]
|
|
def aes(plain,iv):
|
|
plain+=bytes((-len(plain))%16)
|
|
return subprocess.check_output(["openssl","enc","-aes-256-cbc","-nopad","-K",key.hex(),"-iv",iv.hex()],input=plain)
|
|
def block(body):
|
|
size=vint(len(body));return struct.pack("<I",binascii.crc32(size+body))+size+body
|
|
def framed(body):
|
|
plain=block(body);iv=bytes(range(16,32))
|
|
return iv+aes(plain,iv) if headers else plain
|
|
out=[]
|
|
# Like RAR, encrypt the file once and split the AES-CBC stream at unaligned volume offsets.
|
|
iv=bytes(range(32,48));stream=aes(data or bytes(16),iv)
|
|
cuts=[0]+[i*len(stream)//volumes+5 for i in range(1,volumes)]+[len(stream)]
|
|
assert all(cut%16 for cut in cuts[1:-1])
|
|
for i in range(volumes):
|
|
packed=stream[cuts[i]:cuts[i+1]]
|
|
encryption=vint(1)+vint(0)+vint(crypto_flags)+bytes([power])+salt+iv+password_check
|
|
extra=vint(len(encryption))+encryption
|
|
common=3|(8 if i else 0)|(16 if i+1<volumes else 0)
|
|
body=vint(2)+vint(common)+vint(len(extra))+vint(len(packed))+vint(4)+vint(len(data))+vint(0)
|
|
body+=struct.pack("<I",binascii.crc32(data))+vint(compression)+vint(0)+vint(len(name.encode()))+name.encode()+extra
|
|
prefix=b"Rar!\x1a\x07\x01\x00"
|
|
if headers:prefix+=block(vint(4)+vint(0)+vint(0)+vint(1)+bytes([power])+salt+password_check)
|
|
main=vint(1)+vint(0)+(vint(3)+vint(i) if volumes>1 else vint(0))
|
|
service=b""
|
|
if quick_open and i+1==volumes:
|
|
# A quick-open index can carry a valid check-field checksum for a different password.
|
|
# It is optional cached metadata, not a file to extract or authenticate.
|
|
other=hashlib.pbkdf2_hmac("sha256",b"index-password",salt,iterations+32)
|
|
check=bytes(other[j]^other[j+8]^other[j+16]^other[j+24] for j in range(8))
|
|
encryption=vint(1)+vint(0)+vint(1)+bytes([power])+salt+iv+check+hashlib.sha256(check).digest()[:4]
|
|
extra=vint(len(encryption))+encryption
|
|
index=aes(bytes(2*128*1024+16),iv)
|
|
header=vint(3)+vint(3)+vint(len(extra))+vint(len(index))+vint(0)+vint(len(index))+vint(0)
|
|
header+=vint(0)+vint(0)+vint(2)+b"QO"+extra
|
|
service=framed(header)+index
|
|
# A volume may end with filler after its end header; UnRAR opens the next volume regardless.
|
|
filler=bytes(56) if i+1<volumes else b""
|
|
out.append(prefix+framed(main)+framed(body)+packed+service+framed(vint(5)+vint(0)+vint(int(i+1<volumes)))+filler)
|
|
return out
|
|
|
|
def words():
|
|
"""The game.bin inside rar5_encrypted_volumes.part*.rar.uu."""
|
|
r=random.Random(5)
|
|
vocabulary=[bytes(r.choice(b"abcdefghijklmnopqrstuvwxyz") for _ in range(r.randint(2,9))) for _ in range(4000)]
|
|
return b" ".join(r.choice(vocabulary) for _ in range(24000))
|
|
|
|
def uudecode(source,destination):
|
|
lines=source.read_bytes().splitlines();at=next(i for i,line in enumerate(lines) if line.startswith(b"begin "))
|
|
destination.write_bytes(b"".join(binascii.a2b_uu(line) for line in lines[at+1:] if line not in (b"end",b"`",b" ",b"")))
|
|
return destination
|
|
|
|
def main():
|
|
with tempfile.TemporaryDirectory(prefix="ps5-react-archives-") as tmp:
|
|
root=Path(tmp);binary=root/"archive-client"
|
|
prefix=subprocess.check_output(["brew","--prefix","libarchive"],text=True).strip()
|
|
xz=subprocess.check_output(["brew","--prefix","xz"],text=True).strip()
|
|
crypto=subprocess.check_output(["brew","--prefix","openssl@3"],text=True).strip()
|
|
sources=[ROOT/"tools/tests/archive_client.cpp",ROOT/"native/shared/archives.cpp",ROOT/"native/shared/archive_preflight.cpp",ROOT/"native/shared/rar5_password.cpp",ROOT/"native/shared/archive_writer.cpp",ROOT/"native/desktop/rar_worker_process.cpp"]
|
|
# RAR volumes go to rar-extract, as in the app; the other formats stay with libarchive.
|
|
os.environ["PS5_REACT_RAR_WORKER"]=str(build_host(root/"rar-extract"))
|
|
subprocess.run(["clang++","-std=c++20","-O2","-Wall","-Wextra","-Werror","-pthread","-I",str(ROOT/"native/shared"),"-I",prefix+"/include","-I",xz+"/include","-I",crypto+"/include",*map(str,sources),"-L",prefix+"/lib","-L",xz+"/lib","-L",crypto+"/lib","-larchive","-llzma","-lcrypto","-lz","-o",str(binary)],check=True)
|
|
def run(name,parts,limit=1024*1024,password=None):
|
|
args=[] if password is None else ["--password",password]
|
|
out=subprocess.check_output([str(binary),str(root/name),str(limit),*map(str,parts),*args],text=True,timeout=20).splitlines()
|
|
return out
|
|
zip=root/"fixture.zip"
|
|
with zipfile.ZipFile(zip,"w",compression=zipfile.ZIP_DEFLATED) as z:z.writestr("nested/game.ffpfsc",DATA)
|
|
assert run("zip",[zip])[0]=="completed"
|
|
assert (root/"zip/game.ffpfsc").read_bytes()==DATA
|
|
# The game runs as another user than the app: an extracted tree only its owner reads freezes the console
|
|
# at launch. Every folder and file is open to everyone, the extraction root included.
|
|
def open_to_all(tree):
|
|
for p in [tree,*tree.rglob("*")]:
|
|
if p.name.startswith(".ps5-react-"):continue
|
|
mode=p.stat().st_mode&0o777
|
|
assert mode==(0o777 if p.is_dir() else 0o666),(p,oct(mode))
|
|
app=root/"app.zip"
|
|
with zipfile.ZipFile(app,"w") as z:z.writestr("PPSA01473-app/sce_sys/param.json","{}");z.writestr("PPSA01473-app/eboot.bin",DATA)
|
|
assert run("zip-app",[app])[0]=="completed"
|
|
open_to_all(root/"zip-app")
|
|
# A heap without room for an in-process extraction fails it cleanly, with nothing written.
|
|
os.environ["ARCHIVE_NO_MEMORY"]="1"
|
|
try:out=run("zip-no-memory",[zip])
|
|
finally:del os.environ["ARCHIVE_NO_MEMORY"]
|
|
assert out[0]=="failed" and out[2]=="Not enough memory to extract.",out
|
|
assert not (root/"zip-no-memory").exists() and not (root/"zip-no-memory.extracting").exists()
|
|
# Real password-protected ZIP: correct password, incorrect/missing password, and ordered volumes.
|
|
protected=root/"protected.zip";payload=root/"protected.bin";payload.write_bytes(DATA)
|
|
subprocess.run(["/usr/bin/zip","-q","-P","DLPSGAME.COM",str(protected),payload.name],cwd=root,check=True)
|
|
assert run("zip-password",[protected],password="DLPSGAME.COM")[0]=="completed"
|
|
assert (root/"zip-password/protected.bin").read_bytes()==DATA
|
|
for name,password in [("wrong-password","wrong"),("missing-password",None)]:
|
|
assert run(name,[protected],password=password)[0]=="failed"
|
|
assert protected.exists() and not (root/name).exists() and not (root/(name+".extracting")).exists()
|
|
blob=protected.read_bytes();protected_parts=[]
|
|
for i in range(3):
|
|
volume=root/f"protected-part{i}";volume.write_bytes(blob[i*len(blob)//3:(i+1)*len(blob)//3]);protected_parts.append(volume)
|
|
assert run("zip-password-volumes",protected_parts,password="DLPSGAME.COM")[0]=="completed"
|
|
assert (root/"zip-password-volumes/protected.bin").read_bytes()==DATA
|
|
for headers in [True,False]:
|
|
name="rar5-encrypted-headers" if headers else "rar5-encrypted-data"
|
|
rar=root/(name+".rar");rar.write_bytes(encrypted_rar5(DATA,headers=headers)[0])
|
|
result=run(name,[rar],password="DLPSGAME.COM")
|
|
assert result[0]=="completed",result
|
|
assert (root/name/"game.ffpfsc").read_bytes()==DATA
|
|
result=run(name+"-wrong",[rar],password="wrong")
|
|
assert result[0]=="failed" and "password" in result[-1].lower(),result
|
|
assert rar.exists() and not (root/(name+"-wrong")).exists()
|
|
for headers in [True,False]:
|
|
name="rar5-quick-open-headers" if headers else "rar5-quick-open-data"
|
|
rar=root/(name+".rar");blob=encrypted_rar5(DATA,headers=headers,quick_open=True)[0];rar.write_bytes(blob)
|
|
result=run(name,[rar],password="DLPSGAME.COM")
|
|
assert result[0]=="completed",result
|
|
assert (root/name/"game.ffpfsc").read_bytes()==DATA
|
|
assert not any(p.name=="QO" for p in (root/name).rglob("*"))
|
|
if headers:
|
|
parts=[]
|
|
for i in range(3):
|
|
part=root/f"{name}-part-{i}";part.write_bytes(blob[i*len(blob)//3:(i+1)*len(blob)//3]);parts.append(part)
|
|
result=run(name+"-volumes",parts,password="DLPSGAME.COM")
|
|
assert result[0]=="completed",result
|
|
assert (root/(name+"-volumes")/"game.ffpfsc").read_bytes()==DATA
|
|
# The ignored service payload must still be complete; never accept a truncated input.
|
|
truncated=root/(name+"-truncated.rar");truncated.write_bytes(blob[:-100])
|
|
result=run(name+"-truncated",[truncated],password="DLPSGAME.COM")
|
|
assert result[0]=="failed" and not (root/(name+"-truncated")).exists(),result
|
|
# A real file called QO must retain password authentication.
|
|
rar=root/"qo-file.rar";rar.write_bytes(encrypted_rar5(DATA,name="QO",password="other",headers=False)[0])
|
|
assert run("qo-file",[rar],password="DLPSGAME.COM")[0]=="failed"
|
|
rar_parts=[]
|
|
for i,blob in enumerate(encrypted_rar5(DATA,volumes=2)):
|
|
volume=root/f"encrypted-volume-{i}.rar";volume.write_bytes(blob);rar_parts.append(volume)
|
|
result=run("rar5-password-volumes",rar_parts,password="DLPSGAME.COM")
|
|
assert result[0]=="completed",result
|
|
assert (root/"rar5-password-volumes/game.ffpfsc").read_bytes()==DATA
|
|
# Real `rar a -ma5 -m3 -hp -v20k` output: compressed blocks and the one AES-CBC stream of the file
|
|
# cross volumes at unaligned offsets.
|
|
real=[uudecode(source,root/source.stem) for source in sorted((ROOT/"tools/tests/archives").glob("rar5_encrypted_volumes.part*.rar.uu"))]
|
|
result=run("rar5-real-volumes",real,password="fixture-password")
|
|
assert result[0]=="completed",result
|
|
assert (root/"rar5-real-volumes/game.bin").read_bytes()==words()
|
|
corrupt=root/"corrupt-encrypted.rar"
|
|
damaged=bytearray(encrypted_rar5(DATA)[0]);damaged[-64]^=1
|
|
corrupt.write_bytes(damaged)
|
|
result=run("rar5-corrupt",[corrupt],password="DLPSGAME.COM")
|
|
assert result[0]=="failed" and "checksum" in result[-1].lower(),result
|
|
assert corrupt.exists() and not (root/"rar5-corrupt").exists()
|
|
empty=root/"empty-encrypted.rar";empty.write_bytes(encrypted_rar5(b"",compression=4<<7|8<<10)[0])
|
|
result=run("rar5-empty",[empty],password="DLPSGAME.COM");assert result[0]=="completed",result
|
|
assert (root/"rar5-empty/game.ffpfsc").read_bytes()==b""
|
|
for name,kwargs in [("rar5-encrypted-window",{"compression":3<<7|9<<10}),
|
|
("rar5-encrypted-traversal",{"name":"../escape"}),
|
|
("rar5-keyed-checksum",{"crypto_flags":3})]:
|
|
rar=root/(name+".rar");rar.write_bytes(encrypted_rar5(DATA,**kwargs)[0])
|
|
result=run(name,[rar],password="DLPSGAME.COM")
|
|
assert result[0]=="failed",result
|
|
assert not (root/name).exists() and rar.exists()
|
|
# A non-ASCII name in a process left in the C locale, as on the console (the Teardown backport).
|
|
named=root/"named.zip"
|
|
with zipfile.ZipFile(named,"w") as z:z.writestr("PPSA15246 \u2013 USA/eboot.bin",DATA)
|
|
out=subprocess.check_output([str(binary),str(root/"named"),str(1024*1024),str(named)],text=True,timeout=20,env={"LC_ALL":"C"}).splitlines()
|
|
assert out[0]=="completed",out
|
|
assert (root/"named/PPSA15246 \u2013 USA/eboot.bin").read_bytes()==DATA
|
|
tar=root/"fixture.tar"
|
|
with tarfile.open(tar,"w") as t:
|
|
entry=tarfile.TarInfo("nested/file.bin");entry.size=len(DATA);t.addfile(entry,io.BytesIO(DATA))
|
|
assert run("tar",[tar])[0]=="completed"
|
|
# rar-extract: images published at the set root, unsafe names and links refused, a missing middle
|
|
# volume named as such.
|
|
p=root/"plain.rar";p.write_bytes(plain_rar5("nested/deeper/game.ffpfsc",DATA))
|
|
assert run("rar-plain",[p])[0]=="completed"
|
|
assert (root/"rar-plain/game.ffpfsc").read_bytes()==DATA
|
|
# The phase goes from extracting to finishing once every entry is decoded, and a completed task ends in it:
|
|
# through the write ring (files synced of the total), inline writes, rar-extract's "s" line, and an older
|
|
# rar-extract without that line.
|
|
def phases(name,parts,password=None,**env):
|
|
done=subprocess.run([str(binary),str(root/name),str(1024*1024),*map(str,parts),*([] if password is None else ["--password",password])],capture_output=True,text=True,timeout=20,env={**os.environ,"ARCHIVE_PHASES":"1","ARCHIVE_TRACE":"1",**env})
|
|
seen=[line[6:] for line in done.stderr.splitlines() if line.startswith("phase ")]
|
|
assert done.stdout.splitlines()[0]=="completed" and seen[-1]=="finishing" and seen[0] in ("extracting","finishing"),(done.stdout,seen)
|
|
return done.stderr
|
|
several=root/"several.zip"
|
|
with zipfile.ZipFile(several,"w") as z:
|
|
for i in range(5):z.writestr(f"dir/file{i}.bin",DATA[i:i+100000])
|
|
assert "synced 5/5" in phases("phase-ring",[several])
|
|
assert "synced 0/0" in phases("phase-inline",[several],ARCHIVE_PIPELINE_BYTES="0")
|
|
rar_phases=phases("phase-rar",[root/"plain.rar"])
|
|
# rar-extract reads what the set unpacks to from its headers before extracting; libarchive cannot.
|
|
assert "rar worker finishing" in rar_phases and f"total {len(DATA)}" in rar_phases
|
|
assert "total 0" in phases("phase-total-zip",[several])
|
|
older=root/"older-worker";older.write_text(f"#!/bin/sh\n'{os.environ['PS5_REACT_RAR_WORKER']}' | grep --line-buffered -vx s\n");older.chmod(0o755)
|
|
assert "rar worker finishing" not in phases("phase-older-rar",[root/"plain.rar"],PS5_REACT_RAR_WORKER=str(older))
|
|
assert (root/"phase-older-rar/game.ffpfsc").read_bytes()==DATA
|
|
for name,entry,link in [("rar-traversal","../escape",None),("rar-absolute","/escape",None),("rar-backslash","a\\..\\..\\escape",None),
|
|
("rar-dotdot-inner","a/../../escape",None),("rar-reserved","x/.ps5-react-extraction",None),("rar-symlink","link","../escape")]:
|
|
p=root/(name+".rar");p.write_bytes(plain_rar5(entry,DATA,link))
|
|
result=run(name,[p])
|
|
assert result[0]=="failed" and "unsafe" in result[-1],result
|
|
assert not (root/name).exists() and not (root/(name+".extracting")).exists() and not (root/"escape").exists()
|
|
real=sorted(root.glob("rar5_encrypted_volumes.part*.rar"))
|
|
result=run("rar-missing-volume",[real[0],real[1],real[3]],password="fixture-password")
|
|
assert result[0]=="failed" and "missing volume" in result[-1],result
|
|
result=run("rar-no-password",real)
|
|
assert result[0]=="failed" and "password" in result[-1],result
|
|
# Headers encrypted, one file across four volumes: counted once, its whole size.
|
|
assert f"total {len(words())}" in phases("rar-real-total",real,password="fixture-password")
|
|
# A volume cut short is named, so the app downloads that one again.
|
|
cut=[root/f"cut-{i}.rar" for i in range(len(real))]
|
|
for i,copy in enumerate(cut):copy.write_bytes(real[i].read_bytes())
|
|
cut[1].write_bytes(cut[1].read_bytes()[:-3000])
|
|
result=run("rar-cut-volume",cut,password="fixture-password")
|
|
assert result[0]=="failed" and result[-1].endswith("Volume 2 of 4 looks incomplete."),result
|
|
assert "Volume 3 of 3 looks incomplete." in run("rar-missing-volume-named",[real[0],real[1],real[3]],password="fixture-password")[-1]
|
|
# A set without its last or first volume fails before anything is decoded, with exactly these reasons
|
|
# (the app maps them): RAR5 and RAR4, headers encrypted or not, .partN.rar or .rar/.r00 volumes.
|
|
def worker(parts,password=None):
|
|
"""rar-extract's own report lines, without progress and timings."""
|
|
dest=Path(tempfile.mkdtemp(dir=root))
|
|
request="RARX1\nwindow 33554432\nlimit 1048576\n"+("" if password is None else f"password {password.encode().hex()}\n")
|
|
request+=f"dest {dest}\n"+"".join(f"source {p}\n" for p in parts)+"end\n"
|
|
# Its input stays open: the worker reads a closed one as the engine gone.
|
|
process=subprocess.Popen([os.environ["PS5_REACT_RAR_WORKER"]],stdin=subprocess.PIPE,stdout=subprocess.PIPE,text=True)
|
|
process.stdin.write(request);process.stdin.flush()
|
|
out=process.stdout.read();process.stdin.close();process.wait(timeout=20)
|
|
return [line for line in out.splitlines() if line[:2] not in ("p ","t ") and line!="k"]
|
|
rar4=[uudecode(ROOT/f"tools/tests/archives/rar4_encrypted_volumes.{ext}.uu",root/f"rar4_encrypted_volumes.{ext}") for ext in ("rar","r00","r01","r02")]
|
|
rar5_open=[]
|
|
for i,blob in enumerate(encrypted_rar5(DATA,headers=False,volumes=4)):
|
|
volume=root/f"rar5-open-headers.part{i+1}.rar";volume.write_bytes(blob);rar5_open.append(volume)
|
|
sets=[("rar5-encrypted",real,"fixture-password",words()),("rar4-encrypted",rar4,"fixture-password",words()),
|
|
("rar5-open-headers",rar5_open,"DLPSGAME.COM",DATA)]
|
|
sets.append(("rar4-open",[uudecode(source,root/source.stem) for source in sorted((ROOT/"tools/tests/archives").glob("test_rar_multivolume_single_file.part*.rar.uu"))],None,None))
|
|
for name,parts,password,content in sets:
|
|
result=run(name+"-complete",parts,password=password)
|
|
assert result[0]=="completed",(name,result)
|
|
if content is not None:assert next((root/(name+"-complete")).rglob("game.*")).read_bytes()==content
|
|
assert worker(parts[:-1],password)==["ready","fail "+NO_LAST],(name,worker(parts[:-1],password))
|
|
assert run(name+"-no-last",parts[:-1],password=password)[-1]==NO_LAST
|
|
assert worker(parts[1:],password)==["ready","fail "+NO_FIRST],(name,worker(parts[1:],password))
|
|
assert run(name+"-no-first",parts[1:],password=password)[-1]==NO_FIRST
|
|
assert not (root/(name+"-no-last")).exists() and not (root/(name+"-no-first")).exists()
|
|
# Headers it cannot read leave the reason to the extraction: a wrong or missing password stays one. RAR4
|
|
# has no password check, so a wrong one reads as damaged headers, as before.
|
|
for password in ["wrong",None]:
|
|
for parts,rar5 in [(real[:-1],True),(rar4[:-1],False)]:
|
|
result=run("rar-check-password",parts,password=password)
|
|
expected="checksum" if password and not rar5 else "password"
|
|
assert result[0]=="failed" and expected in result[-1].lower(),result
|
|
# Single volumes: RAR4 with and without an end block, and one cut short.
|
|
for name,blob in [("rar4-single",plain_rar4("game.bin",DATA)),("rar4-single-no-end",plain_rar4("game.bin",DATA,end=False))]:
|
|
single=root/(name+".rar");single.write_bytes(blob)
|
|
assert run(name,[single])[0]=="completed"
|
|
assert (root/name/"game.bin").read_bytes()==DATA
|
|
# A whole RAR5 volume without an end block was not the last one; an archive that is no volume set is.
|
|
last=root/"rar5-open-no-end.rar";last.write_bytes(rar5_open[-1].read_bytes()[:-8])
|
|
assert worker([*rar5_open[:-1],last],"DLPSGAME.COM")==["ready","fail "+NO_LAST]
|
|
single=root/"rar5-single-no-end.rar";single.write_bytes(encrypted_rar5(DATA,headers=False)[0][:-8])
|
|
assert run("rar5-single-no-end",[single],password="DLPSGAME.COM")[0]=="completed"
|
|
assert (root/"rar5-single-no-end/game.ffpfsc").read_bytes()==DATA
|
|
# A header damaged within the last volume is not a cut: the shorter volume 2 is still the one named.
|
|
damaged=[root/f"rar5-junk-{i}.rar" for i in range(4)]
|
|
for i,copy in enumerate(damaged):copy.write_bytes(rar5_open[i].read_bytes())
|
|
last=damaged[3].read_bytes();damaged[3].write_bytes(last[:-8]+bytes(4)+bytes([3,9,0,0])+last[-8:])
|
|
damaged[1].write_bytes(damaged[1].read_bytes()[:-3000])
|
|
result=run("rar5-junk",damaged,password="DLPSGAME.COM")
|
|
assert result[0]=="failed" and result[-1].endswith("Volume 2 of 4 looks incomplete."),result
|
|
# A last volume cut short (a download stopped early) is not a missing one: it fails as before, named, so
|
|
# the app downloads that volume again. Cut in its data, in a header, and in the end block.
|
|
for name,parts,password in [("rar5-encrypted",real,"fixture-password"),("rar4-encrypted",rar4,"fixture-password"),("rar5-open-headers",rar5_open,"DLPSGAME.COM")]:
|
|
for cut in (len(parts[-1].read_bytes())//2,40,3):
|
|
copy=root/f"{name}-cut-{cut}";copy.write_bytes(parts[-1].read_bytes()[:-cut])
|
|
result=run(f"{name}-cut-last-{cut}",[*parts[:-1],copy],password=password)
|
|
# Losing only the end block may still leave every file whole, its checksum verified.
|
|
named=result[0]=="failed" and result[-1].endswith(f"Volume {len(parts)} of {len(parts)} looks incomplete.")
|
|
assert named or (result[0]=="completed" and cut<64),(name,cut,result)
|
|
single=root/"rar4-single-cut.rar";single.write_bytes(plain_rar4("game.bin",DATA,end=False)[:-100])
|
|
result=run("rar4-single-cut",[single]);assert result[0]=="failed" and result[-1]!=NO_LAST,result
|
|
single=root/"rar5-single-cut.rar";single.write_bytes(encrypted_rar5(DATA)[0][:-100])
|
|
result=run("rar5-single-cut",[single],password="DLPSGAME.COM");assert result[0]=="failed" and result[-1]!=NO_LAST,result
|
|
# Real archive split byte stream: ordered volumes, no temporary joined copy.
|
|
blob=tar.read_bytes();parts=[]
|
|
for i in range(3):
|
|
p=root/f"tar-part{i}";p.write_bytes(blob[i*len(blob)//3:(i+1)*len(blob)//3]);parts.append(p)
|
|
assert run("split-tar",parts)[0]=="completed"
|
|
assert (root/"split-tar/nested/file.bin").read_bytes()==DATA
|
|
plain=root/"payload.bin";plain.write_bytes(DATA)
|
|
seven=root/"fixture.7z"
|
|
subprocess.run([prefix+"/bin/bsdtar","--format=7zip","-cf",str(seven),"-C",str(root),"payload.bin"],check=True)
|
|
assert run("7zip",[seven])[0]=="completed"
|
|
assert (root/"7zip/payload.bin").read_bytes()==DATA
|
|
blob=seven.read_bytes();seven_parts=[]
|
|
for i in range(3):
|
|
p=root/f"seven-part{i}";p.write_bytes(blob[i*len(blob)//3:(i+1)*len(blob)//3]);seven_parts.append(p)
|
|
result=run("split-7zip",seven_parts)
|
|
assert result[0]=="completed",result
|
|
assert (root/"split-7zip/payload.bin").read_bytes()==DATA
|
|
# Decoder windows above the console budget are refused before libarchive allocates them.
|
|
big=root/"big.7z"
|
|
subprocess.run([prefix+"/bin/bsdtar","--format=7zip","--options=7zip:compression=lzma2,7zip:compression-level=9","-cf",str(big),"-C",str(root),"payload.bin"],check=True)
|
|
# 7-Zip ultra's 64 MiB dictionary is the budget itself.
|
|
assert run("big-7zip",[big])[0]=="completed"
|
|
encoded=root/"encoded.7z";encoded.write_bytes(encode_7z_header(seven.read_bytes()))
|
|
result=run("encoded-7zip",[encoded])
|
|
assert result[0]=="completed",result
|
|
assert (root/"encoded-7zip/payload.bin").read_bytes()==DATA
|
|
encoded_big=root/"encoded-big.7z";encoded_big.write_bytes(encode_7z_header(big.read_bytes()))
|
|
assert run("encoded-big-7zip",[encoded_big])[0]=="completed"
|
|
p=root/"preset9.tar.xz";p.write_bytes(lzma.compress(tar.read_bytes(),preset=9))
|
|
assert run("xz9",[p])[0]=="completed"
|
|
p=root/"huge.tar.xz"
|
|
p.write_bytes(lzma.compress(tar.read_bytes(),format=lzma.FORMAT_XZ,filters=[{"id":lzma.FILTER_LZMA2,"dict_size":128<<20}]))
|
|
assert run("xz-huge",[p])[1:]==["0",TOO_BIG]
|
|
assert not (root/"xz-huge").exists() and not (root/"xz-huge.extracting").exists()
|
|
p=root/"lzma.zip"
|
|
with zipfile.ZipFile(p,"w",compression=zipfile.ZIP_LZMA) as z:z.writestr("file.bin",DATA)
|
|
assert run("zip-lzma",[p])[0]=="completed"
|
|
p=root/"window.rar";p.write_bytes(rar5_with_window(9))
|
|
assert run("rar5-window",[p])[1:]==["0",TOO_BIG]
|
|
# WinRAR 7's default 32 MB dictionary, which libarchive doubles to the 64 MiB budget.
|
|
p=root/"small-window.rar";p.write_bytes(rar5_with_window(8))
|
|
assert run("rar5-small-window",[p])[2]!=TOO_BIG
|
|
# A sparse member arrives out of order, so its digest comes from reading the file back.
|
|
sparse=root/"sparse.bin"
|
|
with sparse.open("wb") as f:f.write(b"head"*1000);f.seek(8*1024*1024);f.write(b"tail"*1000)
|
|
p=root/"sparse.tar";subprocess.run([prefix+"/bin/bsdtar","--read-sparse","-cf",str(p),"-C",str(root),"sparse.bin"],check=True)
|
|
assert run("sparse",[p],16*1024*1024)[0]=="completed"
|
|
assert (root/"sparse/.ps5-react-extraction").read_text().splitlines()[1]==hashlib.sha256(sparse.read_bytes()).hexdigest()+"|sparse.bin"
|
|
assert run("missing-part",parts[:1])[0]=="failed"
|
|
assert not (root/"missing-part").exists()
|
|
rar_parts=[]
|
|
for source in sorted((ROOT/"tools/tests/archives").glob("test_rar_multivolume_single_file.part*.rar.uu")):
|
|
rar_parts.append(uudecode(source,root/source.stem))
|
|
assert len(rar_parts)==3
|
|
result=run("rar-multipart",rar_parts)
|
|
assert result[0]=="completed",result
|
|
assert any(p.is_file() and p.stat().st_size>0 for p in (root/"rar-multipart").rglob("*") if p.name!=".ps5-react-extraction")
|
|
open_to_all(root/"rar-multipart")
|
|
# A set extracted while it downloads: each later volume appears under its final name by a rename, over
|
|
# the empty file that reserves the name on the console.
|
|
def stream(name,blobs,gap,extra=()):
|
|
names=[root/f"{name}-v{i}" for i in range(len(blobs))]
|
|
names[0].write_bytes(blobs[0])
|
|
process=subprocess.Popen([str(binary),str(root/name),str(1024*1024),*map(str,names),"--stream",*extra],stdout=subprocess.PIPE,text=True)
|
|
for i,blob in enumerate(blobs[1:],1):
|
|
if gap is None:break
|
|
time.sleep(gap);partial=root/f"{name}-v{i}.part";partial.write_bytes(blob)
|
|
names[i].touch();time.sleep(gap);os.rename(partial,names[i])
|
|
return process.communicate(timeout=30)[0].splitlines(),names
|
|
blobs=[p.read_bytes() for p in rar_parts]
|
|
result,names=stream("rar-stream",blobs,0.6)
|
|
assert result[0]=="completed",result
|
|
tree=lambda d:{str(p.relative_to(d)):p.read_bytes() for p in d.rglob("*") if p.is_file() and not p.name.startswith(".ps5-react-")}
|
|
assert tree(root/"rar-stream")==tree(root/"rar-multipart"),(sorted(tree(root/"rar-stream")),sorted(tree(root/"rar-multipart")))
|
|
# Its receipt names every volume, as a run after the download writes it: a later run recovers it.
|
|
assert run("rar-stream",names)[0]=="completed"
|
|
started=time.monotonic();result,_=stream("rar-stream-cancel",blobs,None,["--cancel-after","700"])
|
|
assert result[0]=="cancelled" and time.monotonic()-started<5,result
|
|
assert not (root/"rar-stream-cancel").exists() and not (root/"rar-stream-cancel.extracting").exists()
|
|
damaged=list(blobs);damaged[1]=damaged[1][:4000]+bytes(len(damaged[1])-4000)
|
|
result,_=stream("rar-stream-damaged",damaged,0.3)
|
|
assert result[0]=="failed",result
|
|
assert not (root/"rar-stream-damaged").exists() and not (root/"rar-stream-damaged.extracting").exists()
|
|
# Extracting while downloading: the first volume is checked at once, the last one as soon as it is there,
|
|
# however far decoding is.
|
|
waiting=[root/f"stream-no-last-v{i}" for i in range(4)];waiting[0].write_bytes(real[0].read_bytes())
|
|
process=subprocess.Popen([str(binary),str(root/"stream-no-last"),str(1024*1024),*map(str,waiting),"--stream","--password","fixture-password"],stdout=subprocess.PIPE,text=True)
|
|
time.sleep(1);partial=root/"stream-no-last.part";partial.write_bytes(real[2].read_bytes());os.rename(partial,waiting[3])
|
|
started=time.monotonic();out=process.communicate(timeout=20)[0].splitlines()
|
|
assert out[0]=="failed" and out[-1]==NO_LAST and time.monotonic()-started<3,out
|
|
result,_=stream("stream-no-first",[p.read_bytes() for p in real[1:]],None,["--password","fixture-password"])
|
|
assert result[0]=="failed" and result[-1]==NO_FIRST,result
|
|
result,_=stream("zip-stream",[zip.read_bytes()],None)
|
|
assert result[0]=="failed" and "RAR set" in result[-1],result
|
|
# A worker that goes silent may still write into its staging: the next extraction stages beside it,
|
|
# and a later launch removes what it left once the set is extracted.
|
|
silent=root/"silent-worker";silent.write_text("#!/bin/sh\nexec sleep 60\n");silent.chmod(0o755)
|
|
real=os.environ["PS5_REACT_RAR_WORKER"];os.environ["PS5_REACT_RAR_WORKER"]=str(silent)
|
|
try:out=subprocess.check_output([str(binary),str(root/"rar-silent"),str(1024*1024),*map(str,rar_parts),"--cancel-after","100","--again",real],text=True,timeout=60).splitlines()
|
|
finally:os.environ["PS5_REACT_RAR_WORKER"]=real
|
|
assert out[0]=="cancelled" and out[3]=="completed",out
|
|
assert (root/"rar-silent.extracting/.ps5-react-stage").exists() and not (root/"rar-silent.extracting-1").exists()
|
|
assert tree(root/"rar-silent")==tree(root/"rar-multipart")
|
|
assert run("rar-silent",rar_parts)[0]=="completed" and not (root/"rar-silent.extracting").exists()
|
|
# A process interruption leaves owned staging; the next extraction restarts safely.
|
|
st=zip.stat();identity=f"{zip}|{st.st_dev}|{st.st_ino}|{st.st_size}|{st.st_mtime_ns//10**9}|{st.st_mtime_ns%10**9}\n"
|
|
staging=root/"interrupted.extracting";staging.mkdir();(staging/"unfinished").write_bytes(b"partial")
|
|
(staging/".ps5-react-stage").write_text("P5ST001:"+hashlib.sha256(identity.encode()).hexdigest()+"\n")
|
|
assert run("interrupted",[zip])[0]=="completed"
|
|
assert (root/"interrupted/game.ffpfsc").read_bytes()==DATA
|
|
orphan=root/"orphan.extracting";orphan.mkdir();(orphan/"keep").write_bytes(b"unowned")
|
|
assert run("orphan",[zip])[0]=="failed"
|
|
assert (orphan/"keep").read_bytes()==b"unowned"
|
|
# A crash before the ownership record leaves empty staging, which the next extraction takes over.
|
|
(root/"abandoned.extracting").mkdir()
|
|
assert run("abandoned",[zip])[0]=="completed"
|
|
assert run("limit",[zip],100)[0]=="failed"
|
|
assert not (root/"limit").exists()
|
|
for name,path in [("traversal","../escape"),("absolute","/escape"),("drive","C:/escape")]:
|
|
p=root/f"{name}.zip"
|
|
with zipfile.ZipFile(p,"w") as z:z.writestr(path,b"bad")
|
|
assert run(name,[p])[0]=="failed"
|
|
assert not (root/name).exists()
|
|
p=root/"symlink.tar"
|
|
with tarfile.open(p,"w") as t:
|
|
entry=tarfile.TarInfo("link");entry.type=tarfile.SYMTYPE;entry.linkname="../escape";t.addfile(entry)
|
|
assert run("symlink",[p])[0]=="failed"
|
|
assert not (root/"symlink").exists()
|
|
assert run("zip",[zip])[0]=="completed" # Owned output is hash-verified on recovery.
|
|
assert (root/"zip/game.ffpfsc").read_bytes()==DATA
|
|
(root/"zip/game.ffpfsc").write_bytes(b"corrupt")
|
|
assert run("zip",[zip])[0]=="failed"
|
|
# Inspection names a file by its bytes and refuses a too-large window from a download's first bytes.
|
|
def inspect(*paths,budget=None):
|
|
args=[] if budget is None else ["--budget",str(budget)]
|
|
return subprocess.check_output([str(binary),"inspect",*args,*map(str,paths)],text=True,timeout=20).split("\n")[:2]
|
|
def partial(name,content,keep,hole=0):
|
|
p=root/name;p.write_bytes(content[:keep]+bytes(hole));return p
|
|
big_rar=rar5_with_window(9)
|
|
assert inspect(partial("big.rar.part",big_rar,len(big_rar)-40,1<<20))==["rar",TOO_BIG]
|
|
small_rar=rar5_with_window(8)
|
|
assert inspect(partial("small.rar.part",small_rar,len(small_rar)-40,1<<20))==["rar",""]
|
|
# The walk reads the first file header whatever its budget, and stops after it once the budget is spent;
|
|
# within its budget it reaches a window declared thousands of files in.
|
|
assert inspect(partial("big.rar.part",big_rar,len(big_rar)-40,1<<20),budget=0)==["rar",TOO_BIG]
|
|
late=rar5_with_window(*[8]*5000,9);late_rar=partial("late.rar.part",late,len(late)-40,1<<20)
|
|
assert inspect(late_rar,budget=0)==["rar",""]
|
|
assert inspect(late_rar,budget=60000)==["rar",TOO_BIG]
|
|
assert inspect(rar_parts[0])==["rar",""]
|
|
assert inspect(partial("hole.part",big_rar,0,1<<20))==["",""]
|
|
assert inspect(partial("zip.part",zip.read_bytes(),200))==["zip",""]
|
|
assert inspect(partial("7z.part",seven.read_bytes(),40))==["7z",""]
|
|
assert inspect(tar)==["tar",""]
|
|
gz=root/"fixture.tar.gz";gz.write_bytes(__import__("gzip").compress(tar.read_bytes()))
|
|
assert inspect(gz)==["tar.gz",""]
|
|
assert inspect(partial("huge.tar.xz.part",(root/"huge.tar.xz").read_bytes(),4096))==["tar.xz",TOO_BIG]
|
|
assert inspect(partial("game.pkg.part",b"\x7fCNT"+bytes(64),68))==["pkg",""]
|
|
assert inspect(partial("game.exfat",b"\xeb\x76\x90EXFAT "+bytes(500),512))==["exfat",""]
|
|
ufs=bytearray(70000);ufs[65536+1372:65536+1376]=struct.pack("<I",0x19540119)
|
|
assert inspect(partial("game.ffpkg",bytes(ufs),len(ufs)))==["ffpkg",""]
|
|
assert inspect(partial("game.ffpfs",struct.pack("<QQ",1,0x20130315)+bytes(64),80))==["ffpfs",""]
|
|
assert inspect(partial("game.ffpfsc",b"PFSC"+bytes(64),68))==["ffpfsc",""]
|
|
print("PASS: header inspection of partial downloads; ZIP/TAR/7z/multivolume RAR, decoder memory refusal, interrupted staging, verified recovery, missing parts and unsafe archive rejection")
|
|
if __name__=="__main__":main()
|