Merge environment bundles (.fbundle); track astcenc's CMakeLists.txt and LICENSE.txt

astcenc's own .gitignore excluded them, so a fresh clone could not configure.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
ToastconcernandClaude Opus 5.5 committed 2026-10-21 12:27:01 -05:00
commit 95ccea95ca
11 files changed
+1106 -18

No files matched your search

+2 -2
View File
@@ -1,6 +1,6 @@
@echo off
rem Installs Frame Home on the Steam Frame (no Frame Studio needed). Double-click it, or drop .fenv environments on
rem it to copy them to the headset too. Asks for the headset's address the first time (default: frame).
rem Installs Frame Home on the Steam Frame (no Frame Studio needed). Double-click it, or drop .fenv environments or
rem .fbundle bundles on it to copy them to the headset too. Asks for the headset's address the first time (default: frame).
setlocal
set "HOSTFILE=%APPDATA%\FrameStudio\frame-host.txt"
set "FRAMEHOST="
+13 -2
View File
@@ -14,12 +14,15 @@ What you need
Install from a Windows PC
-------------------------
Double-click "Install Frame Home.bat" and type the headset's address (Enter keeps the one shown).
To copy environments to the headset at the same time, drop .fenv files onto "Install Frame Home.bat".
To copy environments to the headset at the same time, drop .fenv files (or .fbundle bundles) onto
"Install Frame Home.bat".
(Needs Windows' OpenSSH Client, under Settings > Apps > Optional features, or Git for Windows.)
Install from Linux or macOS
---------------------------
sh install-frame-home.sh --host steamos@frame [environment.fenv ...]
sh install-frame-home.sh --host steamos@frame [environment.fenv | bundle.fbundle ...]
The first .fenv given is the one shown.
Install on the headset itself
-----------------------------
@@ -29,6 +32,14 @@ Copy this folder to the headset (Desktop Mode), open a terminal (Konsole) in it
Put .fenv environments in ~/frame-home/envs; the Environments panel in the Steam menu lists them.
Bundles
-------
A .fbundle holds several environments in one file, to share or install together (Frame Studio or
"fenv-tool bundle" makes them). Copied into ~/frame-home/envs (the installers do that), it is unpacked there by the
Environments panel into its .fenv environments, each listed on its own, and then deleted. An environment already
there is kept: the same one is skipped, a different one of the same name gets the bundle's as "name 2". A bundle
never changes which environment is shown.
Uninstall
---------
sh install-frame-home.sh --uninstall (Linux / macOS)
+9 -5
View File
@@ -2,11 +2,13 @@
# Frame > Install on Steam Frame. Needs ssh access to the headset (its developer key, or a password) and an ssh
# client: Windows' own OpenSSH (Settings > Optional features) or Git for Windows'.
#
# powershell -ExecutionPolicy Bypass -File install-frame-home.ps1 [-HostName HOST] [-Uninstall] [environment.fenv ...]
# powershell -ExecutionPolicy Bypass -File install-frame-home.ps1 [-HostName HOST] [-Uninstall] [environment.fenv | bundle.fbundle ...]
#
# HOST: the headset's ssh address (default "steamos@frame": an entry named frame in ~/.ssh/config, or the headset's
# name on the network; an IP works too, e.g. steamos@192.168.1.50). Environments (.fenv) given are copied into its
# selector as well; the first is the one shown. -Uninstall gives the environment back to SteamVR (files stay).
# selector as well; the first .fenv is the one shown. Bundles (.fbundle) are copied as they are and the headset's
# Environments panel unpacks them into their environments (they never change the one shown). -Uninstall gives the
# environment back to SteamVR (files stay).
param(
[string]$HostName = "steamos@frame",
[switch]$Uninstall,
@@ -64,9 +66,11 @@ if ($Uninstall) {
}
& $ssh $HostName "sed -i 's/\r`$//' frame-home/.install/frame/install.sh; sh frame-home/.install/frame/install.sh"
if ($LASTEXITCODE -ne 0) { throw "The installer on the headset failed (see above)" }
if ($Environments.Count -gt 0 -and (Test-Path $Environments[0])) { # the first is the one shown
$first = Frame-Name $Environments[0]
& $ssh $HostName "printf '%s\n' '$first' > frame-home/current"
# The first environment given is the one shown (a bundle's are only known once the headset unpacks it: skipped).
$first = $Environments | Where-Object { $_ -match "\.fenv$" } | Select-Object -First 1
if ($first -and (Test-Path $first)) {
$name = Frame-Name $first
& $ssh $HostName "printf '%s\n' '$name' > frame-home/current"
}
}
Say "done"
+7 -5
View File
@@ -2,11 +2,13 @@
# Installs Frame Home on a Steam Frame from a Linux or macOS computer, without Frame Studio: the same steps as
# Studio's Frame > Install on Steam Frame. Needs ssh access to the headset (its developer key, or a password).
#
# sh install-frame-home.sh [--host HOST] [--uninstall] [environment.fenv ...]
# sh install-frame-home.sh [--host HOST] [--uninstall] [environment.fenv | bundle.fbundle ...]
#
# HOST: the headset's ssh address (default "steamos@frame": an entry named frame in ~/.ssh/config, or the
# headset's name on the network; an IP works too, e.g. steamos@192.168.1.50). Environments (.fenv) given are
# copied into its selector as well. --uninstall gives the environment back to SteamVR (files stay).
# copied into its selector as well; the first .fenv is the one shown. Bundles (.fbundle) are copied as they are and
# the headset's Environments panel unpacks them into their environments (they never change the one shown).
# --uninstall gives the environment back to SteamVR (files stay).
set -eu
HERE=$(cd "$(dirname "$0")" && pwd)
PAYLOAD="$HERE/payload"
@@ -17,7 +19,7 @@ while [ $# -gt 0 ]; do
case "$1" in
--host) HOST=$2; shift 2 ;;
--uninstall) MODE=uninstall; shift ;;
-h|--help) sed -n '2,10p' "$0"; exit 0 ;;
-h|--help) sed -n '2,11p' "$0"; exit 0 ;;
*) ENVS="$ENVS
$1"; shift ;;
esac
@@ -44,8 +46,8 @@ if [ "$MODE" = install ]; then
scp -q "$f" "$HOST:frame-home/envs/$name.part" && ssh "$HOST" "mv -f frame-home/envs/$name.part frame-home/envs/$name"
done
ssh "$HOST" "sed -i 's/\r\$//' frame-home/.install/frame/install.sh; sh frame-home/.install/frame/install.sh"
# the first environment given is the one shown
FIRST=$(printf '%s\n' "$ENVS" | sed -n '/./{p;q;}')
# the first environment given is the one shown (a bundle's are only known once the headset unpacks it: skipped)
FIRST=$(printf '%s\n' "$ENVS" | grep -i '\.fenv$' | sed -n '1p')
if [ -n "$FIRST" ] && [ -f "$FIRST" ]; then
name=$(basename "$FIRST" | tr 'A-Z ' 'a-z-' | tr -cd 'a-z0-9._-')
ssh "$HOST" "printf '%s\n' '$name' > frame-home/current"
+375
View File
@@ -0,0 +1,375 @@
#pragma once
// .fbundle — several Frame environments in one file, to share or install together. Each stays its own environment:
// a bundle is only the carrier. It is a ZIP holding bundle.json and the .fenv files as they are:
//
// { "format": "fbundle", "version": 1, "name": "Meta homes",
// "environments": [ { "file": "environments/haven.fenv", "name": "Haven + Calming", "size": 283115520 } ] }
//
// The .fenv members are STORED, not recompressed (their content is compressed already), and copied file to zip and
// zip to file in pieces, so a bundle of any size never passes through memory. A bundle of 4 GB or more is written as
// zip64 from the start (miniz only switches before a file that starts past 4 GB, not for one that ends past it, and
// then can't finish the archive). A member being stored means its bytes sit in the bundle as one run, so a member's
// own fenv.json and thumbnail are read in place, without unpacking anything.
//
// The Steam Frame's switcher unpacks a bundle dropped into ~/frame-home/envs into its environments (switcher.cpp
// unpackBundles); Frame Studio and fenv-tool write, list and unpack them.
#include "fenv.h"
#include <ctime>
#include <fstream>
struct FenvBundleEntry {
std::string file; // member path in the bundle: "environments/<name>.fenv"
std::string name; // the environment's display name (its fenv.json "name")
uint64_t size = 0; // bytes
};
struct FenvBundleManifest {
std::string name;
std::vector<FenvBundleEntry> environments;
};
// progress(bytesDone, bytesTotal), called as the environments' bytes are copied.
using FenvBundleProgress = std::function<void(uint64_t, uint64_t)>;
inline bool isFenvBundlePath(const std::string& p) {
return p.size() > 8 && (p.compare(p.size() - 8, 8, ".fbundle") == 0 || p.compare(p.size() - 8, 8, ".FBUNDLE") == 0);
}
namespace fenv_bundle_detail {
constexpr const char* kManifestName = "bundle.json";
inline std::string manifestJson(const FenvBundleManifest& m) {
std::string j = "{\n \"format\": \"fbundle\",\n \"version\": 1,\n \"name\": " + fenv_detail::quote(m.name) + ",\n \"environments\": [";
for (size_t i = 0; i < m.environments.size(); ++i) {
const FenvBundleEntry& e = m.environments[i];
j += std::string(i ? "," : "") + "\n { \"file\": " + fenv_detail::quote(e.file) + ", \"name\": " + fenv_detail::quote(e.name) +
", \"size\": " + std::to_string(e.size) + " }";
}
return j + "\n ]\n}\n";
}
inline bool parseManifest(const std::string& json, FenvBundleManifest& m) {
try {
const tinyjson::Value doc = tinyjson::parse(json);
if (!doc.isObject() || doc["format"].asString() != "fbundle") return false;
m = FenvBundleManifest{};
m.name = doc["name"].asString();
for (size_t i = 0; i < doc["environments"].size(); ++i) {
const auto& v = doc["environments"][i];
FenvBundleEntry e;
e.file = v["file"].asString();
e.name = v["name"].asString();
e.size = static_cast<uint64_t>(std::max<int64_t>(0, v["size"].asInt()));
if (fenv_detail::safeMember(e.file) && isFenvPath(e.file)) m.environments.push_back(e);
}
return !m.environments.empty();
} catch (...) {
return false;
}
}
// An opened bundle: the ZIP and its manifest; closes the ZIP when it goes.
struct OpenBundle {
mz_zip_archive zip = {};
bool open = false;
FenvBundleManifest manifest;
bool load(const std::string& path, std::string& error) {
if (!mz_zip_reader_init_file(&zip, path.c_str(), 0)) { error = "can't open " + path; return false; }
open = true;
const std::vector<uint8_t> j = fenv_detail::readMember(zip, kManifestName, uint64_t(1) << 20);
if (j.empty() || !parseManifest(std::string(j.begin(), j.end()), manifest)) { error = "not a Frame environment bundle (.fbundle): " + path; return false; }
return true;
}
// The member's index and stat, refusing anything that isn't stored whole (a damaged or foreign bundle).
bool member(const FenvBundleEntry& e, mz_uint& index, mz_zip_archive_file_stat& st, std::string& error) {
const int i = mz_zip_reader_locate_file(&zip, e.file.c_str(), nullptr, 0);
if (i < 0 || !mz_zip_reader_file_stat(&zip, static_cast<mz_uint>(i), &st)) { error = "the bundle has no " + e.file; return false; }
if (st.m_method != 0 || st.m_comp_size != st.m_uncomp_size || st.m_is_encrypted) { error = e.file + " isn't stored as-is in the bundle"; return false; }
index = static_cast<mz_uint>(i);
return true;
}
~OpenBundle() { if (open) mz_zip_reader_end(&zip); }
};
// A file read at a fixed offset: one stored member of the bundle, read as a ZIP of its own.
struct FileSlice {
std::ifstream file;
uint64_t start = 0;
static size_t read(void* opaque, mz_uint64 offset, void* buffer, size_t n) {
FileSlice& s = *static_cast<FileSlice*>(opaque);
s.file.clear();
s.file.seekg(static_cast<std::streamoff>(s.start + offset));
s.file.read(static_cast<char*>(buffer), static_cast<std::streamsize>(n));
return static_cast<size_t>(s.file.gcount());
}
};
// Where a stored member's bytes begin: after its local header (30 bytes, then its name and extra field, whose
// lengths the local header holds; they may differ from the central directory's).
inline bool memberDataOffset(const std::string& bundlePath, const mz_zip_archive_file_stat& st, uint64_t& offset) {
std::ifstream f(std::filesystem::u8path(bundlePath), std::ios::binary);
unsigned char header[30];
f.seekg(static_cast<std::streamoff>(st.m_local_header_ofs));
if (!f.read(reinterpret_cast<char*>(header), sizeof header)) return false;
if (header[0] != 'P' || header[1] != 'K' || header[2] != 3 || header[3] != 4) return false;
const unsigned nameLength = header[26] | (header[27] << 8), extraLength = header[28] | (header[29] << 8);
offset = st.m_local_header_ofs + 30 + nameLength + extraLength;
return true;
}
// Reads a .fenv's own file from disk, piece by piece, as the ZIP writer asks for it; counts the bytes for progress.
struct SourceReader {
std::ifstream file;
uint64_t* done = nullptr;
uint64_t total = 0;
const FenvBundleProgress* progress = nullptr;
static size_t read(void* opaque, mz_uint64 offset, void* buffer, size_t n) {
SourceReader& r = *static_cast<SourceReader*>(opaque);
r.file.clear(); // (after the last piece: reads 0, the writer's sign that it's all there)
r.file.seekg(static_cast<std::streamoff>(offset));
r.file.read(static_cast<char*>(buffer), static_cast<std::streamsize>(n));
const size_t got = static_cast<size_t>(r.file.gcount());
*r.done += got;
if (*r.progress) (*r.progress)(*r.done, r.total);
return got;
}
};
// Where a member's bytes go as they come out of the bundle: into `out` (when open), or compared against a file already
// there (`compare` open: equal means that environment is installed already); checksummed either way but the last.
struct MemberSink {
std::ofstream out;
std::ifstream compare;
bool same = true;
mz_uint32 crc = MZ_CRC32_INIT;
uint64_t* done = nullptr; // with progress: bytes so far, of `total`
uint64_t total = 0;
const FenvBundleProgress* progress = nullptr;
static size_t write(void* opaque, mz_uint64, const void* buffer, size_t n) {
MemberSink& s = *static_cast<MemberSink*>(opaque);
if (s.compare.is_open()) {
char theirs[65536];
for (size_t at = 0; at < n && s.same;) {
const size_t piece = std::min(n - at, sizeof theirs);
s.same = static_cast<bool>(s.compare.read(theirs, static_cast<std::streamsize>(piece))) &&
std::memcmp(theirs, static_cast<const char*>(buffer) + at, piece) == 0;
at += piece;
}
return s.same ? n : 0; // (a difference stops the extraction: the answer is known)
}
if (s.out.is_open() && !s.out.write(static_cast<const char*>(buffer), static_cast<std::streamsize>(n))) return 0;
s.crc = static_cast<mz_uint32>(mz_crc32(s.crc, static_cast<const unsigned char*>(buffer), n));
if (s.done) {
*s.done += n;
if (s.progress && *s.progress) (*s.progress)(*s.done, s.total);
}
return n;
}
};
// "haven.fenv", "haven 2.fenv", "haven 3.fenv"...: the first name `taken` doesn't refuse.
inline std::string freeName(const std::string& stem, const std::function<bool(const std::string&)>& taken) {
std::string name = stem + ".fenv";
for (int n = 2; taken(name); ++n) name = stem + " " + std::to_string(n) + ".fenv";
return name;
}
inline std::string lowercase(std::string s) {
for (char& c : s) if (c >= 'A' && c <= 'Z') c = static_cast<char>(c - 'A' + 'a');
return s;
}
} // namespace fenv_bundle_detail
// Packs the .fenv files into one bundle at outPath (written as outPath + ".part", renamed when whole). Every file
// must be a readable Frame environment; two of the same file name become "name" and "name 2". `name` is the bundle's
// own name (empty: outPath's file name).
inline bool fenvBundleWrite(const std::string& outPath, const std::vector<std::string>& fenvPaths, const std::string& name, std::string& error,
const FenvBundleProgress& progress = {}) {
namespace fs = std::filesystem;
using namespace fenv_bundle_detail;
if (fenvPaths.empty()) { error = "no environments to bundle"; return false; }
FenvBundleManifest manifest;
manifest.name = name.empty() ? fs::u8path(outPath).stem().u8string() : name;
std::vector<std::string> takenLowercase; // member names compared without case: they must unpack apart on Windows too
uint64_t total = 0;
for (const std::string& path : fenvPaths) {
std::error_code ec;
FenvManifest m;
if (!isFenvPath(path)) { error = "not a Frame environment (.fenv): " + path; return false; }
const uint64_t size = fs::file_size(fs::u8path(path), ec);
if (ec || !fenvReadManifest(path, m)) { error = "can't read the environment " + path; return false; }
FenvBundleEntry e;
const std::string member = freeName(fs::u8path(path).stem().u8string(), [&](const std::string& n) {
return std::find(takenLowercase.begin(), takenLowercase.end(), lowercase(n)) != takenLowercase.end();
});
takenLowercase.push_back(lowercase(member));
e.file = "environments/" + member;
e.name = m.name.empty() ? fs::u8path(path).stem().u8string() : m.name;
e.size = size;
total += size;
manifest.environments.push_back(e);
}
const std::string tmp = outPath + ".part", json = manifestJson(manifest);
const uint64_t headroom = json.size() + (uint64_t(1) << 20); // names, headers, the central directory
const mz_uint zip64 = total + headroom >= 0xFFFFFFFFull ? MZ_ZIP_FLAG_WRITE_ZIP64 : 0;
mz_zip_archive zip = {};
if (!mz_zip_writer_init_file_v2(&zip, tmp.c_str(), 0, zip64)) { error = "can't write " + outPath; return false; }
bool ok = mz_zip_writer_add_mem(&zip, kManifestName, json.data(), json.size(), MZ_DEFAULT_COMPRESSION);
if (!ok) error = "couldn't write " + outPath;
uint64_t done = 0;
const MZ_TIME_T now = std::time(nullptr);
for (size_t i = 0; ok && i < fenvPaths.size(); ++i) {
const FenvBundleEntry& e = manifest.environments[i];
SourceReader reader;
reader.file.open(fs::u8path(fenvPaths[i]), std::ios::binary);
reader.done = &done;
reader.total = total;
reader.progress = &progress;
const uint64_t before = done;
ok = reader.file.is_open() &&
mz_zip_writer_add_read_buf_callback(&zip, e.file.c_str(), &SourceReader::read, &reader, e.size, &now, nullptr, 0, MZ_NO_COMPRESSION,
nullptr, 0, nullptr, 0) &&
done - before == e.size; // (a file that changed size while it was copied)
if (!ok) error = "couldn't add " + fenvPaths[i] + " to the bundle";
}
ok = ok && mz_zip_writer_finalize_archive(&zip);
if (!ok && error.empty()) error = "couldn't finish " + outPath;
if (!ok) error += std::string(" (") + mz_zip_get_error_string(mz_zip_get_last_error(&zip)) + ")";
mz_zip_writer_end(&zip);
std::error_code ec;
if (!ok) { fs::remove(fs::u8path(tmp), ec); return false; }
fs::rename(fs::u8path(tmp), fs::u8path(outPath), ec);
if (ec) { fs::remove(fs::u8path(outPath), ec); fs::rename(fs::u8path(tmp), fs::u8path(outPath), ec); }
if (ec) { fs::remove(fs::u8path(tmp), ec); error = "couldn't replace " + outPath + ": " + ec.message(); return false; }
return true;
}
// Reads bundle.json alone.
inline bool fenvBundleReadManifest(const std::string& path, FenvBundleManifest& manifest, std::string* error = nullptr) {
fenv_bundle_detail::OpenBundle bundle;
std::string why;
if (!bundle.load(path, why)) { if (error) *error = why; return false; }
manifest = bundle.manifest;
return true;
}
// One environment's own manifest (and, if asked, its thumbnail bytes) read in place inside the bundle. `memberFile`
// is its "file" from the bundle's manifest.
inline bool fenvBundleReadEnvironment(const std::string& bundlePath, const std::string& memberFile, FenvManifest& manifest,
std::vector<uint8_t>* thumbnail = nullptr, std::string* error = nullptr) {
using namespace fenv_bundle_detail;
std::string why;
const auto fail = [&](const std::string& w) { if (error) *error = w; return false; };
OpenBundle bundle;
if (!bundle.load(bundlePath, why)) return fail(why);
const auto it = std::find_if(bundle.manifest.environments.begin(), bundle.manifest.environments.end(),
[&](const FenvBundleEntry& e) { return e.file == memberFile; });
if (it == bundle.manifest.environments.end()) return fail("the bundle lists no " + memberFile);
mz_uint index = 0;
mz_zip_archive_file_stat st;
if (!bundle.member(*it, index, st, why)) return fail(why);
FileSlice slice;
if (!memberDataOffset(bundlePath, st, slice.start)) return fail("damaged bundle: " + bundlePath);
slice.file.open(std::filesystem::u8path(bundlePath), std::ios::binary);
mz_zip_archive inner = {};
inner.m_pRead = &FileSlice::read;
inner.m_pIO_opaque = &slice;
if (!slice.file.is_open() || !mz_zip_reader_init(&inner, st.m_uncomp_size, 0)) return fail(memberFile + " in the bundle isn't a Frame environment");
const std::vector<uint8_t> json = fenv_detail::readMember(inner, "fenv.json");
bool ok = !json.empty() && fenvParseManifest(std::string(json.begin(), json.end()), manifest);
if (ok && thumbnail) *thumbnail = manifest.thumbnail.empty() ? std::vector<uint8_t>() : fenv_detail::readMember(inner, manifest.thumbnail.c_str());
mz_zip_reader_end(&inner);
return ok || fail(memberFile + " in the bundle isn't a Frame environment");
}
// Unpacks the bundle's environments into outDir as separate .fenv files (`onlyMember`: just that one, by its "file").
// Each is written as "<name>.part" and renamed when whole and checked against the bundle's checksum. A file already
// there of the same name is never replaced: if it is the same environment byte for byte it counts as unpacked,
// otherwise this one becomes "<name> 2.fenv". `unpacked` gets the environments' paths, in the bundle's order.
inline bool fenvBundleExtract(const std::string& bundlePath, const std::string& outDir, std::vector<std::string>& unpacked, std::string& error,
const std::string& onlyMember = {}, const FenvBundleProgress& progress = {}) {
namespace fs = std::filesystem;
using namespace fenv_bundle_detail;
unpacked.clear();
OpenBundle bundle;
if (!bundle.load(bundlePath, error)) return false;
std::vector<FenvBundleEntry> wanted;
for (const FenvBundleEntry& e : bundle.manifest.environments)
if (onlyMember.empty() || e.file == onlyMember) wanted.push_back(e);
if (wanted.empty()) { error = "the bundle lists no " + onlyMember; return false; }
uint64_t total = 0, done = 0;
for (const FenvBundleEntry& e : wanted) total += e.size;
std::error_code ec;
const fs::path dir = fs::u8path(outDir);
fs::create_directories(dir, ec);
for (const FenvBundleEntry& e : wanted) {
mz_uint index = 0;
mz_zip_archive_file_stat st;
if (!bundle.member(e, index, st, error)) return false;
const std::string stem = fs::u8path(e.file).stem().u8string();
if (stem.empty() || stem[0] == '.') { error = "the bundle names an environment \"" + e.file + "\""; return false; }
// The first free name, or the file that already is this environment.
std::string existing;
const std::string name = freeName(stem, [&](const std::string& candidate) {
const fs::path p = dir / fs::u8path(candidate);
if (!fs::exists(p, ec)) return false;
if (existing.empty() && fs::is_regular_file(p, ec) && fs::file_size(p, ec) == st.m_uncomp_size) {
MemberSink check;
check.compare.open(p, std::ios::binary);
if (check.compare.is_open() && mz_zip_reader_extract_to_callback(&bundle.zip, index, &MemberSink::write, &check, 0) && check.same) {
existing = candidate;
return false; // stop here: this one is it
}
}
return true;
});
const fs::path target = dir / fs::u8path(name);
if (!existing.empty()) {
done += e.size;
if (progress) progress(done, total);
unpacked.push_back(target.u8string());
continue;
}
const fs::path part = dir / fs::u8path(name + ".part");
MemberSink sink;
sink.out.open(part, std::ios::binary | std::ios::trunc);
sink.done = &done;
sink.total = total;
sink.progress = &progress;
bool ok = sink.out.is_open() && mz_zip_reader_extract_to_callback(&bundle.zip, index, &MemberSink::write, &sink, 0);
sink.out.close();
ok = ok && static_cast<bool>(sink.out) && sink.crc == st.m_crc32;
if (!ok) { fs::remove(part, ec); error = "couldn't unpack " + e.file + " from " + bundlePath + " (damaged, or the disk is full)"; return false; }
fs::rename(part, target, ec);
if (ec) { fs::remove(part, ec); error = "couldn't write " + target.u8string() + ": " + ec.message(); return false; }
unpacked.push_back(target.u8string());
}
return true;
}
// Checks a bundle without unpacking it: its manifest, each environment present, stored whole and of the size listed,
// with a readable fenv.json of its own; with `checkData`, each one's bytes against its checksum too (reads them all).
inline bool fenvBundleValidate(const std::string& path, std::string& error, bool checkData = true) {
using namespace fenv_bundle_detail;
OpenBundle bundle;
if (!bundle.load(path, error)) return false;
for (const FenvBundleEntry& e : bundle.manifest.environments) {
mz_uint index = 0;
mz_zip_archive_file_stat st;
if (!bundle.member(e, index, st, error)) return false;
if (st.m_uncomp_size != e.size) { error = e.file + " is " + std::to_string(st.m_uncomp_size) + " bytes, the bundle says " + std::to_string(e.size); return false; }
FenvManifest m;
if (!fenvBundleReadEnvironment(path, e.file, m, nullptr, &error)) return false;
if (!checkData) continue;
MemberSink sum; // (nothing open: checksums only)
if (!mz_zip_reader_extract_to_callback(&bundle.zip, index, &MemberSink::write, &sum, 0) || sum.crc != st.m_crc32) {
error = e.file + " in the bundle is damaged (checksum)";
return false;
}
}
return true;
}
+60 -2
View File
@@ -6,7 +6,11 @@
// inputs: a home APK (HSL, Clay .opa, glTF .ovrscene), a footprint + its vista, models (.glb .gltf .fbx
// .obj). Every home is baked (scene.bin: opens several times faster; a Clay / glTF home's animation goes in as
// data, anim_bake.h). --keep-apks stores the source APKs as well.
// fenv-tool info file.fenv
// fenv-tool info <file.fenv | file.fbundle> a bundle: its environments, their names and sizes
// fenv-tool bundle out.fbundle [--name NAME] file.fenv... several environments in one file, to share or install
// together (fenv_bundle.h); each stays its own environment
// fenv-tool unbundle file.fbundle outdir the bundle's environments as .fenv files (one there already is kept:
// the same one is skipped, a different one makes this "name 2.fenv")
// fenv-tool convert outdir <input>... one .fenv per input ("a.apk+b.apk" pairs a footprint with a vista)
// fenv-tool upgrade file.fenv... re-bake a .fenv's scene with this build, everything else kept (one that
// loads from its homes is baked: its APKs, models and world.json give way)
@@ -16,6 +20,7 @@
// the world's best settings on this headset (walked by frame-home --bench, paced at the display rate), stored in
// the .fenv (or <apk>.settings.json); prints them as JSON last
#include "fenv.h"
#include "fenv_bundle.h"
#include "home_report.h"
#include "tex_mips.h"
#include "scene_bake.h"
@@ -128,7 +133,46 @@ int pack(const std::string& out, const std::vector<std::string>& inputs, std::st
return 0;
}
// A bundle's environments: name, member file and size, each read from the bundle in place.
int bundleInfo(const std::string& path) {
FenvBundleManifest bundle;
std::string err;
if (!fenvBundleReadManifest(path, bundle, &err)) { std::fprintf(stderr, "fenv-tool: %s\n", err.c_str()); return 1; }
std::printf("%s\n bundle: %s, %zu environment%s\n", path.c_str(), bundle.name.c_str(), bundle.environments.size(), bundle.environments.size() == 1 ? "" : "s");
int rc = 0;
for (const FenvBundleEntry& e : bundle.environments) {
FenvManifest m;
std::vector<u8> thumb;
const bool readable = fenvBundleReadEnvironment(path, e.file, m, &thumb, &err);
std::printf(" %-36s %8.1f MB %s%s\n", e.name.c_str(), e.size / 1048576.0, e.file.c_str(),
!readable ? " (damaged)" : thumb.empty() ? "" : " (thumbnail)");
if (!readable) rc = 1;
}
return rc;
}
int bundle(const std::string& out, const std::vector<std::string>& inputs, const std::string& name) {
std::string err;
int shown = -10;
const bool ok = fenvBundleWrite(out, inputs, name, err, [&](uint64_t done, uint64_t total) {
const int percent = total ? static_cast<int>(done * 100 / total) : 100;
if (percent / 10 != shown / 10) { shown = percent; std::fprintf(stderr, "\r %d%%", percent); }
});
std::fprintf(stderr, "\r \r");
if (!ok) { std::fprintf(stderr, "fenv-tool: %s\n", err.c_str()); return 1; }
return bundleInfo(out);
}
int unbundle(const std::string& path, const std::string& outDir) {
std::vector<std::string> unpacked;
std::string err;
if (!fenvBundleExtract(path, outDir, unpacked, err)) { std::fprintf(stderr, "fenv-tool: %s\n", err.c_str()); return 1; }
for (const std::string& f : unpacked) std::printf("%s\n", f.c_str());
return 0;
}
int info(const std::string& path) {
if (isFenvBundlePath(path)) return bundleInfo(path);
FenvManifest m;
std::vector<u8> thumb;
if (!fenvReadManifest(path, m, &thumb)) { std::fprintf(stderr, "fenv-tool: not a .fenv: %s\n", path.c_str()); return 1; }
@@ -392,6 +436,17 @@ int tune(const std::string& home, const std::string& benchArg, int seconds, doub
int main(int argc, char** argv) {
if (argc >= 3 && !std::strcmp(argv[1], "info")) return info(argv[2]);
if (argc >= 4 && !std::strcmp(argv[1], "bundle")) {
std::string name;
std::vector<std::string> inputs;
for (int i = 3; i < argc; ++i) {
if (!std::strcmp(argv[i], "--name") && i + 1 < argc) name = argv[++i];
else inputs.push_back(argv[i]);
}
if (!isFenvBundlePath(argv[2])) { std::fprintf(stderr, "fenv-tool bundle: the bundle's name must end in .fbundle: %s\n", argv[2]); return 2; }
return bundle(argv[2], inputs, name);
}
if (argc >= 4 && !std::strcmp(argv[1], "unbundle")) return unbundle(argv[2], argv[3]);
if (argc >= 3 && !std::strcmp(argv[1], "upgrade")) {
for (int i = 2; i < argc; ++i) if (!std::strcmp(argv[i], "--no-optimize")) g_optimize = false;
int rc = 0;
@@ -467,7 +522,10 @@ int main(int argc, char** argv) {
#endif
std::fprintf(stderr,
"usage:\n fenv-tool pack out.fenv [--name NAME] [--edits edits.json] [--thumb thumb.png] [--keep-apks] [--fast] <home.apk|vista.apk|model>...\n"
" fenv-tool info file.fenv\n fenv-tool convert outdir <home.apk | footprint.apk+vista.apk | model>...\n"
" fenv-tool info <file.fenv | file.fbundle>\n"
" fenv-tool bundle out.fbundle [--name NAME] file.fenv... several environments in one file\n"
" fenv-tool unbundle file.fbundle outdir its environments as .fenv files\n"
" fenv-tool convert outdir <home.apk | footprint.apk+vista.apk | model>...\n"
" fenv-tool upgrade file.fenv... re-bake with this build (keeps world, edits, sound, picture)\n"
" fenv-tool report [--json] <home.apk | home.fenv | footprint.apk+vista.apk>... what it costs and what's wrong with it\n"
" fenv-tool settings <home.fenv | home.apk> [JSON | --clear] a world's own settings, shown or set\n"
+35 -2
View File
@@ -1,6 +1,7 @@
// frame-switcher — the Steam Frame's environment switcher: a SteamVR dashboard panel listing every environment
// in ~/frame-home/envs (Frame environments .fenv, home APKs, and footprint + vista pairs from pairs.txt) with
// thumbnails. Point and click one: ~/frame-home/current changes and Frame Home switches to it live.
// thumbnails. Point and click one: ~/frame-home/current changes and Frame Home switches to it live. A bundle of
// environments (.fbundle) put there is unpacked into its .fenv files, which the panel then lists one by one.
//
// frame-switcher the dashboard panel (SteamVR starts it: see --install)
// frame-switcher --install register with SteamVR and start it with SteamVR from now on
@@ -8,8 +9,10 @@
// frame-switcher --set NAME switch (NAME: the file name, or a number from --list)
// frame-switcher --thumbs render missing thumbnails (frame-home --snapshot) into envs/.thumbs
// frame-switcher --delete NAME delete an environment (like the panel's delete button; NAME as for --set)
// frame-switcher --unpack-bundles unpack the bundles (.fbundle) in envs now (the panel does it by itself)
// frame-switcher --test-delete DIR self-check of deleting, in a fake root made in DIR (must not exist yet)
#include "fenv.h"
#include "fenv_bundle.h"
#include "overlay_texture.h"
#include "overlays.h"
#include "perf_settings.h"
@@ -36,6 +39,7 @@
#include <filesystem>
#include <fstream>
#include <map>
#include <set>
#include <sstream>
#include <string>
#include <thread>
@@ -1433,8 +1437,35 @@ std::string statsLine() {
volatile sig_atomic_t g_quit = 0;
// Bundles (.fbundle, fenv_bundle.h) in envs/ become the environments they carry: each is unpacked beside them as
// its own .fenv (an environment already there is kept: the same one is skipped, a different one of the same name
// makes this one "<name> 2.fenv"), then the bundle is deleted, as everything it holds is in envs/ now. One that
// can't be unpacked (damaged, or still being copied in) stays and is tried again once its size or time changes.
void unpackBundles() {
static std::set<std::string> failed; // "<file>|<size>|<time>" of bundles that didn't unpack
std::error_code ec;
const fs::path envs = rootDir() / "envs";
for (const auto& e : fs::directory_iterator(envs, ec)) {
if (!e.is_regular_file(ec) || !isFenvBundlePath(e.path().string())) continue;
const std::string state = e.path().filename().string() + "|" + std::to_string(e.file_size(ec)) + "|" +
std::to_string(static_cast<long long>(e.last_write_time(ec).time_since_epoch().count()));
if (failed.count(state)) continue;
std::vector<std::string> unpacked;
std::string err;
const auto t0 = std::chrono::steady_clock::now();
if (!fenvBundleExtract(e.path().string(), envs.string(), unpacked, err)) {
LOG("can't unpack the bundle %s: %s", e.path().filename().c_str(), err.c_str());
failed.insert(state);
continue;
}
fs::remove(e.path(), ec);
LOG("unpacked the bundle %s into %zu environments (%.1f s)", e.path().filename().c_str(), unpacked.size(),
std::chrono::duration<double>(std::chrono::steady_clock::now() - t0).count());
}
}
// New environments are unpacked into the .fenv cache in the background as soon as they appear (a big one takes
// ~10 s on the Frame), so switching to one is as quick as switching back to it.
// ~10 s on the Frame), so switching to one is as quick as switching back to it. Bundles are opened up first.
void prepareEnvironments() {
static std::atomic<bool> busy{false}, again{false};
again = true; // files added while a run is going get a run of their own
@@ -1443,6 +1474,7 @@ void prepareEnvironments() {
setpriority(PRIO_PROCESS, 0, 15);
for (;;) {
while (again.exchange(false)) try {
unpackBundles();
std::error_code ec;
for (const auto& e : fs::directory_iterator(rootDir() / "envs", ec)) {
if (e.path().extension() != ".fenv") continue;
@@ -1904,6 +1936,7 @@ int main(int argc, char** argv) {
const std::string a = argc > 1 ? argv[1] : "";
if (a == "--install") return runInstall();
if (a == "--thumbs") return runThumbs();
if (a == "--unpack-bundles") { unpackBundles(); return 0; }
if (a == "--test-delete" && argc > 2) return runTestDelete(argv[2]);
if (a == "--render" && argc > 2) { // the panel as it looks now, to a PPM (no SteamVR needed)
// frame-switcher --render out.ppm [hover] [page] [scroll px] [switching card]
+170
View File
@@ -0,0 +1,170 @@
# SPDX-License-Identifier: Apache-2.0
# ----------------------------------------------------------------------------
# Copyright 2020-2026 Arm Limited
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy
# of the License at:
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# ----------------------------------------------------------------------------
# CMake configuration
cmake_minimum_required(VERSION 3.15)
cmake_policy(SET CMP0069 NEW) # LTO support
cmake_policy(SET CMP0091 NEW) # MSVC runtime support
if(MSVC)
# Disable structure was padded due to alignment specifier
add_compile_options("/wd4324")
endif()
project(astcencoder VERSION 5.6.0)
set(CMAKE_CXX_STANDARD 14)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
set(CMAKE_CXX_EXTENSIONS OFF)
set(CMAKE_EXPORT_COMPILE_COMMANDS 1)
set(CMAKE_XCODE_ATTRIBUTE_VALID_ARCHS "x86_64 x86_64h arm64")
include(CTest)
option(ASTCENC_ISA_AVX2 "Enable astcenc builds for AVX2 SIMD")
option(ASTCENC_ISA_SSE41 "Enable astcenc builds for SSE4.1 SIMD")
option(ASTCENC_ISA_SSE2 "Enable astcenc builds for SSE2 SIMD")
option(ASTCENC_ISA_SVE_256 "Enable astcenc builds for 256-bit SVE SIMD")
option(ASTCENC_ISA_SVE_128 "Enable astcenc builds for 128-bit SVE SIMD")
option(ASTCENC_ISA_NEON "Enable astcenc builds for NEON SIMD")
option(ASTCENC_ISA_NONE "Enable astcenc builds for no SIMD")
option(ASTCENC_ISA_NATIVE "Enable astcenc builds for native SIMD")
option(ASTCENC_DECOMPRESSOR "Enable astcenc builds for decompression only")
option(ASTCENC_SHAREDLIB "Enable astcenc builds with core library shared objects")
option(ASTCENC_BIG_ENDIAN "Enable astcenc big-endian support")
option(ASTCENC_DIAGNOSTICS "Enable astcenc builds with diagnostic trace")
option(ASTCENC_ASAN "Enable astcenc builds with address sanitizer")
option(ASTCENC_UBSAN "Enable astcenc builds with undefined behavior sanitizer")
option(ASTCENC_UNITTEST "Enable astcenc builds with unit tests")
option(ASTCENC_INVARIANCE "Enable astcenc floating point invariance" ON)
option(ASTCENC_CLI "Enable build of astcenc command line tools" ON)
option(ASTCENC_X86_GATHERS "Enable use of native x86 gathers" ON)
option(ASTCENC_WERROR "Force builds to treat warnings as errors" ON)
# Preflight for some macOS-specific build options
if("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin")
option(ASTCENC_UNIVERSAL_BUILD "Enable universal multi-arch build" ON)
if(${ASTCENC_UNIVERSAL_BUILD})
set(ASTCENC_ISA_SSE41 ON)
set(ASTCENC_ISA_AVX2 ON)
set(ASTCENC_ISA_NEON ON)
if(${ASTCENC_ISA_SSE2})
message(FATAL_ERROR "ISA_SSE2 cannot be used in a universal build")
endif()
if(${ASTCENC_ISA_NONE})
message(FATAL_ERROR "ISA_NONE cannot be used in a universal build")
endif()
if(${ASTCENC_ISA_NATIVE})
message(FATAL_ERROR "ISA_NATIVE cannot be used in a universal build")
endif()
endif()
else()
set(ASTCENC_UNIVERSAL_BUILD OFF)
endif()
# Count options which MUST be x64
set(ASTCENC_X64_ISA_COUNT 0)
set(ASTCENC_CONFIGS ${ASTCENC_ISA_AVX2} ${ASTCENC_ISA_SSE41} ${ASTCENC_ISA_SSE2})
foreach(ASTCENC_CONFIG ${ASTCENC_CONFIGS})
if(${ASTCENC_CONFIG})
math(EXPR ASTCENC_X64_ISA_COUNT "${ASTCENC_X64_ISA_COUNT} + 1")
endif()
endforeach()
# Count options which MUST be arm64
set(ASTCENC_ARM64_ISA_COUNT 0)
set(ASTCENC_CONFIGS ${ASTCENC_ISA_NEON} ${ASTCENC_ISA_SVE_128} ${ASTCENC_ISA_SVE_256})
foreach(ASTCENC_CONFIG ${ASTCENC_CONFIGS})
if(${ASTCENC_CONFIG})
math(EXPR ASTCENC_ARM64_ISA_COUNT "${ASTCENC_ARM64_ISA_COUNT} + 1")
endif()
endforeach()
# Non-macOS builds
if(NOT "${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin")
if(("${ASTCENC_ARM64_ISA_COUNT}" GREATER 0) AND ("${ASTCENC_X64_ISA_COUNT}" GREATER 0))
message(FATAL_ERROR "Builds can only support a single architecture per configure.")
endif()
endif()
# If nothing more specific is set then fall back on the compiler's defaults
if(("${ASTCENC_ARM64_ISA_COUNT}" EQUAL 0) AND ("${ASTCENC_X64_ISA_COUNT}" EQUAL 0) AND (NOT "${ASTCENC_ISA_NONE}"))
set(ASTCENC_ISA_NATIVE ON)
endif()
function(printopt optName optVal)
if(${optVal})
message(STATUS " ${optName} - ON")
else()
message(STATUS " ${optName} - OFF")
endif()
endfunction()
if("${ASTCENC_BLOCK_MAX_TEXELS}")
message(STATUS " Max block texels - ${ASTCENC_BLOCK_MAX_TEXELS}")
endif()
message(STATUS "Arm backend options")
printopt("SVE 256b backend " ${ASTCENC_ISA_SVE_256})
printopt("SVE 128b backend " ${ASTCENC_ISA_SVE_128})
printopt("NEON backend " ${ASTCENC_ISA_NEON})
message(STATUS "x86-64 backend options")
printopt("AVX2 backend " ${ASTCENC_ISA_AVX2})
printopt("SSE4.1 backend " ${ASTCENC_ISA_SSE41})
printopt("SSE2 backend " ${ASTCENC_ISA_SSE2})
printopt("Use native gathers " ${ASTCENC_X86_GATHERS})
message(STATUS "Agnostic backend options")
printopt("NONE backend " ${ASTCENC_ISA_NONE})
printopt("NATIVE backend " ${ASTCENC_ISA_NATIVE})
message(STATUS "Build options")
if("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin")
printopt("Universal bin " ${ASTCENC_UNIVERSAL_BUILD})
endif()
printopt("Invariance " ${ASTCENC_INVARIANCE})
if(${ASTCENC_BIG_ENDIAN})
printopt("Big endian " ${ASTCENC_BIG_ENDIAN})
endif()
printopt("Shared libs " ${ASTCENC_SHAREDLIB})
printopt("Decompressor " ${ASTCENC_DECOMPRESSOR})
message(STATUS "Developer options")
printopt("Diagnostics " ${ASTCENC_DIAGNOSTICS})
printopt("ASAN " ${ASTCENC_ASAN})
printopt("UBSAN " ${ASTCENC_UBSAN})
printopt("Unit tests " ${ASTCENC_UNITTEST})
# Subcomponents
add_subdirectory(Source)
# Configure package archive
if(ASTCENC_PACKAGE)
if("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin")
string(TOLOWER "macOS" ASTCENC_PKG_OS)
else()
string(TOLOWER ${CMAKE_SYSTEM_NAME} ASTCENC_PKG_OS)
endif()
set(CPACK_PACKAGE_FILE_NAME "astcenc-${CMAKE_PROJECT_VERSION}-${ASTCENC_PKG_OS}-${ASTCENC_PACKAGE}")
set(CPACK_INCLUDE_TOPLEVEL_DIRECTORY FALSE)
set(CPACK_PACKAGE_CHECKSUM SHA256)
set(CPACK_GENERATOR ZIP)
include(CPack) # Must be included after CPack configuration.
endif()
+175
View File
@@ -0,0 +1,175 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
+21
View File
@@ -0,0 +1,21 @@
# Environment bundles (src/fenv_bundle.h), checked natively on Linux (WSL) against real .fenv files:
# bundletest <workdir> a.fenv b.fenv [c.fenv...] [--big]
# bundles them (the first twice: the copy must come out as "<name> 2"), reads the bundle's and each
# environment's manifest and thumbnail in place, validates, unpacks (byte for byte against the originals),
# unpacks again (nothing new) and beside a different file of the same name; refuses what isn't an environment.
# --big: also a bundle past 4 GB (zip64: a 2.3 GB made-up environment, twice), ~9 GB of disk for a minute
# cmake -S tools/bundletest -B <build> -G Ninja && ninja -C <build>
cmake_minimum_required(VERSION 3.16)
project(bundletest C CXX)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_BUILD_TYPE Release)
set(R ${CMAKE_CURRENT_SOURCE_DIR}/../..)
set(TP ${R}/third_party)
add_library(miniz STATIC ${TP}/miniz/miniz.c ${TP}/miniz/miniz_tdef.c ${TP}/miniz/miniz_tinfl.c ${TP}/miniz/miniz_zip.c)
target_include_directories(miniz PUBLIC ${TP}/miniz)
target_compile_definitions(miniz PRIVATE MINIZ_DISABLE_ZIP_READER_CRC32_CHECKS) # as the real builds have it
add_library(lzma STATIC ${TP}/lzma/LzmaEnc.c ${TP}/lzma/LzmaDec.c ${TP}/lzma/LzFind.c ${TP}/lzma/LzFindOpt.c ${TP}/lzma/LzFindMt.c ${TP}/lzma/Threads.c ${TP}/lzma/CpuArch.c)
target_include_directories(lzma PUBLIC ${TP}/lzma)
add_executable(bundletest bundletest.cpp ${R}/src/fenv_lzma.cpp)
target_include_directories(bundletest PRIVATE ${R}/src ${TP}/qhe)
target_link_libraries(bundletest miniz lzma pthread)
+239
View File
@@ -0,0 +1,239 @@
// bundletest <workdir> a.fenv b.fenv [c.fenv...] [--big]: environment bundles (fenv_bundle.h) against real
// environments; see CMakeLists.txt. Prints each check, exits 1 when any failed.
#include "fenv_bundle.h"
#include <cstdio>
namespace fs = std::filesystem;
namespace {
int g_failed = 0;
void check(bool ok, const std::string& what) {
std::printf("%s %s\n", ok ? "ok " : "FAIL", what.c_str());
if (!ok) ++g_failed;
}
bool sameBytes(const fs::path& a, const fs::path& b) {
std::ifstream fa(a, std::ios::binary), fb(b, std::ios::binary);
if (!fa || !fb) return false;
std::vector<char> ba(1 << 20), bb(1 << 20);
for (;;) {
fa.read(ba.data(), static_cast<std::streamsize>(ba.size()));
fb.read(bb.data(), static_cast<std::streamsize>(bb.size()));
if (fa.gcount() != fb.gcount() || std::memcmp(ba.data(), bb.data(), static_cast<size_t>(fa.gcount())) != 0) return false;
if (!fa) return !fb;
}
}
std::vector<std::string> filesIn(const fs::path& dir) {
std::vector<std::string> names;
std::error_code ec;
for (const auto& e : fs::directory_iterator(dir, ec)) names.push_back(e.path().filename().u8string());
std::sort(names.begin(), names.end());
return names;
}
// Made-up environment bytes, the same for the same offset (so any byte read from the wrong place shows).
struct PatternSource {
uint64_t size = 0;
static size_t read(void* opaque, mz_uint64 offset, void* buffer, size_t n) {
const uint64_t size = static_cast<PatternSource*>(opaque)->size;
n = offset >= size ? 0 : static_cast<size_t>(std::min<uint64_t>(n, size - offset)); // (0 at the end: the writer's sign to stop)
auto* out = static_cast<unsigned char*>(buffer);
for (size_t i = 0; i < n; ++i) {
uint64_t x = (offset + i) / 8 * 0x9E3779B97F4A7C15ull;
x ^= x >> 29;
out[i] = static_cast<unsigned char>(x >> (8 * ((offset + i) % 8)));
}
return n;
}
};
// A bundle bigger than 4 GB: two copies of a 2.3 GB environment, so the second starts past the 32-bit offsets.
void bigBundle(const fs::path& work) {
const fs::path bigFenv = work / "big.fenv", bundlePath = work / "big.fbundle", outDir = work / "big-out";
std::error_code ec;
const uint64_t sceneSize = (uint64_t(23) << 30) / 10;
{
mz_zip_archive z = {};
PatternSource pattern;
pattern.size = sceneSize;
FenvManifest m;
m.name = "Big";
m.scene = "scene.bin";
const std::string json = fenv_detail::manifestJson(m);
bool ok = mz_zip_writer_init_file_v2(&z, bigFenv.c_str(), 0, 0) &&
mz_zip_writer_add_mem(&z, "fenv.json", json.data(), json.size(), MZ_DEFAULT_COMPRESSION) &&
mz_zip_writer_add_read_buf_callback(&z, "scene.bin", &PatternSource::read, &pattern, sceneSize, nullptr, nullptr, 0, MZ_NO_COMPRESSION,
nullptr, 0, nullptr, 0) &&
mz_zip_writer_finalize_archive(&z);
const std::string why = mz_zip_get_error_string(mz_zip_get_last_error(&z));
mz_zip_writer_end(&z);
check(ok, "big: made a 2.3 GB environment" + (ok ? std::string() : " (" + why + ")"));
if (!ok) return;
}
std::string err;
uint64_t lastDone = 0, lastTotal = 0;
bool ok = fenvBundleWrite(bundlePath.u8string(), {bigFenv.u8string(), bigFenv.u8string()}, "Big", err,
[&](uint64_t d, uint64_t t) { lastDone = d; lastTotal = t; });
check(ok, "big: bundled " + err);
const uint64_t bundleSize = fs::file_size(bundlePath, ec);
check(bundleSize > (uint64_t(4) << 30), "big: bundle is " + std::to_string(bundleSize >> 20) + " MB (> 4 GB)");
check(lastTotal == 2 * fs::file_size(bigFenv, ec) && lastDone == lastTotal, "big: progress reached the total");
{ // a zip64 end of central directory record is there (signature PK\6\6)
std::ifstream f(bundlePath, std::ios::binary);
f.seekg(-4096, std::ios::end);
std::string tail(4096, '\0');
f.read(&tail[0], 4096);
check(tail.find(std::string("PK\x06\x06", 4)) != std::string::npos, "big: zip64 end record present");
}
FenvBundleManifest bm;
check(fenvBundleReadManifest(bundlePath.u8string(), bm) && bm.environments.size() == 2 && bm.environments[1].file == "environments/big 2.fenv",
"big: manifest lists big + big 2");
FenvManifest second;
check(fenvBundleReadEnvironment(bundlePath.u8string(), "environments/big 2.fenv", second) && second.name == "Big",
"big: the member past 4 GB reads in place");
ok = fenvBundleValidate(bundlePath.u8string(), err);
check(ok, "big: validates with checksums " + (ok ? std::string() : err));
std::vector<std::string> unpacked;
ok = fenvBundleExtract(bundlePath.u8string(), outDir.u8string(), unpacked, err, "environments/big 2.fenv");
check(ok && unpacked.size() == 1 && sameBytes(unpacked[0], bigFenv), "big: the member past 4 GB unpacks byte for byte");
fs::remove(bigFenv, ec);
fs::remove(bundlePath, ec);
fs::remove_all(outDir, ec);
}
// A copy of an environment with a thumbnail added (the converted test homes have none).
bool withThumbnail(const std::string& from, const fs::path& to, std::vector<uint8_t>& thumb) {
FenvManifest m;
if (!fenvReadManifest(from, m)) return false;
m.thumbnail = "thumb.png";
thumb.resize(3000);
for (size_t i = 0; i < thumb.size(); ++i) thumb[i] = static_cast<uint8_t>(i * 7 + 1);
mz_zip_archive in = {}, out = {};
if (!mz_zip_reader_init_file(&in, from.c_str(), 0)) return false;
const std::string json = fenv_detail::manifestJson(m);
bool ok = mz_zip_writer_init_file_v2(&out, to.c_str(), 0, 0) &&
mz_zip_writer_add_mem(&out, "fenv.json", json.data(), json.size(), MZ_DEFAULT_COMPRESSION) &&
mz_zip_writer_add_mem(&out, "thumb.png", thumb.data(), thumb.size(), MZ_NO_COMPRESSION);
for (mz_uint i = 0; ok && i < mz_zip_reader_get_num_files(&in); ++i) {
char name[512];
mz_zip_reader_get_filename(&in, i, name, sizeof name);
if (std::strcmp(name, "fenv.json") != 0) ok = mz_zip_writer_add_from_zip_reader(&out, &in, i);
}
ok = ok && mz_zip_writer_finalize_archive(&out);
mz_zip_writer_end(&out);
mz_zip_reader_end(&in);
return ok;
}
} // namespace
int main(int argc, char** argv) {
if (argc < 4) { std::fprintf(stderr, "usage: bundletest <workdir> a.fenv b.fenv [c.fenv...] [--big]\n"); return 2; }
const fs::path work = fs::u8path(argv[1]);
std::vector<std::string> originals;
bool big = false;
for (int i = 2; i < argc; ++i) {
if (!std::strcmp(argv[i], "--big")) big = true;
else originals.push_back(argv[i]);
}
std::error_code ec;
fs::remove_all(work, ec);
fs::create_directories(work, ec);
std::string err;
// Bundled: every original, one with a thumbnail, then the first again (a second file of the same name).
std::vector<uint8_t> thumb;
const fs::path thumbFenv = work / "with-thumbnail.fenv";
check(withThumbnail(originals.front(), thumbFenv, thumb), "made " + thumbFenv.filename().u8string());
std::vector<std::string> inputs = originals;
inputs.push_back(thumbFenv.u8string());
inputs.push_back(originals.front());
const std::string bundlePath = (work / "test.fbundle").u8string();
uint64_t total = 0, lastDone = 0, lastTotal = 0, calls = 0;
for (const auto& f : inputs) total += fs::file_size(fs::u8path(f), ec);
bool ok = fenvBundleWrite(bundlePath, inputs, "Test bundle", err, [&](uint64_t d, uint64_t t) { lastDone = d; lastTotal = t; ++calls; });
check(ok, "bundle written " + err);
check(lastDone == total && lastTotal == total, "progress reached " + std::to_string(total) + " bytes in " + std::to_string(calls) + " calls");
check(!fs::exists(bundlePath + ".part", ec), "no .part left");
check(fs::file_size(fs::u8path(bundlePath), ec) < total + 4096, "members stored (bundle " + std::to_string(fs::file_size(fs::u8path(bundlePath), ec)) +
" bytes for " + std::to_string(total) + ")");
FenvBundleManifest bm;
ok = fenvBundleReadManifest(bundlePath, bm, &err);
check(ok && bm.name == "Test bundle" && bm.environments.size() == inputs.size(), "manifest: name and " + std::to_string(bm.environments.size()) + " environments");
const std::string firstStem = fs::u8path(originals.front()).stem().u8string();
for (size_t i = 0; ok && i < inputs.size(); ++i) {
const FenvBundleEntry& e = bm.environments[i];
const std::string want = "environments/" + (i + 1 == inputs.size() ? firstStem + " 2" : fs::u8path(inputs[i]).stem().u8string()) + ".fenv";
FenvManifest original, inBundle;
std::vector<uint8_t> originalThumb, bundleThumb;
fenvReadManifest(inputs[i], original, &originalThumb);
const bool read = fenvBundleReadEnvironment(bundlePath, e.file, inBundle, &bundleThumb, &err);
check(e.file == want && e.size == fs::file_size(fs::u8path(inputs[i]), ec) && e.name == original.name,
"entry " + e.file + " \"" + e.name + "\" " + std::to_string(e.size) + " bytes");
check(read && inBundle.name == original.name && inBundle.scene == original.scene && bundleThumb == originalThumb &&
(inputs[i] != thumbFenv.u8string() || bundleThumb == thumb),
" its fenv.json + thumbnail (" + std::to_string(bundleThumb.size()) + " bytes) read in place");
}
ok = fenvBundleValidate(bundlePath, err);
check(ok, "validates with checksums " + (ok ? std::string() : err));
// Unpacked: byte for byte the originals.
const fs::path out = work / "out";
std::vector<std::string> unpacked;
ok = fenvBundleExtract(bundlePath, out.u8string(), unpacked, err);
check(ok && unpacked.size() == inputs.size(), "unpacked " + std::to_string(unpacked.size()) + " " + err);
for (size_t i = 0; ok && i < inputs.size(); ++i)
check(sameBytes(unpacked[i], inputs[i]), " " + fs::u8path(unpacked[i]).filename().u8string() + " == " + inputs[i]);
const std::vector<std::string> before = filesIn(out);
std::vector<std::string> again;
ok = fenvBundleExtract(bundlePath, out.u8string(), again, err);
check(ok && again == unpacked && filesIn(out) == before, "unpacked again: the same files, nothing new");
// Beside a different file of the same name: that file stays, this one becomes "<name> 2".
const fs::path clash = work / "clash";
fs::create_directories(clash, ec);
std::ofstream(clash / (firstStem + ".fenv")) << "someone else's";
ok = fenvBundleExtract(bundlePath, clash.u8string(), unpacked, err, "environments/" + firstStem + ".fenv");
check(ok && unpacked.size() == 1 && fs::u8path(unpacked[0]).filename().u8string() == firstStem + " 2.fenv" && sameBytes(unpacked[0], originals.front()) &&
fs::file_size(clash / (firstStem + ".fenv"), ec) == std::string("someone else's").size(),
"one member beside a different same-named file -> \"" + firstStem + " 2.fenv\"");
// Refused: what isn't an environment (and nothing left behind).
std::ofstream(work / "fake.fenv") << "not a zip";
std::ofstream(work / "notes.txt") << "text";
const std::string refused = (work / "refused.fbundle").u8string();
const std::vector<std::pair<std::string, std::string>> notEnvironments = {
{"a broken .fenv", (work / "fake.fenv").u8string()}, {"a non-.fenv", (work / "notes.txt").u8string()}, {"a missing file", (work / "missing.fenv").u8string()}};
for (const auto& [what, file] : notEnvironments) {
ok = fenvBundleWrite(refused, {originals.front(), file}, "", err);
check(!ok && !fs::exists(refused, ec) && !fs::exists(refused + ".part", ec), "refuses " + what + " (" + err + ")");
}
check(!fenvBundleReadManifest(originals.front(), bm), "a .fenv isn't a bundle");
{ // a damaged bundle: one byte flipped inside a member fails validation and unpacking, leaving no file
const std::string damaged = (work / "damaged.fbundle").u8string();
fs::copy_file(fs::u8path(bundlePath), fs::u8path(damaged), ec);
std::fstream f(fs::u8path(damaged), std::ios::binary | std::ios::in | std::ios::out);
const auto at = static_cast<std::streamoff>(fs::file_size(fs::u8path(damaged), ec) / 2);
f.seekg(at);
char c = 0;
f.get(c);
f.seekp(at);
f.put(static_cast<char>(c ^ 0x5a));
f.close();
ok = fenvBundleValidate(damaged, err);
check(!ok, "a flipped byte fails validation (" + err + ")");
const fs::path damagedOut = work / "damaged-out";
ok = fenvBundleExtract(damaged, damagedOut.u8string(), unpacked, err);
bool partLeft = false;
for (const auto& n : filesIn(damagedOut)) partLeft = partLeft || n.find(".part") != std::string::npos;
check(!ok && !partLeft, " and unpacking stops without leaving a .part (" + err + ")");
}
if (big) bigBundle(work);
std::printf("%s: %d failed\n", g_failed ? "FAILED" : "passed", g_failed);
return g_failed ? 1 : 0;
}