diff --git a/Engine/Core/Platform/path_text.h b/Engine/Core/Platform/path_text.h new file mode 100644 index 0000000..11bded8 --- /dev/null +++ b/Engine/Core/Platform/path_text.h @@ -0,0 +1,25 @@ +#pragma once + +#include +#include + +namespace dingosdk { + +// A path as narrow text. +// +// std::filesystem::path::string() converts to the process's ANSI code page and +// throws when a character has no mapping there: "No mapping for the Unicode +// character exists in the target multi-byte code page". A folder named in +// Cyrillic or CJK is enough, and so is an invisible one -- a directory called +// U+200E (left-to-right mark) has been seen in the wild. The failure lands far +// from the cause, because the path is usually only being written into a log or +// an error message at the time. +// +// UTF-8 can represent every path Windows allows, so every path that becomes a +// narrow string goes through here instead. +inline std::string path_utf8(const std::filesystem::path& path) { + const auto text = path.u8string(); + return {text.begin(), text.end()}; +} + +} // namespace dingosdk diff --git a/Engine/Vfs/game_archives.cpp b/Engine/Vfs/game_archives.cpp index 2770a66..f816703 100644 --- a/Engine/Vfs/game_archives.cpp +++ b/Engine/Vfs/game_archives.cpp @@ -1,5 +1,6 @@ #include "game_archives.h" #include "Engine/Resource/cas_codec.h" +#include "Engine/Core/Platform/path_text.h" #include #include #include @@ -26,7 +27,7 @@ Layout read_layout(const fs::path& path) { Layout layout{{std::istreambuf_iterator(input), std::istreambuf_iterator()}, {}}; if (layout.buffer.size() <= native_db::envelope_size || std::memcmp(layout.buffer.data(), native_db::magic, sizeof(native_db::magic))) - throw std::runtime_error("Unrecognized native envelope in " + path.string()); + throw std::runtime_error("Unrecognized native envelope in " + path_utf8(path)); layout.root = native_db::read(std::span(layout.buffer).subspan(native_db::envelope_size), "layout.toc", nullptr, {.unique_fields = false}); return layout; @@ -108,11 +109,11 @@ std::vector GameArchives::read(const fs::path& root, const fb::CasIde const auto path = root / L"Win32" / fs::path(directory(location.installChunk)) / fs::path(archive_file(location.archive)); std::ifstream input(path, std::ios::binary); - if (!input) throw std::runtime_error("Cannot open " + path.string()); + if (!input) throw std::runtime_error("Cannot open " + path_utf8(path)); input.seekg(static_cast(offset)); std::vector bytes(size); if (size && !input.read(reinterpret_cast(bytes.data()), static_cast(size))) - throw std::runtime_error("Cannot read " + path.string()); + throw std::runtime_error("Cannot read " + path_utf8(path)); return bytes; } diff --git a/Engine/Vfs/game_bundles.cpp b/Engine/Vfs/game_bundles.cpp index 3f05489..14de075 100644 --- a/Engine/Vfs/game_bundles.cpp +++ b/Engine/Vfs/game_bundles.cpp @@ -1,5 +1,6 @@ #include "game_bundles.h" #include "Engine/Resource/cas_codec.h" +#include "Engine/Core/Platform/path_text.h" #include #include @@ -50,10 +51,10 @@ GameData::GameData(fs::path gameRoot) fb::TocDocument GameData::read_toc(std::string_view relative) const { const auto path = gameRoot_ / L"Data" / fs::path(relative); std::ifstream input(path, std::ios::binary); - if (!input) throw std::runtime_error("Cannot open " + path.string()); + if (!input) throw std::runtime_error("Cannot open " + path_utf8(path)); std::vector bytes(static_cast(fs::file_size(path))); if (!input.read(reinterpret_cast(bytes.data()), static_cast(bytes.size()))) - throw std::runtime_error("Cannot read " + path.string()); + throw std::runtime_error("Cannot read " + path_utf8(path)); return fb::read_toc(bytes); } diff --git a/Engine/Vfs/initfs.cpp b/Engine/Vfs/initfs.cpp index 3d57d29..9fca0bf 100644 --- a/Engine/Vfs/initfs.cpp +++ b/Engine/Vfs/initfs.cpp @@ -5,6 +5,7 @@ #include "Engine/Game/Build/supported_build.h" #include "Engine/Game/Build/addresses.h" #include "Engine/Game/Build/20260929/initfs.h" +#include "Engine/Core/Platform/path_text.h" #include #include @@ -37,13 +38,13 @@ std::string lower(std::string_view text) { } Bytes read_file(const fs::path& path, std::size_t maximum) { std::ifstream input(path, std::ios::binary | std::ios::ate); - if (!input) fail("Cannot open " + path.string()); + if (!input) fail("Cannot open " + path_utf8(path)); const auto length = input.tellg(); - if (length < 0 || static_cast(length) > maximum) fail("File exceeds size limit: " + path.string()); + if (length < 0 || static_cast(length) > maximum) fail("File exceeds size limit: " + path_utf8(path)); Bytes bytes(static_cast(length)); input.seekg(0); if (!bytes.empty() && !input.read(reinterpret_cast(bytes.data()), static_cast(bytes.size()))) - fail("Cannot read " + path.string()); + fail("Cannot read " + path_utf8(path)); return bytes; } @@ -128,7 +129,7 @@ void apply_archive(const fs::path& path, const Secret& key, Entries& effective, auto input = read_file(path, maximum_archive); if (input.size() <= native_db::envelope_size || std::memcmp(input.data(), native_db::magic, sizeof(native_db::magic))) - fail("Unrecognized native envelope in " + path.string()); + fail("Unrecognized native envelope in " + path_utf8(path)); const auto body = std::span(input).subspan(native_db::envelope_size); std::size_t consumed{}; auto root = native_db::read(body, "InitFS", &consumed); @@ -160,12 +161,12 @@ void apply_archive(const fs::path& path, const Secret& key, Entries& effective, if (const auto* file_system = file->field("fs"); file_system && (file_system->type != 7 || (!file_system->text.empty() && file_system->text != "/data"))) fail("Unsupported InitFS filesystem destination for " + name->text); - Entry entry{name->text, path.string(), Bytes(payload->bytes.begin(), payload->bytes.end())}; + Entry entry{name->text, path_utf8(path), Bytes(payload->bytes.begin(), payload->bytes.end())}; if (!entries.emplace(lower(name->text), std::move(entry)).second) fail("Duplicate InitFS virtual path"); } if (!inherit) effective.clear(); for (auto& [name, entry] : entries) effective.insert_or_assign(name, std::move(entry)); - sources.push_back(Json{{"path", path.string()}, {"sha256", launcher::sha256_file(path)}, {"inherit", inherit}}); + sources.push_back(Json{{"path", path_utf8(path)}, {"sha256", launcher::sha256_file(path)}, {"inherit", inherit}}); } bool within(const fs::path& root, const fs::path& path) { @@ -178,7 +179,7 @@ bool within(const fs::path& root, const fs::path& path) { bool write_atomic(const fs::path& root, const fs::path& path, std::span bytes, bool replace) { if (!within(root, fs::weakly_canonical(path))) fail("Output path escapes the export directory"); if (!replace && fs::exists(path)) { - if (!fs::is_regular_file(path)) fail("Output path is not a regular file: " + path.string()); + if (!fs::is_regular_file(path)) fail("Output path is not a regular file: " + path_utf8(path)); return false; } fs::create_directories(path.parent_path()); @@ -186,17 +187,17 @@ bool write_atomic(const fs::path& root, const fs::path& path, std::span(bytes.size()), &written, nullptr) && written == bytes.size() && FlushFileBuffers(file); CloseHandle(file); - if (!ready) { DeleteFileW(temporary.c_str()); fail("Cannot write output file: " + path.string()); } + if (!ready) { DeleteFileW(temporary.c_str()); fail("Cannot write output file: " + path_utf8(path)); } if (MoveFileExW(temporary.c_str(), path.c_str(), MOVEFILE_WRITE_THROUGH | (replace ? MOVEFILE_REPLACE_EXISTING : 0))) return true; const auto error = GetLastError(); DeleteFileW(temporary.c_str()); if (!replace && (error == ERROR_ALREADY_EXISTS || error == ERROR_FILE_EXISTS) && fs::is_regular_file(path)) return false; - fail("Cannot publish output file: " + path.string()); + fail("Cannot publish output file: " + path_utf8(path)); } Json read_preferences(const fs::path& root) { const auto path = root / L"ReSkate.settings.json"; diff --git a/Engine/Vfs/item_thumbnails.cpp b/Engine/Vfs/item_thumbnails.cpp index ebd011e..09daeae 100644 --- a/Engine/Vfs/item_thumbnails.cpp +++ b/Engine/Vfs/item_thumbnails.cpp @@ -1,5 +1,6 @@ #include "item_thumbnails.h" #include "game_bundles.h" +#include "Engine/Core/Platform/path_text.h" #include #include @@ -32,7 +33,7 @@ ThumbnailRead read_build_kit_thumbnails(const std::filesystem::path& gameRoot, s bundle = data.read_bundle(toc, bundle_name); } catch (const std::exception& failure) { if (problems.size() < 600) - problems += "; " + it->path().filename().string() + ": " + failure.what(); + problems += "; " + path_utf8(it->path().filename()) + ": " + failure.what(); } } if (!bundle) throw std::runtime_error("The game's thumbnail bundle was not found" + problems); diff --git a/Engine/Vfs/mod_list.cpp b/Engine/Vfs/mod_list.cpp index fb9dd25..3d68090 100644 --- a/Engine/Vfs/mod_list.cpp +++ b/Engine/Vfs/mod_list.cpp @@ -2,6 +2,7 @@ #include "Engine/Game/Build/supported_build.h" #include "Engine/Core/Json/json.h" +#include "Engine/Core/Platform/path_text.h" #include @@ -35,15 +36,15 @@ std::string lower(std::string_view text) { std::vector read_file(const fs::path& path, std::size_t maximum) { std::ifstream input(path, std::ios::binary | std::ios::ate); - if (!input) throw std::runtime_error("Cannot open " + path.string()); + if (!input) throw std::runtime_error("Cannot open " + path_utf8(path)); const auto length = input.tellg(); if (length < 0 || static_cast(length) > maximum) - throw std::runtime_error("File exceeds size limit: " + path.string()); + throw std::runtime_error("File exceeds size limit: " + path_utf8(path)); std::vector bytes(static_cast(length)); input.seekg(0); if (!bytes.empty() && !input.read(reinterpret_cast(bytes.data()), static_cast(bytes.size()))) - throw std::runtime_error("Cannot read " + path.string()); + throw std::runtime_error("Cannot read " + path_utf8(path)); return bytes; } @@ -420,12 +421,12 @@ void save_mod_order(const std::filesystem::path& mods_root, const std::vector(text.size())) || !output.flush()) - throw std::runtime_error("Cannot write " + temporary.string()); + throw std::runtime_error("Cannot write " + path_utf8(temporary)); } if (!MoveFileExW(temporary.c_str(), target.c_str(), MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH)) { std::error_code ignored; fs::remove(temporary, ignored); - throw std::runtime_error("Cannot replace " + target.string() + " (Windows error " + + throw std::runtime_error("Cannot replace " + path_utf8(target) + " (Windows error " + std::to_string(GetLastError()) + ")"); } } diff --git a/Engine/Vfs/mod_merge_cas_store.cpp b/Engine/Vfs/mod_merge_cas_store.cpp index e4f56e9..9223420 100644 --- a/Engine/Vfs/mod_merge_cas_store.cpp +++ b/Engine/Vfs/mod_merge_cas_store.cpp @@ -1,4 +1,5 @@ #include "mod_merge_internal.h" +#include "Engine/Core/Platform/path_text.h" #include #include @@ -24,7 +25,7 @@ fb::BundleFileInfo CasStore::write(std::uint32_t installChunk, std::uint16_t arc std::ofstream out(path, std::ios::binary | std::ios::app); if (!out || !out.write(reinterpret_cast(encoded.data()), static_cast(encoded.size()))) - throw std::runtime_error("Cannot append to " + path.string()); + throw std::runtime_error("Cannot append to " + path_utf8(path)); fb::BundleFileInfo info{{true, installChunk, archive}, static_cast(offset), static_cast(encoded.size())}; offset += encoded.size(); diff --git a/Engine/Vfs/mod_merge_combine.cpp b/Engine/Vfs/mod_merge_combine.cpp index 76b4e10..d3a5640 100644 --- a/Engine/Vfs/mod_merge_combine.cpp +++ b/Engine/Vfs/mod_merge_combine.cpp @@ -5,6 +5,7 @@ #include "Engine/Resource/ebx_merge.h" #include "Engine/Resource/ebx_writer.h" #include "Engine/Resource/shader_lookup.h" +#include "Engine/Core/Platform/path_text.h" #include #include @@ -153,7 +154,7 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot, auto region = fb::read_bundle_region(bundle.region); const auto provider = providers.find(key); const auto shared = provider != providers.end() && provider->second > 1; - const auto modName = isBase ? std::string("base") : root.filename().string(); + const auto modName = isBase ? std::string("base") : path_utf8(root.filename()); // The bundle holding the live material grid is rebuilt to carry the // combined one, and a mod whose surfaces moved has its collision // renumbered; both need the asset list even where one mod ships it. diff --git a/Engine/Vfs/mod_merge_files.cpp b/Engine/Vfs/mod_merge_files.cpp index c7e5147..a169b9e 100644 --- a/Engine/Vfs/mod_merge_files.cpp +++ b/Engine/Vfs/mod_merge_files.cpp @@ -1,5 +1,6 @@ #include "mod_merge_internal.h" #include "Engine/Core/Json/json.h" +#include "Engine/Core/Platform/path_text.h" #include #include @@ -21,15 +22,15 @@ std::string lower(std::string_view text) { std::vector read_file(const fs::path& path) { std::ifstream input(path, std::ios::binary | std::ios::ate); - if (!input) throw std::runtime_error("Cannot open " + path.string()); + if (!input) throw std::runtime_error("Cannot open " + path_utf8(path)); const auto length = input.tellg(); if (length < 0 || static_cast(length) > maximumTocBytes) - throw std::runtime_error("File exceeds the TOC size limit: " + path.string()); + throw std::runtime_error("File exceeds the TOC size limit: " + path_utf8(path)); std::vector bytes(static_cast(length)); input.seekg(0); if (!bytes.empty() && !input.read(reinterpret_cast(bytes.data()), static_cast(bytes.size()))) - throw std::runtime_error("Cannot read " + path.string()); + throw std::runtime_error("Cannot read " + path_utf8(path)); return bytes; } @@ -40,12 +41,12 @@ void write_file(const fs::path& path, std::span bytes) { std::ofstream output(temporary, std::ios::binary | std::ios::trunc); if (!output || !output.write(reinterpret_cast(bytes.data()), static_cast(bytes.size()))) - throw std::runtime_error("Cannot write " + temporary.string()); + throw std::runtime_error("Cannot write " + path_utf8(temporary)); } if (!MoveFileExW(temporary.c_str(), path.c_str(), MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH)) { const auto error = GetLastError(); DeleteFileW(temporary.c_str()); - throw std::runtime_error("Cannot publish " + path.string() + + throw std::runtime_error("Cannot publish " + path_utf8(path) + " (Windows error " + std::to_string(error) + ")"); } } @@ -64,12 +65,12 @@ void unshare(const fs::path& path) { const auto temporary = fs::path(path.wstring() + L".unshared"); std::error_code error; fs::copy_file(path, temporary, fs::copy_options::overwrite_existing, error); - if (error) throw std::runtime_error("Cannot give " + path.string() + " a private copy: " + + if (error) throw std::runtime_error("Cannot give " + path_utf8(path) + " a private copy: " + error.message()); if (!MoveFileExW(temporary.c_str(), path.c_str(), MOVEFILE_REPLACE_EXISTING)) { const auto failure = GetLastError(); DeleteFileW(temporary.c_str()); - throw std::runtime_error("Cannot replace the shared " + path.string() + + throw std::runtime_error("Cannot replace the shared " + path_utf8(path) + " (Windows error " + std::to_string(failure) + ")"); } } @@ -78,22 +79,22 @@ std::uint64_t append_file(const fs::path& from, const fs::path& to) { fs::create_directories(to.parent_path()); unshare(to); std::ifstream input(from, std::ios::binary); - if (!input) throw std::runtime_error("Cannot read " + from.string()); + if (!input) throw std::runtime_error("Cannot read " + path_utf8(from)); // tellp() on a stream opened for append reads 0 until the first write, so // the block's start comes from the file itself. std::error_code error; const auto existing = fs::file_size(to, error); const auto start = error ? std::uint64_t{} : existing; std::ofstream output(to, std::ios::binary | std::ios::app); - if (!output) throw std::runtime_error("Cannot append to " + to.string()); + if (!output) throw std::runtime_error("Cannot append to " + path_utf8(to)); // A megabyte of stack is a stack overflow; the copy buffer lives on the heap. std::vector buffer(1 << 20); while (input) { input.read(buffer.data(), static_cast(buffer.size())); if (input.gcount() && !output.write(buffer.data(), input.gcount())) - throw std::runtime_error("Cannot append to " + to.string()); + throw std::runtime_error("Cannot append to " + path_utf8(to)); } - if (!output) throw std::runtime_error("Cannot append to " + to.string()); + if (!output) throw std::runtime_error("Cannot append to " + path_utf8(to)); return start; } @@ -104,7 +105,7 @@ void link_or_copy(const fs::path& from, const fs::path& to) { fs::remove(to, error); if (CreateHardLinkW(to.c_str(), from.c_str(), nullptr)) return; fs::copy_file(from, to, fs::copy_options::overwrite_existing, error); - if (error) throw std::runtime_error("Cannot place " + to.string() + ": " + error.message()); + if (error) throw std::runtime_error("Cannot place " + path_utf8(to) + ": " + error.message()); } RelativeFiles scan(const fs::path& directory) { diff --git a/Engine/Vfs/mod_merge_grid.cpp b/Engine/Vfs/mod_merge_grid.cpp index fc8ba2c..11dcf39 100644 --- a/Engine/Vfs/mod_merge_grid.cpp +++ b/Engine/Vfs/mod_merge_grid.cpp @@ -2,6 +2,7 @@ #include "Engine/Resource/cas_codec.h" #include "Engine/Resource/ebx_document.h" #include "Engine/Resource/ebx_writer.h" +#include "Engine/Core/Platform/path_text.h" #include #include @@ -128,7 +129,7 @@ GridPlan plan_material_grid(const std::vector& mods, // Lowest priority first, the order the merge layers mods in. for (auto at = mods.rbegin(); at != mods.rend(); ++at) { const auto* mod = *at; - const auto modName = mod->directory.filename().string(); + const auto modName = path_utf8(mod->directory.filename()); for (const auto& relative : modFiles.at(mod).tocs) { try { const auto toc = fb::read_toc(read_file(mod->directory / fs::path(relative))); @@ -190,7 +191,7 @@ GridPlan plan_material_grid(const std::vector& mods, grid::Combiner combiner(*base); std::size_t combined{}; for (auto& contribution : contributions) { - const auto modName = contribution.mod->directory.filename().string(); + const auto modName = path_utf8(contribution.mod->directory.filename()); const auto before = combiner.slots(); grid::SlotMap slots; try { diff --git a/Engine/Vfs/mod_merge_load_screens.cpp b/Engine/Vfs/mod_merge_load_screens.cpp index 51f11cf..8ed98e9 100644 --- a/Engine/Vfs/mod_merge_load_screens.cpp +++ b/Engine/Vfs/mod_merge_load_screens.cpp @@ -1,5 +1,6 @@ #include "mod_merge_internal.h" #include "Engine/Resource/ebx_document.h" +#include "Engine/Core/Platform/path_text.h" #include #include @@ -33,7 +34,7 @@ std::vector read_load_screens(const std::vector result; for (const auto* mod : mods) { - const auto modName = mod->directory.filename().string(); + const auto modName = path_utf8(mod->directory.filename()); for (const auto& relative : modFiles.at(mod).tocs) { if (lower(relative) != configurationToc) continue; try { diff --git a/Engine/Vfs/mod_merge_stamp.cpp b/Engine/Vfs/mod_merge_stamp.cpp index 2f067f6..3161150 100644 --- a/Engine/Vfs/mod_merge_stamp.cpp +++ b/Engine/Vfs/mod_merge_stamp.cpp @@ -1,4 +1,5 @@ #include "mod_merge_internal.h" +#include "Engine/Core/Platform/path_text.h" #include #include @@ -69,7 +70,7 @@ std::string merge_fingerprint(const Catalog& catalog, const std::vectoris_regular_file(error) && !error) files.push_back(it->path()); error.clear(); } - if (error) throw std::runtime_error("Cannot list " + mod->directory.string()); + if (error) throw std::runtime_error("Cannot list " + path_utf8(mod->directory)); std::ranges::sort(files); for (const auto& file : files) describe(utf8(fs::relative(file, mod->directory, error)), file); @@ -151,10 +152,10 @@ void write_stamp(const fs::path& output, const std::string& fingerprint, const M for (fs::recursive_directory_iterator it(output, error), end; it != end && !error; it.increment(error)) { if (!it->is_regular_file(error) || error) { error.clear(); continue; } const auto size = fs::file_size(it->path(), error); - if (error) throw std::runtime_error("Cannot size " + it->path().string()); + if (error) throw std::runtime_error("Cannot size " + path_utf8(it->path())); text += "file " + std::to_string(size) + ' ' + utf8(fs::relative(it->path(), output, error)) + '\n'; } - if (error) throw std::runtime_error("Cannot list " + output.string()); + if (error) throw std::runtime_error("Cannot list " + path_utf8(output)); for (auto note : report.notes) { std::ranges::replace(note, '\n', ' '); std::ranges::replace(note, '\r', ' '); diff --git a/Engine/Vfs/world_layer_scan.cpp b/Engine/Vfs/world_layer_scan.cpp index 34c84f2..a019c1d 100644 --- a/Engine/Vfs/world_layer_scan.cpp +++ b/Engine/Vfs/world_layer_scan.cpp @@ -3,6 +3,7 @@ #include "game_bundles.h" #include "Engine/Core/Json/json.h" #include "Engine/Resource/ebx_document.h" +#include "Engine/Core/Platform/path_text.h" #include #include #include @@ -252,7 +253,7 @@ WorldLayerCatalog load_or_scan(const fs::path& gameRoot, const fs::path& file) { WorldLayerCatalog read(const fs::path& file) { std::ifstream input(file, std::ios::binary); - if (!input) throw std::runtime_error("Cannot open " + file.string()); + if (!input) throw std::runtime_error("Cannot open " + path_utf8(file)); const std::string text{std::istreambuf_iterator(input), std::istreambuf_iterator()}; return from_json(Json::parse(text.begin(), text.end())); } diff --git a/Launcher/updater.cpp b/Launcher/updater.cpp index 4f8557c..5532a66 100644 --- a/Launcher/updater.cpp +++ b/Launcher/updater.cpp @@ -5,6 +5,7 @@ #include "Engine/Core/Platform/launcher_support.h" #include "launcher_update_config.h" #include "text_encoding.h" +#include "Engine/Core/Platform/path_text.h" #include #include @@ -197,7 +198,7 @@ struct Zip { // Writes one ZIP entry to `output`. void extract_index(Zip& zip, mz_uint index, const mz_zip_archive_file_stat& entry, const fs::path& output) { Handle file{CreateFileW(output.c_str(), GENERIC_WRITE, 0, nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr)}; - if (file.value == INVALID_HANDLE_VALUE) fail("Cannot create " + output.string()); + if (file.value == INVALID_HANDLE_VALUE) fail("Cannot create " + path_utf8(output)); struct Sink { HANDLE file; std::uint64_t count{}; } sink{file.value}; const auto write = [](void* opaque, mz_uint64 offset, const void* buffer, size_t count) -> size_t { auto& target = *static_cast(opaque);