Write paths as UTF-8, not the ANSI code page

A player whose game sat under a folder named U+200E (left-to-right mark,
an invisible directory on their Desktop) could not launch: the status
tile read "No mapping for the Unicode character exists in the target
multi-byte code page". Nothing was wrong with their mods, and the merge
in their log had just succeeded.

fs::path::string() converts to the process's ANSI code page and throws
exactly that when a character has no mapping there. initfs.cpp called it
on the normal path, not in an error handler, so exporting loose files
threw on every launch. Their log stops right where that export begins.
Cyrillic, CJK or accented characters in an install path do the same on a
code page that cannot hold them.

- path_utf8() in Engine/Core/Platform/path_text.h: UTF-8 represents every
  path Windows allows, so it never throws.
- Every path that becomes a narrow string uses it -- 35 of them across 13
  files. Most are log lines and error messages, which were a second trap:
  reporting a problem with a path threw this instead of the problem.
- The ~45 .string() calls on Json values are untouched; those are not
  paths.

Nothing changes for an ASCII path, where both encodings agree, and a path
that is not ASCII had no working behaviour to preserve.

Reported with a log by a player on Discord. Moving the game to an ordinary
folder works around it on an unpatched build.
This commit is contained in:
AyeZeeBB committed 2026-10-03 17:36:12 -05:00
1 parent 201571c225
commit 1004e012bf
14 files changed
+81 -43

No files matched your search

+25
View File
@@ -0,0 +1,25 @@
#pragma once
#include <filesystem>
#include <string>
namespace dingosdk {
// A path as narrow text.
//
// std::filesystem::path::string() converts to the process's ANSI code page and
// throws when a character has no mapping there: "No mapping for the Unicode
// character exists in the target multi-byte code page". A folder named in
// Cyrillic or CJK is enough, and so is an invisible one -- a directory called
// U+200E (left-to-right mark) has been seen in the wild. The failure lands far
// from the cause, because the path is usually only being written into a log or
// an error message at the time.
//
// UTF-8 can represent every path Windows allows, so every path that becomes a
// narrow string goes through here instead.
inline std::string path_utf8(const std::filesystem::path& path) {
const auto text = path.u8string();
return {text.begin(), text.end()};
}
} // namespace dingosdk
+4 -3
View File
@@ -1,5 +1,6 @@
#include "game_archives.h"
#include "Engine/Resource/cas_codec.h"
#include "Engine/Core/Platform/path_text.h"
#include <cstdio>
#include <cstring>
#include <fstream>
@@ -26,7 +27,7 @@ Layout read_layout(const fs::path& path) {
Layout layout{{std::istreambuf_iterator<char>(input), std::istreambuf_iterator<char>()}, {}};
if (layout.buffer.size() <= native_db::envelope_size ||
std::memcmp(layout.buffer.data(), native_db::magic, sizeof(native_db::magic)))
throw std::runtime_error("Unrecognized native envelope in " + path.string());
throw std::runtime_error("Unrecognized native envelope in " + path_utf8(path));
layout.root = native_db::read(std::span<const unsigned char>(layout.buffer).subspan(native_db::envelope_size),
"layout.toc", nullptr, {.unique_fields = false});
return layout;
@@ -108,11 +109,11 @@ std::vector<std::byte> GameArchives::read(const fs::path& root, const fb::CasIde
const auto path = root / L"Win32" / fs::path(directory(location.installChunk)) /
fs::path(archive_file(location.archive));
std::ifstream input(path, std::ios::binary);
if (!input) throw std::runtime_error("Cannot open " + path.string());
if (!input) throw std::runtime_error("Cannot open " + path_utf8(path));
input.seekg(static_cast<std::streamoff>(offset));
std::vector<std::byte> bytes(size);
if (size && !input.read(reinterpret_cast<char*>(bytes.data()), static_cast<std::streamsize>(size)))
throw std::runtime_error("Cannot read " + path.string());
throw std::runtime_error("Cannot read " + path_utf8(path));
return bytes;
}
+3 -2
View File
@@ -1,5 +1,6 @@
#include "game_bundles.h"
#include "Engine/Resource/cas_codec.h"
#include "Engine/Core/Platform/path_text.h"
#include <fstream>
#include <stdexcept>
@@ -50,10 +51,10 @@ GameData::GameData(fs::path gameRoot)
fb::TocDocument GameData::read_toc(std::string_view relative) const {
const auto path = gameRoot_ / L"Data" / fs::path(relative);
std::ifstream input(path, std::ios::binary);
if (!input) throw std::runtime_error("Cannot open " + path.string());
if (!input) throw std::runtime_error("Cannot open " + path_utf8(path));
std::vector<std::byte> bytes(static_cast<std::size_t>(fs::file_size(path)));
if (!input.read(reinterpret_cast<char*>(bytes.data()), static_cast<std::streamsize>(bytes.size())))
throw std::runtime_error("Cannot read " + path.string());
throw std::runtime_error("Cannot read " + path_utf8(path));
return fb::read_toc(bytes);
}
+11 -10
View File
@@ -5,6 +5,7 @@
#include "Engine/Game/Build/supported_build.h"
#include "Engine/Game/Build/addresses.h"
#include "Engine/Game/Build/20260929/initfs.h"
#include "Engine/Core/Platform/path_text.h"
#include <Windows.h>
#include <bcrypt.h>
@@ -37,13 +38,13 @@ std::string lower(std::string_view text) {
}
Bytes read_file(const fs::path& path, std::size_t maximum) {
std::ifstream input(path, std::ios::binary | std::ios::ate);
if (!input) fail("Cannot open " + path.string());
if (!input) fail("Cannot open " + path_utf8(path));
const auto length = input.tellg();
if (length < 0 || static_cast<std::uint64_t>(length) > maximum) fail("File exceeds size limit: " + path.string());
if (length < 0 || static_cast<std::uint64_t>(length) > maximum) fail("File exceeds size limit: " + path_utf8(path));
Bytes bytes(static_cast<std::size_t>(length));
input.seekg(0);
if (!bytes.empty() && !input.read(reinterpret_cast<char*>(bytes.data()), static_cast<std::streamsize>(bytes.size())))
fail("Cannot read " + path.string());
fail("Cannot read " + path_utf8(path));
return bytes;
}
@@ -128,7 +129,7 @@ void apply_archive(const fs::path& path, const Secret& key, Entries& effective,
auto input = read_file(path, maximum_archive);
if (input.size() <= native_db::envelope_size ||
std::memcmp(input.data(), native_db::magic, sizeof(native_db::magic)))
fail("Unrecognized native envelope in " + path.string());
fail("Unrecognized native envelope in " + path_utf8(path));
const auto body = std::span<const unsigned char>(input).subspan(native_db::envelope_size);
std::size_t consumed{};
auto root = native_db::read(body, "InitFS", &consumed);
@@ -160,12 +161,12 @@ void apply_archive(const fs::path& path, const Secret& key, Entries& effective,
if (const auto* file_system = file->field("fs"); file_system &&
(file_system->type != 7 || (!file_system->text.empty() && file_system->text != "/data")))
fail("Unsupported InitFS filesystem destination for " + name->text);
Entry entry{name->text, path.string(), Bytes(payload->bytes.begin(), payload->bytes.end())};
Entry entry{name->text, path_utf8(path), Bytes(payload->bytes.begin(), payload->bytes.end())};
if (!entries.emplace(lower(name->text), std::move(entry)).second) fail("Duplicate InitFS virtual path");
}
if (!inherit) effective.clear();
for (auto& [name, entry] : entries) effective.insert_or_assign(name, std::move(entry));
sources.push_back(Json{{"path", path.string()}, {"sha256", launcher::sha256_file(path)}, {"inherit", inherit}});
sources.push_back(Json{{"path", path_utf8(path)}, {"sha256", launcher::sha256_file(path)}, {"inherit", inherit}});
}
bool within(const fs::path& root, const fs::path& path) {
@@ -178,7 +179,7 @@ bool within(const fs::path& root, const fs::path& path) {
bool write_atomic(const fs::path& root, const fs::path& path, std::span<const unsigned char> bytes, bool replace) {
if (!within(root, fs::weakly_canonical(path))) fail("Output path escapes the export directory");
if (!replace && fs::exists(path)) {
if (!fs::is_regular_file(path)) fail("Output path is not a regular file: " + path.string());
if (!fs::is_regular_file(path)) fail("Output path is not a regular file: " + path_utf8(path));
return false;
}
fs::create_directories(path.parent_path());
@@ -186,17 +187,17 @@ bool write_atomic(const fs::path& root, const fs::path& path, std::span<const un
const auto temporary = fs::path(path.wstring() + L".reskate-" + std::to_wstring(GetCurrentProcessId()) +
L"-" + std::to_wstring(GetTickCount64()) + L"-" + std::to_wstring(sequence.fetch_add(1)) + L".tmp");
const auto file = CreateFileW(temporary.c_str(), GENERIC_WRITE, 0, nullptr, CREATE_NEW, FILE_ATTRIBUTE_NORMAL, nullptr);
if (file == INVALID_HANDLE_VALUE) fail("Cannot create output file: " + path.string());
if (file == INVALID_HANDLE_VALUE) fail("Cannot create output file: " + path_utf8(path));
DWORD written{};
const bool ready = WriteFile(file, bytes.data(), static_cast<DWORD>(bytes.size()), &written, nullptr) &&
written == bytes.size() && FlushFileBuffers(file);
CloseHandle(file);
if (!ready) { DeleteFileW(temporary.c_str()); fail("Cannot write output file: " + path.string()); }
if (!ready) { DeleteFileW(temporary.c_str()); fail("Cannot write output file: " + path_utf8(path)); }
if (MoveFileExW(temporary.c_str(), path.c_str(), MOVEFILE_WRITE_THROUGH | (replace ? MOVEFILE_REPLACE_EXISTING : 0))) return true;
const auto error = GetLastError();
DeleteFileW(temporary.c_str());
if (!replace && (error == ERROR_ALREADY_EXISTS || error == ERROR_FILE_EXISTS) && fs::is_regular_file(path)) return false;
fail("Cannot publish output file: " + path.string());
fail("Cannot publish output file: " + path_utf8(path));
}
Json read_preferences(const fs::path& root) {
const auto path = root / L"ReSkate.settings.json";
+2 -1
View File
@@ -1,5 +1,6 @@
#include "item_thumbnails.h"
#include "game_bundles.h"
#include "Engine/Core/Platform/path_text.h"
#include <algorithm>
#include <stdexcept>
@@ -32,7 +33,7 @@ ThumbnailRead read_build_kit_thumbnails(const std::filesystem::path& gameRoot, s
bundle = data.read_bundle(toc, bundle_name);
} catch (const std::exception& failure) {
if (problems.size() < 600)
problems += "; " + it->path().filename().string() + ": " + failure.what();
problems += "; " + path_utf8(it->path().filename()) + ": " + failure.what();
}
}
if (!bundle) throw std::runtime_error("The game's thumbnail bundle was not found" + problems);
+6 -5
View File
@@ -2,6 +2,7 @@
#include "Engine/Game/Build/supported_build.h"
#include "Engine/Core/Json/json.h"
#include "Engine/Core/Platform/path_text.h"
#include <Windows.h>
@@ -35,15 +36,15 @@ std::string lower(std::string_view text) {
std::vector<unsigned char> read_file(const fs::path& path, std::size_t maximum) {
std::ifstream input(path, std::ios::binary | std::ios::ate);
if (!input) throw std::runtime_error("Cannot open " + path.string());
if (!input) throw std::runtime_error("Cannot open " + path_utf8(path));
const auto length = input.tellg();
if (length < 0 || static_cast<std::uint64_t>(length) > maximum)
throw std::runtime_error("File exceeds size limit: " + path.string());
throw std::runtime_error("File exceeds size limit: " + path_utf8(path));
std::vector<unsigned char> bytes(static_cast<std::size_t>(length));
input.seekg(0);
if (!bytes.empty() && !input.read(reinterpret_cast<char*>(bytes.data()),
static_cast<std::streamsize>(bytes.size())))
throw std::runtime_error("Cannot read " + path.string());
throw std::runtime_error("Cannot read " + path_utf8(path));
return bytes;
}
@@ -420,12 +421,12 @@ void save_mod_order(const std::filesystem::path& mods_root, const std::vector<Mo
{
std::ofstream output(temporary, std::ios::binary | std::ios::trunc);
if (!output || !output.write(text.data(), static_cast<std::streamsize>(text.size())) || !output.flush())
throw std::runtime_error("Cannot write " + temporary.string());
throw std::runtime_error("Cannot write " + path_utf8(temporary));
}
if (!MoveFileExW(temporary.c_str(), target.c_str(), MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH)) {
std::error_code ignored;
fs::remove(temporary, ignored);
throw std::runtime_error("Cannot replace " + target.string() + " (Windows error " +
throw std::runtime_error("Cannot replace " + path_utf8(target) + " (Windows error " +
std::to_string(GetLastError()) + ")");
}
}
+2 -1
View File
@@ -1,4 +1,5 @@
#include "mod_merge_internal.h"
#include "Engine/Core/Platform/path_text.h"
#include <fstream>
#include <limits>
@@ -24,7 +25,7 @@ fb::BundleFileInfo CasStore::write(std::uint32_t installChunk, std::uint16_t arc
std::ofstream out(path, std::ios::binary | std::ios::app);
if (!out || !out.write(reinterpret_cast<const char*>(encoded.data()),
static_cast<std::streamsize>(encoded.size())))
throw std::runtime_error("Cannot append to " + path.string());
throw std::runtime_error("Cannot append to " + path_utf8(path));
fb::BundleFileInfo info{{true, installChunk, archive}, static_cast<std::uint32_t>(offset),
static_cast<std::uint32_t>(encoded.size())};
offset += encoded.size();
+2 -1
View File
@@ -5,6 +5,7 @@
#include "Engine/Resource/ebx_merge.h"
#include "Engine/Resource/ebx_writer.h"
#include "Engine/Resource/shader_lookup.h"
#include "Engine/Core/Platform/path_text.h"
#include <algorithm>
#include <map>
@@ -153,7 +154,7 @@ fb::TocDocument combine(const fs::path& baseToc, const fs::path& baseRoot,
auto region = fb::read_bundle_region(bundle.region);
const auto provider = providers.find(key);
const auto shared = provider != providers.end() && provider->second > 1;
const auto modName = isBase ? std::string("base") : root.filename().string();
const auto modName = isBase ? std::string("base") : path_utf8(root.filename());
// The bundle holding the live material grid is rebuilt to carry the
// combined one, and a mod whose surfaces moved has its collision
// renumbered; both need the asset list even where one mod ships it.
+13 -12
View File
@@ -1,5 +1,6 @@
#include "mod_merge_internal.h"
#include "Engine/Core/Json/json.h"
#include "Engine/Core/Platform/path_text.h"
#include <Windows.h>
#include <bcrypt.h>
@@ -21,15 +22,15 @@ std::string lower(std::string_view text) {
std::vector<std::byte> read_file(const fs::path& path) {
std::ifstream input(path, std::ios::binary | std::ios::ate);
if (!input) throw std::runtime_error("Cannot open " + path.string());
if (!input) throw std::runtime_error("Cannot open " + path_utf8(path));
const auto length = input.tellg();
if (length < 0 || static_cast<std::uint64_t>(length) > maximumTocBytes)
throw std::runtime_error("File exceeds the TOC size limit: " + path.string());
throw std::runtime_error("File exceeds the TOC size limit: " + path_utf8(path));
std::vector<std::byte> bytes(static_cast<std::size_t>(length));
input.seekg(0);
if (!bytes.empty() &&
!input.read(reinterpret_cast<char*>(bytes.data()), static_cast<std::streamsize>(bytes.size())))
throw std::runtime_error("Cannot read " + path.string());
throw std::runtime_error("Cannot read " + path_utf8(path));
return bytes;
}
@@ -40,12 +41,12 @@ void write_file(const fs::path& path, std::span<const std::byte> bytes) {
std::ofstream output(temporary, std::ios::binary | std::ios::trunc);
if (!output || !output.write(reinterpret_cast<const char*>(bytes.data()),
static_cast<std::streamsize>(bytes.size())))
throw std::runtime_error("Cannot write " + temporary.string());
throw std::runtime_error("Cannot write " + path_utf8(temporary));
}
if (!MoveFileExW(temporary.c_str(), path.c_str(), MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH)) {
const auto error = GetLastError();
DeleteFileW(temporary.c_str());
throw std::runtime_error("Cannot publish " + path.string() +
throw std::runtime_error("Cannot publish " + path_utf8(path) +
" (Windows error " + std::to_string(error) + ")");
}
}
@@ -64,12 +65,12 @@ void unshare(const fs::path& path) {
const auto temporary = fs::path(path.wstring() + L".unshared");
std::error_code error;
fs::copy_file(path, temporary, fs::copy_options::overwrite_existing, error);
if (error) throw std::runtime_error("Cannot give " + path.string() + " a private copy: " +
if (error) throw std::runtime_error("Cannot give " + path_utf8(path) + " a private copy: " +
error.message());
if (!MoveFileExW(temporary.c_str(), path.c_str(), MOVEFILE_REPLACE_EXISTING)) {
const auto failure = GetLastError();
DeleteFileW(temporary.c_str());
throw std::runtime_error("Cannot replace the shared " + path.string() +
throw std::runtime_error("Cannot replace the shared " + path_utf8(path) +
" (Windows error " + std::to_string(failure) + ")");
}
}
@@ -78,22 +79,22 @@ std::uint64_t append_file(const fs::path& from, const fs::path& to) {
fs::create_directories(to.parent_path());
unshare(to);
std::ifstream input(from, std::ios::binary);
if (!input) throw std::runtime_error("Cannot read " + from.string());
if (!input) throw std::runtime_error("Cannot read " + path_utf8(from));
// tellp() on a stream opened for append reads 0 until the first write, so
// the block's start comes from the file itself.
std::error_code error;
const auto existing = fs::file_size(to, error);
const auto start = error ? std::uint64_t{} : existing;
std::ofstream output(to, std::ios::binary | std::ios::app);
if (!output) throw std::runtime_error("Cannot append to " + to.string());
if (!output) throw std::runtime_error("Cannot append to " + path_utf8(to));
// A megabyte of stack is a stack overflow; the copy buffer lives on the heap.
std::vector<char> buffer(1 << 20);
while (input) {
input.read(buffer.data(), static_cast<std::streamsize>(buffer.size()));
if (input.gcount() && !output.write(buffer.data(), input.gcount()))
throw std::runtime_error("Cannot append to " + to.string());
throw std::runtime_error("Cannot append to " + path_utf8(to));
}
if (!output) throw std::runtime_error("Cannot append to " + to.string());
if (!output) throw std::runtime_error("Cannot append to " + path_utf8(to));
return start;
}
@@ -104,7 +105,7 @@ void link_or_copy(const fs::path& from, const fs::path& to) {
fs::remove(to, error);
if (CreateHardLinkW(to.c_str(), from.c_str(), nullptr)) return;
fs::copy_file(from, to, fs::copy_options::overwrite_existing, error);
if (error) throw std::runtime_error("Cannot place " + to.string() + ": " + error.message());
if (error) throw std::runtime_error("Cannot place " + path_utf8(to) + ": " + error.message());
}
RelativeFiles scan(const fs::path& directory) {
+3 -2
View File
@@ -2,6 +2,7 @@
#include "Engine/Resource/cas_codec.h"
#include "Engine/Resource/ebx_document.h"
#include "Engine/Resource/ebx_writer.h"
#include "Engine/Core/Platform/path_text.h"
#include <optional>
#include <stdexcept>
@@ -128,7 +129,7 @@ GridPlan plan_material_grid(const std::vector<const Mod*>& mods,
// Lowest priority first, the order the merge layers mods in.
for (auto at = mods.rbegin(); at != mods.rend(); ++at) {
const auto* mod = *at;
const auto modName = mod->directory.filename().string();
const auto modName = path_utf8(mod->directory.filename());
for (const auto& relative : modFiles.at(mod).tocs) {
try {
const auto toc = fb::read_toc(read_file(mod->directory / fs::path(relative)));
@@ -190,7 +191,7 @@ GridPlan plan_material_grid(const std::vector<const Mod*>& mods,
grid::Combiner combiner(*base);
std::size_t combined{};
for (auto& contribution : contributions) {
const auto modName = contribution.mod->directory.filename().string();
const auto modName = path_utf8(contribution.mod->directory.filename());
const auto before = combiner.slots();
grid::SlotMap slots;
try {
+2 -1
View File
@@ -1,5 +1,6 @@
#include "mod_merge_internal.h"
#include "Engine/Resource/ebx_document.h"
#include "Engine/Core/Platform/path_text.h"
#include <cstring>
#include <stdexcept>
@@ -33,7 +34,7 @@ std::vector<MergeReport::LoadScreen> read_load_screens(const std::vector<const M
const fs::path& gameRoot, MergeReport& report) {
std::vector<MergeReport::LoadScreen> result;
for (const auto* mod : mods) {
const auto modName = mod->directory.filename().string();
const auto modName = path_utf8(mod->directory.filename());
for (const auto& relative : modFiles.at(mod).tocs) {
if (lower(relative) != configurationToc) continue;
try {
+4 -3
View File
@@ -1,4 +1,5 @@
#include "mod_merge_internal.h"
#include "Engine/Core/Platform/path_text.h"
#include <Windows.h>
#include <algorithm>
@@ -69,7 +70,7 @@ std::string merge_fingerprint(const Catalog& catalog, const std::vector<const Mo
if (it->is_regular_file(error) && !error) files.push_back(it->path());
error.clear();
}
if (error) throw std::runtime_error("Cannot list " + mod->directory.string());
if (error) throw std::runtime_error("Cannot list " + path_utf8(mod->directory));
std::ranges::sort(files);
for (const auto& file : files)
describe(utf8(fs::relative(file, mod->directory, error)), file);
@@ -151,10 +152,10 @@ void write_stamp(const fs::path& output, const std::string& fingerprint, const M
for (fs::recursive_directory_iterator it(output, error), end; it != end && !error; it.increment(error)) {
if (!it->is_regular_file(error) || error) { error.clear(); continue; }
const auto size = fs::file_size(it->path(), error);
if (error) throw std::runtime_error("Cannot size " + it->path().string());
if (error) throw std::runtime_error("Cannot size " + path_utf8(it->path()));
text += "file " + std::to_string(size) + ' ' + utf8(fs::relative(it->path(), output, error)) + '\n';
}
if (error) throw std::runtime_error("Cannot list " + output.string());
if (error) throw std::runtime_error("Cannot list " + path_utf8(output));
for (auto note : report.notes) {
std::ranges::replace(note, '\n', ' ');
std::ranges::replace(note, '\r', ' ');
+2 -1
View File
@@ -3,6 +3,7 @@
#include "game_bundles.h"
#include "Engine/Core/Json/json.h"
#include "Engine/Resource/ebx_document.h"
#include "Engine/Core/Platform/path_text.h"
#include <algorithm>
#include <array>
#include <fstream>
@@ -252,7 +253,7 @@ WorldLayerCatalog load_or_scan(const fs::path& gameRoot, const fs::path& file) {
WorldLayerCatalog read(const fs::path& file) {
std::ifstream input(file, std::ios::binary);
if (!input) throw std::runtime_error("Cannot open " + file.string());
if (!input) throw std::runtime_error("Cannot open " + path_utf8(file));
const std::string text{std::istreambuf_iterator<char>(input), std::istreambuf_iterator<char>()};
return from_json(Json::parse(text.begin(), text.end()));
}
+2 -1
View File
@@ -5,6 +5,7 @@
#include "Engine/Core/Platform/launcher_support.h"
#include "launcher_update_config.h"
#include "text_encoding.h"
#include "Engine/Core/Platform/path_text.h"
#include <winhttp.h>
#include <miniz.h>
@@ -197,7 +198,7 @@ struct Zip {
// Writes one ZIP entry to `output`.
void extract_index(Zip& zip, mz_uint index, const mz_zip_archive_file_stat& entry, const fs::path& output) {
Handle file{CreateFileW(output.c_str(), GENERIC_WRITE, 0, nullptr, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, nullptr)};
if (file.value == INVALID_HANDLE_VALUE) fail("Cannot create " + output.string());
if (file.value == INVALID_HANDLE_VALUE) fail("Cannot create " + path_utf8(output));
struct Sink { HANDLE file; std::uint64_t count{}; } sink{file.value};
const auto write = [](void* opaque, mz_uint64 offset, const void* buffer, size_t count) -> size_t {
auto& target = *static_cast<Sink*>(opaque);