diff --git a/Engine/Game/Multiplayer/tick_settings.h b/Engine/Game/Multiplayer/tick_settings.h index 3f534404..43fa9b37 100644 --- a/Engine/Game/Multiplayer/tick_settings.h +++ b/Engine/Game/Multiplayer/tick_settings.h @@ -18,4 +18,10 @@ constexpr bool valid_pose_interval(std::uint32_t interval) noexcept { if (interval == multiplayer_pose_interval(rate)) return true; return false; } +// The most poses a second one player is sent on a dedicated server, whatever the crowd +// around them (Extension/Multiplayer/Session/room.h, crowd_limits); 0: no limit. +inline constexpr unsigned crowd_pose_budget = 600, min_crowd_budget = 300, max_crowd_budget = 20000; +constexpr bool valid_crowd_budget(unsigned budget) noexcept { + return !budget || (budget >= min_crowd_budget && budget <= max_crowd_budget); +} } // namespace dingosdk diff --git a/Extension/Multiplayer/Net/delta_codec.cpp b/Extension/Multiplayer/Net/delta_codec.cpp index 2ad5c1d9..1d71a4c6 100644 --- a/Extension/Multiplayer/Net/delta_codec.cpp +++ b/Extension/Multiplayer/Net/delta_codec.cpp @@ -43,7 +43,8 @@ WireUpdate DeltaSender::build(const Packet &p, std::span raw } const auto found = bases_.find({p.source, p.kind}); if (found == bases_.end() || found->second.epoch != p.epoch || found->second.world != p.world || p.time_us < found->second.time || - (p.kind != PacketKind::cosmetics && p.time_us - found->second.time >= 2000000)) { + (p.kind != PacketKind::cosmetics && + (aligned_ ? p.time_us / refresh_us_ != found->second.time / refresh_us_ : p.time_us - found->second.time >= refresh_us_))) { if (wire.size() + 4 > max_packet) return out; out.bytes.assign(wire.begin(), wire.end()); diff --git a/Extension/Multiplayer/Net/delta_codec.h b/Extension/Multiplayer/Net/delta_codec.h index 7ca5fbfc..456ced83 100644 --- a/Extension/Multiplayer/Net/delta_codec.h +++ b/Extension/Multiplayer/Net/delta_codec.h @@ -32,6 +32,8 @@ class DeltaSender { }; std::map bases_; std::uint64_t clock_{}; + std::uint64_t refresh_us_ = 2000000; + bool aligned_{}; WireUpdate build(const Packet &, std::span raw, std::span wire, DeltaCache *cache) const; @@ -44,6 +46,16 @@ class DeltaSender { WireUpdate prepare(const Packet &, std::span raw, std::span wire, DeltaCache &cache) const; void sent(const Packet &, WireUpdate &&); + // How old a pose or sound reference may get before a whole state replaces it. Each one + // is sent reliably, so Steam resends it until it arrives: a dedicated server, with a + // stream for every pair of players, keeps them rarer than the two seconds a game does. + // `aligned`: on the source's own clock, at each multiple of the time, instead of that + // long after this recipient's last one. Every recipient of a source then takes the same + // packet as its reference, and one patch serves them all (DeltaCache). + void set_refresh(std::uint64_t microseconds, bool aligned = false) { + refresh_us_ = microseconds; + aligned_ = aligned; + } }; class DeltaReceiver { struct Base { diff --git a/Extension/Multiplayer/Net/protocol.h b/Extension/Multiplayer/Net/protocol.h index 25c3540e..85792c60 100644 --- a/Extension/Multiplayer/Net/protocol.h +++ b/Extension/Multiplayer/Net/protocol.h @@ -322,6 +322,8 @@ class PoseBuffer { bool sample(std::uint64_t now_us, Pose &out) const; std::optional sample(std::uint64_t now_us) const; bool sample_remote(std::uint64_t now_us, Pose &out); + // Whether a pose arrived within this long: sampling gives up after a second without one. + bool heard_within(std::uint64_t now_us, std::uint64_t age_us) const; std::optional sample_remote(std::uint64_t now_us); PosePlayback playback() const { return playback_; } void clear(); diff --git a/Extension/Multiplayer/Remote/playback_buffers.cpp b/Extension/Multiplayer/Remote/playback_buffers.cpp index 7d1ac485..79f461d9 100644 --- a/Extension/Multiplayer/Remote/playback_buffers.cpp +++ b/Extension/Multiplayer/Remote/playback_buffers.cpp @@ -314,6 +314,9 @@ void PoseBuffer::correct(Pose &pose, std::uint64_t now) const { } offset_pose(pose, offset); } +bool PoseBuffer::heard_within(std::uint64_t now, std::uint64_t age) const { + return !frames_.empty() && (now < frames_.back().arrival || now - frames_.back().arrival <= age); +} bool PoseBuffer::sample_remote(std::uint64_t now, Pose &pose) { if (playback_at_ && (now < playback_at_ || now - playback_at_ > 200000)) { correction_start_ = correction_end_ = 0; diff --git a/Extension/Multiplayer/Session/room.h b/Extension/Multiplayer/Session/room.h index f1688390..49958abf 100644 --- a/Extension/Multiplayer/Session/room.h +++ b/Extension/Multiplayer/Session/room.h @@ -1,7 +1,9 @@ #pragma once #include "Extension/Multiplayer/Net/protocol.h" #include +#include #include +#include #include namespace dingosdk::multiplayer { @@ -76,7 +78,40 @@ struct PoseDelivery { std::uint64_t source{}, epoch{}, last_sent{}; std::uint32_t interval_us = 50000; std::uint64_t next_source_time{}; + std::uint32_t by_distance{}; // what the distance alone asked for, before any crowd limit }; +// A crowd in one place: every player there is within full-rate distance of every other, and +// what one player is sent grows with the crowd until their connection cannot carry it. The +// default budget is what was seen to arrive through Steam's relays (about 1 MB/s a player, +// whatever the send rate): nobody near is slowed until about 20 players are in one place at +// 30 TPS; past that the farthest of them are. A +// player is sent at most `budget` poses a second: everyone at the low rate at least, the +// nearest at the full rate with two thirds of what is left, the next nearest at the half +// rate with the rest. These are the squared distances beyond which a player is sent at the +// half and at the low rate; `squared` (the distances to the others) is sorted here. +struct CrowdLimits { + float half = std::numeric_limits::infinity(), low = std::numeric_limits::infinity(); +}; +inline CrowdLimits crowd_limits(std::span squared, unsigned tps, unsigned budget = crowd_pose_budget) { + const unsigned full = 1000000U / multiplayer_pose_interval(tps), half = 10, low = 5; + const auto count = static_cast(squared.size()); + if (count * full <= budget) return {}; + std::sort(squared.begin(), squared.end()); + const unsigned spare = budget > count * low ? budget - count * low : 0; + const unsigned at_full = std::min(count, spare * 2 / 3 / (full - low)); + const unsigned at_half = std::min(count - at_full, (spare - at_full * (full - low)) / (half - low)); + CrowdLimits limits; + // Between two players the limit falls halfway, so that neither sits on it. + const auto after = [&](unsigned sent) { + return sent >= count ? std::numeric_limits::infinity() : sent ? (squared[sent - 1] + squared[sent]) / 2 : -1.f; + }; + limits.half = after(at_full); + limits.low = after(at_full + at_half); + return limits; +} +inline std::uint32_t crowd_interval(std::uint32_t interval, float distance_squared, const CrowdLimits &limits) noexcept { + return distance_squared > limits.low ? 200000U : distance_squared > limits.half ? std::max(interval, 100000U) : interval; +} // Preserve the timer phase across variable client frames/packet arrivals. Only // the newest state is sent; elapsed slots are skipped without catch-up packets. inline void advance_pose_deadline(std::uint64_t &next, std::uint64_t now, std::uint64_t interval) { diff --git a/Extension/Multiplayer/Session/session.cpp b/Extension/Multiplayer/Session/session.cpp index 697b2100..08304d2c 100644 --- a/Extension/Multiplayer/Session/session.cpp +++ b/Extension/Multiplayer/Session/session.cpp @@ -503,13 +503,19 @@ void render(Session &s, std::uintptr_t client, const NativeFrame &local, std::ui if (local.ready && !p.render_failed) { const bool sampled = s.mode == Mode::echo ? p.poses.sample(now, p.render_pose) : p.poses.sample_remote(now, p.render_pose); + // Nothing new from them for over a second (a stall here, at the host or on the way): + // their skater stays where it was for a while longer. Taking it down and building + // it again three seconds later is the most expensive thing a frame can do, and on + // a busy server one slow frame made it happen to everyone at once, which made + // the next frames slower still. + const bool held = !sampled && was_visible && p.poses.heard_within(now, 8000000); // Showing a player without an actor spawns one (native_skater_spawn.cpp). const bool spawning = sampled && p.appearance.value() && !was_visible && !remote_skater_entity(); if (spawning && now < p.next_spawn) { p.native_status = "Waiting to show the player again."; } else if (spawning && native_pass) { p.native_status = "Waiting for another player's skater to finish spawning."; - } else if (sampled && p.appearance.value()) { + } else if ((sampled || held) && p.appearance.value()) { if (spawning) { native_pass = true; p.applied_cosmetics = 0; // the new actor wears no recipe yet diff --git a/Extension/Multiplayer/Session/session_internal.h b/Extension/Multiplayer/Session/session_internal.h index 3229f371..343bd1fb 100644 --- a/Extension/Multiplayer/Session/session_internal.h +++ b/Extension/Multiplayer/Session/session_internal.h @@ -1,6 +1,7 @@ #pragma once #include "Engine/Game/Multiplayer/chat_rate.h" #include "session.h" +#include "Extension/Multiplayer/developer_identity.h" #include "Extension/Customization/developer_hoodie.h" #include "Extension/Customization/developer_board.h" #include "Extension/Multiplayer/Remote/native_skater.h" @@ -324,9 +325,12 @@ inline bool dedicated_host(const Session &s) { return s.mode == Mode::join && ga // Whether Steam itself vouches for this player's identity to this PC: a host's guests and a // guest's host are connected directly, and so is another guest once the direct handshake is // done. Anyone else is known only from the host's roster, which a host can fill as it likes, -// so what rests on who a player is (the developer and friend marks) waits for this. +// so what rests on who a player is (the developer and friend marks) waits for this. An +// official server is the exception: Steam vouches for it to this PC and for each player to +// it, and it is ours, so its roster is taken at its word. On a busy one a player is +// connected directly to only the few nearest, and the rest would show no tag. inline bool steam_vouched(const Session &s, const Peer &peer) { - return s.mode != Mode::join || peer.member.id == s.host_id || peer.direct_ready; + return s.mode != Mode::join || peer.member.id == s.host_id || peer.direct_ready || official_server(s.host_id); } // Whether a player shows the tag the backend gives them, and whether the items that come with // it animate: not until their appearance has arrived, and not when it says they turned that diff --git a/Extension/Multiplayer/Session/session_receive.cpp b/Extension/Multiplayer/Session/session_receive.cpp index b1cda91e..3353b013 100644 --- a/Extension/Multiplayer/Session/session_receive.cpp +++ b/Extension/Multiplayer/Session/session_receive.cpp @@ -73,7 +73,8 @@ constexpr std::size_t chat_proofs_kept = 8; // Whether a line the host passed on as this player's would show a badge that rests on who // they are: another guest's, known to Steam, on one of the backend's lists and showing it. bool owes_proof(const Session &s, const Peer &sender) { - return s.mode == Mode::join && sender.member.id != s.host_id && steam_vouched(s, sender) && shows_tag(sender) && + // Their own copy only comes over a direct connection: without one there is none to wait for. + return s.mode == Mode::join && sender.member.id != s.host_id && sender.direct_ready && shows_tag(sender) && identity_mark(sender.member.id); } std::string chat_name(Session &s, const Peer &sender) { diff --git a/Extension/Multiplayer/Steam/steam_lanes.h b/Extension/Multiplayer/Steam/steam_lanes.h index 4eb11b68..e7fbb105 100644 --- a/Extension/Multiplayer/Steam/steam_lanes.h +++ b/Extension/Multiplayer/Steam/steam_lanes.h @@ -53,6 +53,10 @@ inline std::uint64_t lane_queue_time(const SteamNetConnectionRealTimeStatus_t &i if (time == std::numeric_limits::max()) return 0; return static_cast(std::max(0, time)); } +// A whole connection's queue time, with the same care for Steam's "cannot estimate" value. +inline std::uint64_t queue_time(const SteamNetConnectionRealTimeStatus_t &info) { + return lane_queue_time(info, {}, TrafficLane::gameplay); +} inline bool lane_congested(const SteamNetConnectionRealTimeStatus_t &info, std::span lanes, TrafficLane lane) { if (info.m_eState != k_ESteamNetworkingConnectionState_Connected) return false; diff --git a/Extension/Multiplayer/Steam/steam_server_browser.cpp b/Extension/Multiplayer/Steam/steam_server_browser.cpp index 4b0f07ae..50c37a92 100644 --- a/Extension/Multiplayer/Steam/steam_server_browser.cpp +++ b/Extension/Multiplayer/Steam/steam_server_browser.cpp @@ -159,6 +159,10 @@ const MultiplayerLobby *SteamServerBrowser::find(std::uint64_t id) const { } void SteamServerBrowser::refresh(std::uint64_t now) { if (!api().open()) return; + // Steam stops answering a game that asks often (an empty list, for minutes): one search at a + // time, a quarter of a minute apart, and three quarters after one that came back empty. + // Opening or refreshing the browser sooner shows what the last search found. + if (started_ && (!searches_.empty() || now - started_ < (internet_listed_ ? 15000000 : 45000000))) return; servers_ = api().servers(); if (!servers_) return; release(); @@ -248,11 +252,12 @@ void SteamServerBrowser::tick(std::uint64_t now) { if (done) { release(); // An internet list with servers in it is the whole truth: what it left out is gone. - // An empty one is Steam not answering (see the header): keep what was listed in the - // last two minutes. + // An empty one is Steam not answering (see the header), and it has stayed that way for + // minutes for a player who searched often: keep what was listed in the last quarter + // of an hour. A server that closed meanwhile only fails to join. const auto before = found_.size(); std::erase_if(found_, [&](const auto &entry) { - return entry.second.search != search_ && (internet_listed_ || now - entry.second.listed > 120000000); + return entry.second.search != search_ && (internet_listed_ || now - entry.second.listed > 900000000); }); if (found_.size() != before) read(now); std::string list; diff --git a/Extension/Multiplayer/Steam/steam_transport.cpp b/Extension/Multiplayer/Steam/steam_transport.cpp index 16ccf320..40153c6f 100644 --- a/Extension/Multiplayer/Steam/steam_transport.cpp +++ b/Extension/Multiplayer/Steam/steam_transport.cpp @@ -94,6 +94,39 @@ struct SteamTransport::Impl { bool recheck{}; // a status callback named this connection: read its state this poll }; std::map links; + // How the connections Steam closed ended (take_closed). A handful at most wait here. + std::map closed; + // What each connection may send a second, and Steam's call that changes an open one. + int send_rate = connection_send_rate, listen_rate = connection_send_rate; // now, and when the listen socket opened + void *utils{}; + bool (*set_config)(void *, ESteamNetworkingConfigValue, ESteamNetworkingConfigScope, intptr_t, + ESteamNetworkingConfigDataType, const void *){}; + // Gives one open connection the current send rate. A connection a listen socket + // accepts starts with the rate the socket was opened with, so each one is given it. + bool apply_rate(HSteamNetConnection handle) { + if (!handle || !set_config || !utils) return false; + bool ok = true; + const auto set = [&](ESteamNetworkingConfigValue option, int32 value) { + ok &= set_config(utils, option, k_ESteamNetworkingConfig_Connection, static_cast(handle), + k_ESteamNetworkingConfig_Int32, &value); + }; + // The minimum never above the maximum on the way, whatever the rate was before. + set(k_ESteamNetworkingConfig_SendRateMin, min_send_rate); + set(k_ESteamNetworkingConfig_SendRateMax, send_rate); + set(k_ESteamNetworkingConfig_SendRateMin, send_rate); + return ok; + } + // "ping 48 ms, quality 99%/97%, 0 B queued": what was last measured of a link. + static std::string measured(const Link &link) { + if (!link.status_ok) return "link not measured"; + const auto &s = link.status; + const auto percent = [](float quality) { return quality < 0 ? std::string("?") : std::to_string(static_cast(quality * 100.f + .5f)) + "%"; }; + return "ping " + std::to_string(s.m_nPing) + " ms, quality " + percent(s.m_flConnectionQualityLocal) + " here / " + + percent(s.m_flConnectionQualityRemote) + " there, " + std::to_string(s.m_cbPendingReliable + s.m_cbPendingUnreliable) + + " B queued (" + std::to_string(queue_time(s) / 1000) + " ms), " + + std::to_string(static_cast(s.m_flOutBytesPerSec / 1024.f + .5f)) + " KB/s out, " + + std::to_string(static_cast(s.m_flInBytesPerSec / 1024.f + .5f)) + " KB/s in"; + } std::uint64_t frame{}; ULONGLONG next_measure{}, next_sweep{}; struct Name { @@ -146,12 +179,20 @@ struct SteamTransport::Impl { reinterpret_cast(&changed)); return option; } - std::array options() { - std::array out{}; + std::array options() { + std::array out{}; out[0] = callback_option(); - // Raise this connection's ceiling; Steam retains congestion control and - // its default minimum. Do not alter the game's global Steam settings. - out[1].SetInt32(k_ESteamNetworkingConfig_SendRateMax, 1024 * 1024); + // This connection's send rate. Steam asks for the minimum and the maximum to be set + // to the same value: with only the maximum raised a connection stays at the 256 KB/s + // default, which a busy server fills for every player. Do not alter the game's + // global Steam settings. + listen_rate = send_rate; + out[1].SetInt32(k_ESteamNetworkingConfig_SendRateMax, send_rate); + out[3].SetInt32(k_ESteamNetworkingConfig_SendRateMin, send_rate); + // Room for what must arrive. Steam refuses a reliable message once 512 KB wait, its + // default, and a busy connection already holds half of that in poses: the refusal + // then ends the session for a player whose connection was only slow. + out[4].SetInt32(k_ESteamNetworkingConfig_SendBufferSize, 4 * 1024 * 1024); // Every connection goes through Steam's relays, which is how players who are not // Steam friends already connect: no player's IP address is shared with the others // in a session, whatever each one's own Steam setting is. @@ -303,6 +344,12 @@ bool SteamTransport::bind(void *library, void *sockets, void *networking_utils) } catch (...) { p.lanes = {}; } + p.utils = networking_utils; + try { + p.set_config = symbol(module, "SteamAPI_ISteamNetworkingUtils_SetConfigValue"); + } catch (...) { + p.set_config = nullptr; // open connections keep their rate; new ones get the new one + } symbol(module, "SteamAPI_ISteamNetworkingUtils_InitRelayNetworkAccess")(networking_utils); symbol( @@ -407,6 +454,9 @@ void SteamTransport::disconnect(std::uint64_t id, const char *reason) { return; const auto handle = found->second.handle; p.links.erase(found); + // Closed from this side, unless Steam's own account of the end is already kept. + if (p.closed.size() >= 256) p.closed.clear(); + p.closed.try_emplace(id, std::string("closed by this side: ") + (reason ? reason : "")); if (handle && p.close) p.close(p.sockets, handle, 1000, reason, false); p.state.detail = reason; @@ -476,6 +526,7 @@ bool SteamTransport::connect_peer(std::uint64_t id) { auto options = p.options(); const auto connection = p.connect(p.sockets, &identity, virtual_port, static_cast(options.size()), options.data()); + p.closed.erase(id); if (connection) p.links.emplace(id, Impl::Link{connection, GetTickCount64(), false, name(id), p.lanes.setup(p.sockets, connection)}); @@ -539,8 +590,10 @@ void SteamTransport::poll() { p.close(p.sockets, event.m_hConn, 4003, "Cannot accept connection", false); continue; } + p.closed.erase(id); p.links.emplace(id, Impl::Link{event.m_hConn, now, false, name(id), p.lanes.setup(p.sockets, event.m_hConn)}); + if (p.send_rate != p.listen_rate) p.apply_rate(event.m_hConn); } // Connected links report changes through the status callback (above), so read their // state only then, plus once a second in case a callback was dropped. @@ -565,6 +618,13 @@ void SteamTransport::poll() { } else if (info.m_eState == k_ESteamNetworkingConnectionState_ClosedByPeer || info.m_eState == k_ESteamNetworkingConnectionState_ProblemDetectedLocally) { const auto detail = bounded(info.m_szEndDebug, sizeof(info.m_szEndDebug)); + // Steam's end reasons: 1xxx the application closed it, 3xxx a problem at this end, + // 4xxx one at the other end (4001: it stopped answering), 5xxx anything else. + if (p.closed.size() >= 256) p.closed.clear(); + p.closed[id] = std::string(info.m_eState == k_ESteamNetworkingConnectionState_ClosedByPeer ? "closed by their game" + : "connection lost") + + ", Steam reason " + std::to_string(info.m_eEndReason) + (detail.empty() ? std::string{} : " \"" + detail + "\"") + + "; last " + Impl::measured(link); disconnect(id, detail.empty() ? "Steam connection closed." : detail.c_str()); } else if (now - link.connecting_since > 20000) disconnect(id, "Steam connection timed out."); @@ -575,6 +635,61 @@ void SteamTransport::poll() { } p.publish_links(); } +std::string SteamTransport::take_closed(std::uint64_t id) { + auto &p = *impl_; + const auto found = p.closed.find(id); + if (found == p.closed.end()) return {}; + auto text = std::move(found->second); + p.closed.erase(found); + return text; +} +int SteamTransport::send_rate() const { return impl_->send_rate; } +bool SteamTransport::set_send_rate(int rate) { + auto &p = *impl_; + if (rate < min_send_rate || rate > max_send_rate) return false; + p.send_rate = rate; + bool all = true; + for (auto &[id, link] : p.links) + if (link.handle) all &= p.apply_rate(link.handle); + return all; +} +std::int64_t SteamTransport::pending(std::uint64_t id) const { + const auto found = impl_->links.find(id); + if (found == impl_->links.end() || !found->second.status_ok) return 0; + return std::int64_t{found->second.status.m_cbPendingReliable} + found->second.status.m_cbPendingUnreliable; +} +std::string SteamTransport::link_report(std::uint64_t id) { + auto &p = *impl_; + const auto found = p.links.find(id); + if (found == p.links.end()) return "no connection"; + if (found->second.handle && found->second.connected) p.read_status(found->second); + return Impl::measured(found->second); +} +std::vector SteamTransport::links() { + auto &p = *impl_; + std::vector out; + out.reserve(p.links.size()); + for (auto &[id, link] : p.links) { + TransportLink row; + row.id = id; + row.connected = link.connected; + if (link.handle && link.connected) p.read_status(link); + if (link.connected && link.status_ok) { + const auto &s = link.status; + row.measured = true; + row.ping_ms = s.m_nPing; + row.pending_bytes = s.m_cbPendingReliable + s.m_cbPendingUnreliable; + row.send_rate = s.m_nSendRateBytesPerSecond; + row.quality_local = s.m_flConnectionQualityLocal; + row.quality_remote = s.m_flConnectionQualityRemote; + row.out_bps = s.m_flOutBytesPerSec; + row.in_bps = s.m_flInBytesPerSec; + row.queue_us = queue_time(s); + } + out.push_back(row); + } + return out; +} std::string SteamTransport::name(std::uint64_t id) { auto &p = *impl_; if (!p.friends || !id) @@ -674,33 +789,41 @@ std::vector SteamTransport::receive() { const auto &link = (it++)->second; if (!link.connected) continue; - SteamNetworkingMessage_t *messages[128]{}; - const int count = p.receive_messages(p.sockets, link.handle, messages, 128); - if (count < 0) { - disconnect(id, "Steam receive failed."); - continue; - } - for (int i = 0; i < count; ++i) { - auto *message = messages[i]; - if (!message) - continue; - struct Release { - Impl &p; - SteamNetworkingMessage_t *m; - ~Release() { p.release_message(m); } - } release{p, message}; - if (message->m_cbSize <= 0 || message->m_cbSize > static_cast(max_packet) || - !message->m_pData) { - ++p.state.dropped; - ++p.state.invalid_messages; - continue; + // Everything that has arrived, not one batch a call: a dedicated server sends a busy + // session's player a few thousand messages a second, and what a slow frame leaves + // behind would wait in Steam, later and later, taking memory until it is read. + bool failed{}; + for (unsigned batch = 0; batch < 64 && !failed; ++batch) { + SteamNetworkingMessage_t *messages[128]{}; + const int count = p.receive_messages(p.sockets, link.handle, messages, 128); + if (count < 0) { + failed = true; + break; } - const auto *data = static_cast(message->m_pData); - result.push_back({id, {data, data + message->m_cbSize}, - message->m_usecTimeReceived > 0 ? static_cast(message->m_usecTimeReceived) : 0}); - ++p.state.received; - p.state.received_bytes += static_cast(message->m_cbSize); + for (int i = 0; i < count; ++i) { + auto *message = messages[i]; + if (!message) + continue; + struct Release { + Impl &p; + SteamNetworkingMessage_t *m; + ~Release() { p.release_message(m); } + } release{p, message}; + if (message->m_cbSize <= 0 || message->m_cbSize > static_cast(max_packet) || + !message->m_pData) { + ++p.state.dropped; + ++p.state.invalid_messages; + continue; + } + const auto *data = static_cast(message->m_pData); + result.push_back({id, {data, data + message->m_cbSize}, + message->m_usecTimeReceived > 0 ? static_cast(message->m_usecTimeReceived) : 0}); + ++p.state.received; + p.state.received_bytes += static_cast(message->m_cbSize); + } + if (count < 128) break; } + if (failed) disconnect(id, "Steam receive failed."); } return result; } diff --git a/Extension/Multiplayer/Steam/steam_transport.h b/Extension/Multiplayer/Steam/steam_transport.h index d035b136..8f9fb97e 100644 --- a/Extension/Multiplayer/Steam/steam_transport.h +++ b/Extension/Multiplayer/Steam/steam_transport.h @@ -13,6 +13,15 @@ struct TransportPeer { std::uint64_t id{}; bool connected{}; }; +// What Steam measures of one connection right now (SteamTransport::links). +struct TransportLink { + std::uint64_t id{}; + bool connected{}, measured{}; + int ping_ms{}, pending_bytes{}, send_rate{}; // send_rate: bytes a second Steam will let out to them + float quality_local = -1, quality_remote = -1; // 0-1 of packets delivered each way; -1 unknown + float out_bps{}, in_bps{}; + std::uint64_t queue_us{}; // how long a message sent now would wait +}; struct TransportMessage { std::uint64_t peer{}; std::vector bytes; @@ -44,6 +53,8 @@ struct TransportStatus { std::uint64_t queue_us{}, skipped{}, send_failures{}, invalid_messages{}, sent_bytes{}, received_bytes{}, raw_sent_bytes{}; }; +// What each connection may send a second (bytes): a crowded server's players, mostly. +inline constexpr int connection_send_rate = 900 * 1024, min_send_rate = 128 * 1024, max_send_rate = 16 * 1024 * 1024; class SteamTransport { public: SteamTransport(); @@ -69,6 +80,19 @@ class SteamTransport { std::vector receive(); std::string name(std::uint64_t id); const TransportStatus &status() const; + // For a host's log. How Steam says a connection it closed ended, once (empty when this + // side closed it, or it is still open): who closed it, Steam's reason code and words, and + // the last ping and quality measured. And a connected player's link right now. + std::string take_closed(std::uint64_t id); + std::string link_report(std::uint64_t id); + // Bytes waiting to go out to a player, as last measured (a few times a second). + std::int64_t pending(std::uint64_t id) const; + // What each connection may send a second from now on, the open ones included. False when + // out of range, or when an open connection could not be changed (new ones still get it). + bool set_send_rate(int bytes_per_second); + int send_rate() const; + // Every connection, measured now: one Steam call each, so for a command, not every tick. + std::vector links(); private: bool bind(void *steam_api, void *sockets, void *networking_utils); diff --git a/Extension/Multiplayer/Test/bandwidth_tests.cpp b/Extension/Multiplayer/Test/bandwidth_tests.cpp index 6af0044b..ba2588a4 100644 --- a/Extension/Multiplayer/Test/bandwidth_tests.cpp +++ b/Extension/Multiplayer/Test/bandwidth_tests.cpp @@ -135,6 +135,44 @@ void rate_checks() { pose_interval(49*49, 100000) == 50000, "Near-rate hysteresis failed"); check(pose_interval(171*171, 100000) == 200000 && pose_interval(160*160, 200000) == 200000 && pose_interval(149*149, 200000) == 100000, "Far-rate hysteresis failed"); + { + // A crowd: no limit while everyone fits at the full rate, then the nearest at full, + // the next at half, the rest at low, within the budget. + std::vector few(20), crowd(49), packed(127); + for (std::size_t i = 0; i < packed.size(); ++i) { + const auto d = static_cast((packed.size() - i) * (packed.size() - i)); // farthest first + packed[i] = d; + if (i < crowd.size()) crowd[i] = d; + if (i < few.size()) few[i] = d; + } + const auto none = crowd_limits(few, 30); + // The default leaves twenty players in one place at the full rate, each to each. + std::vector twenty(crowd.begin(), crowd.begin() + 19); + check(crowd_limits(twenty, 30).half == none.half && dingosdk::valid_crowd_budget(0) && + dingosdk::valid_crowd_budget(dingosdk::crowd_pose_budget) && !dingosdk::valid_crowd_budget(50), + "The default crowd budget slowed twenty players, or a bad one was valid"); + check(crowd_interval(33333, 1e9f, none) == 33333, "A small group was limited"); + const auto count = [](std::span all, const CrowdLimits &limits, unsigned tps) { + unsigned sent{}; + for (const auto d : all) sent += 1000000U / crowd_interval(dingosdk::multiplayer_pose_interval(tps), d, limits); + return sent; + }; + for (const unsigned tps : {20U, 30U, 60U, 120U}) { + const auto limits = crowd_limits(crowd, tps, 600); + const auto sent = count(crowd, limits, tps); + check(sent <= 602 && sent > 450, "A crowd was not sent within the budget"); + check(crowd_interval(dingosdk::multiplayer_pose_interval(tps), crowd.front(), limits) == dingosdk::multiplayer_pose_interval(tps) && + crowd_interval(dingosdk::multiplayer_pose_interval(tps), crowd.back(), limits) == 200000, + "A crowd's nearest and farthest were not sent at the full and low rates"); + check(limits.half < limits.low, "A crowd had no half-rate ring"); + } + // Too many for the budget even at the low rate: all at the low rate, which is the floor. + const auto all_low = crowd_limits(packed, 30, 600); + check(count(packed, all_low, 30) == packed.size() * 5, "An over-full crowd was not sent at the low rate"); + // Distance still slows what the crowd limit would send faster. + check(crowd_interval(200000, 0, none) == 200000 && crowd_interval(100000, 0, crowd_limits(crowd, 30, 600)) == 100000, + "A crowd limit sped a far player up"); + } for (auto interval : {8333U, 16666U, 33333U, 50000U, 100000U, 200000U}) { PoseBuffer buffer; auto p = fixture(); p.pose_interval_us = interval; diff --git a/Server/README.txt b/Server/README.txt index 4c6d3b50..170323ac 100644 --- a/Server/README.txt +++ b/Server/README.txt @@ -55,6 +55,27 @@ map_rotation_minutes Minutes on each map before the server moves to the next warning; the clock waits while nobody is on, and starts over whenever the map changes (by a vote or an admin too). max_players 1-249. +send_rate The most the server sends one player, in KB/s (default 900, + 128-16384). About 1100 is what reaches a player through + Steam's relays: set higher, what is lost is resent until the + connection is full of resends and half of everything is + lost. Console: rate , also for the players already on. +crowd_budget The most position updates a second one player is sent + (default 600, 0 for no limit). Players near each other are + sent at the full rate (tps); this only matters once more are + in one place than budget / tps, about 20 at 30 TPS. Then the + nearest stay at the full rate and the farthest of the crowd + drop to 10 and 5 a second, instead of everyone's connection + filling up. About 1000 is what reaches a player through + Steam's relays; more than that was seen to lose half. + Console: crowd |off. +reserved_slots How many of those are kept for the players in "reserved" and + the admins (default 0). With 64 players and 4 reserved slots, + anyone can join until 60 are on; the last 4 only they can + take. The browser still shows 64, and everyone else is told + the remaining slots are reserved. +reserved SteamID64s of the players with a reserved slot, e.g. + ["76561198000000000"]. Admins always have one. password Empty for anyone; otherwise players type it to join. welcome A chat line sent to each player as they join. listed false hides the server; players then need the code. @@ -95,7 +116,7 @@ speed_check Catch players whose game runs faster than normal (Cheat Engine's speedhack and the like), measured from the timing of what their game sends: "warn" (default) takes them out of throwdowns and coop challenges until their speed is normal - again and tells the admins, "kick" removes them from the + again and tells them so (the log has it for the admins), "kick" removes them from the server, "off" does not check. score_check Catch players whose mods change how many points tricks score (per-trick points, the scoring multipliers, the @@ -148,6 +169,15 @@ and change voice, distances, placement and kicks from the Multiplayer menu. help A short list of every command. status Name, map, players, code. players Connected players and their SteamID64s. + reserved Who has a reserved slot and how many are kept. + reserved slots Keep n slots for them (console only), 0 for none. + reserved add|remove (console only) + net [player] How the connections are doing right now: traffic, + queues, the server's own loop timing, and the twelve + connections doing worst. With a player, that one's + ping, delivery, queue and when it was last heard from. + The log also gets a [network] line every minute and, + when a player's connection ends, how Steam says it ended. say Chat as the server (console only). msg Private message, shown to them as "[DM from ] ...". Name start (one word) or SteamID64. diff --git a/Server/Test/server_config_tests.cpp b/Server/Test/server_config_tests.cpp index 96072bed..40a3b0a3 100644 --- a/Server/Test/server_config_tests.cpp +++ b/Server/Test/server_config_tests.cpp @@ -60,6 +60,45 @@ int run() { check(token_refused(tokened), "A steam_token with other characters was accepted"); tokened.steam_token = std::string(65, 'A'); check(token_refused(tokened), "An overlong steam_token was accepted"); + { + check(has("crowd_budget") && config.crowd_budget == 600, "crowd_budget is not a new setting of 600"); + check(has("send_rate") && config.send_rate == 900, "send_rate is not a new setting of 900"); + ServerConfig rate; + rate.send_rate = 64; + check(config_error(rate).find("send_rate") != std::string::npos, "A send rate too low to play with was accepted"); + ServerConfig crowd; + crowd.crowd_budget = 50; + check(config_error(crowd).find("crowd_budget") != std::string::npos, "A crowd budget too small to play with was accepted"); + } + // Reserved slots: none unless set; the last of them only the listed players and admins take. + { + check(has("reserved_slots") && has("reserved") && config.reserved_slots == 0 && config.reserved.empty(), + "Reserved slots are not new settings that reserve nothing"); + ServerConfig slots; + slots.max_players = 4; + const std::uint64_t vip = 76561198000000001ULL, admin = 76561198000000002ULL, anyone = 76561198000000003ULL; + check(may_join(slots, anyone, 3) && !may_join(slots, anyone, 4) && !may_join(slots, vip, 4), "A server with nothing reserved did not fill up plainly"); + slots.reserved_slots = 2; + slots.reserved = {vip}; + slots.admins = {admin}; + check(may_join(slots, anyone, 1) && !may_join(slots, anyone, 2) && !may_join(slots, anyone, 3), "Reserved slots were given to anyone"); + check(may_join(slots, vip, 2) && may_join(slots, vip, 3) && may_join(slots, admin, 3), "A reserved player or an admin was kept out of a reserved slot"); + check(!may_join(slots, vip, 4) && !may_join(slots, admin, 4), "A full server let someone in"); + const auto refused = [](const ServerConfig &c) { return config_error(c).find("reserved") != std::string::npos; }; + check(!refused(slots), "Valid reserved slots were refused"); + slots.reserved_slots = 4; + check(refused(slots), "A server with every slot reserved was accepted"); + slots.reserved_slots = 2; + slots.reserved.push_back(42); + check(refused(slots), "A reserved entry that is not a player was accepted"); + auto kept = reloaded; + kept.reserved_slots = 3; + kept.reserved = {vip}; + save_config(kept); + const auto back = load_config(file); + check(back.reserved_slots == 3 && back.reserved == std::vector{vip}, "Reserved slots were not kept"); + save_config(reloaded); + } // object_limit: 100 unless set (0 is no limit), and kept on a rewrite. check(has("object_limit") && config.object_limit == 100, "object_limit is not a new setting of 100"); auto limited = reloaded; diff --git a/Server/Test/speed_check_tests.cpp b/Server/Test/speed_check_tests.cpp index 34bb5726..ad25a03e 100644 --- a/Server/Test/speed_check_tests.cpp +++ b/Server/Test/speed_check_tests.cpp @@ -56,8 +56,9 @@ int main() { check(fast.flagged && std::abs(fast.speed - 1.5) < 0.02, "A 1.5x speedhack was not caught"); const auto slight = run(1.03, 120, false); check(!slight.flagged, "A 3% difference was flagged"); - const auto subtle = run(1.1, 60, false); - check(subtle.flagged, "A 1.1x speedhack was not caught"); + // Under half as fast again is left alone; from there up it is caught. + check(!run(1.1, 60, false).flagged && !run(1.3, 60, false).flagged, "A game under 1.5x speed was flagged"); + check(run(2.0, 60, false).flagged, "A 2x speedhack was not caught"); const auto slow = run(0.5, 60, false); check(!slow.flagged, "A slowed-down client was flagged"); // Loading (no poses for a while) starts the measurement over. @@ -65,5 +66,5 @@ int main() { for (int i = 0; i < 30 * 15; ++i) gaps.sample(1000000 + i * 50000, 1000000 + i * 33333); gaps.sample(1000000 + 30 * 15 * 50000, 1000000 + 30 * 15 * 33333 + 5000000); check(!gaps.flagged() && gaps.speed() == 1.0, "A pause did not restart the measurement"); - std::cout << "Speed check: real time, lag, 1.5x and 1.1x speedhacks, small drift, slow motion and pauses passed.\n"; + std::cout << "Speed check: real time, lag, 1.5x and 2x speedhacks, 1.1x and 1.3x left alone, small drift, slow motion and pauses passed.\n"; } diff --git a/Server/server_config.cpp b/Server/server_config.cpp index 518f3cb8..b109db62 100644 --- a/Server/server_config.cpp +++ b/Server/server_config.cpp @@ -23,6 +23,12 @@ Json to_json(const ServerConfig &c) { root["map_pool"] = std::move(pool); root["map_rotation_minutes"] = c.map_rotation; root["max_players"] = c.max_players; + root["reserved_slots"] = c.reserved_slots; + root["crowd_budget"] = c.crowd_budget; + root["send_rate"] = c.send_rate; + auto reserved = Json::array(); + for (const auto id : c.reserved) reserved.push_back(std::to_string(id)); // as strings, like the admins + root["reserved"] = std::move(reserved); root["password"] = c.password; root["welcome"] = c.welcome; root["listed"] = c.listed; @@ -124,6 +130,11 @@ ServerConfig load_config(const std::filesystem::path &file, std::vector parse_scoring(std::string_view text) { if (!value) return std::nullopt; return value; } +bool may_join(const ServerConfig &config, std::uint64_t id, std::size_t on) noexcept { + if (on >= config.max_players) return false; + const auto listed = [&](const std::vector &ids) { return std::find(ids.begin(), ids.end(), id) != ids.end(); }; + return on + config.reserved_slots < config.max_players || listed(config.reserved) || listed(config.admins); +} std::string config_error(const ServerConfig &c) { using namespace multiplayer; if (!valid_server_name(c.name)) return std::string("name must be ") + server_name_rule + "."; + for (const auto id : c.reserved) + if (!individual_steam_id(id)) return "reserved must be SteamID64s (17 digits starting 7656119)."; + if (c.reserved_slots >= c.max_players) return "reserved_slots must be less than max_players, so that anyone can join at all."; + if (c.reserved.size() > 1024) return "reserved holds at most 1024 players."; + if (c.send_rate < 128 || c.send_rate > 16384) return "send_rate must be 128 to 16384 (KB/s for each player)."; + if (!valid_crowd_budget(c.crowd_budget)) + return "crowd_budget must be 0 (no limit) or " + std::to_string(min_crowd_budget) + " to " + + std::to_string(max_crowd_budget) + "."; if (c.steam_token.size() > 64 || !std::all_of(c.steam_token.begin(), c.steam_token.end(), [](unsigned char ch) { return std::isalnum(ch); })) return "steam_token must be a game server login token (letters and digits), or empty to sign in anonymously."; if (c.map.empty() || !valid_map_destination(map_destination(c.map))) diff --git a/Server/server_config.h b/Server/server_config.h index 61d54208..a46dc361 100644 --- a/Server/server_config.h +++ b/Server/server_config.h @@ -36,6 +36,14 @@ struct ServerConfig { std::vector map_pool; // maps for votes and the rotation, in order; empty: every map unsigned map_rotation = 0; // minutes per map before the next pool map (0: off) unsigned max_players = 16; // players; the server itself is not one + // Of those, how many are kept for the players in `reserved` and the admins: everyone else + // is told the server is full once only these are left. 0: none are kept. + unsigned reserved_slots = 0; + // The most poses a second one player is sent (crowd_limits); 0: no limit. + unsigned crowd_budget = crowd_pose_budget; + // What the server may send each player, in KB/s (128-16384). + unsigned send_rate = 900; + std::vector reserved; std::string password; // empty: anyone may join std::string welcome; // sent to each player as they join bool listed = true; // shown in the in-game server browser @@ -88,6 +96,9 @@ struct ServerConfig { // their names go to `added` ("votes.seconds" for a nested one). ServerConfig load_config(const std::filesystem::path &file, std::vector *added = nullptr); void save_config(const ServerConfig &config); +// Whether a player who is not yet on may join a server with `on` players on it: anyone while +// an unreserved slot is free, then only the reserved players and the admins until it is full. +bool may_join(const ServerConfig &config, std::uint64_t id, std::size_t on) noexcept; // Why `config` cannot run, or empty. std::string config_error(const ServerConfig &config); // A scoring fingerprint as the config and console write it (16 hex digits), and read back diff --git a/Server/server_host.cpp b/Server/server_host.cpp index 56c9afe1..8c843e85 100644 --- a/Server/server_host.cpp +++ b/Server/server_host.cpp @@ -42,16 +42,16 @@ std::uint64_t nonce() { return value; } constexpr std::string_view help_text = - "status | players | say | msg | msg-party | msg-admins | kick | ban [name] | unban | bans\n" + "status | net [player] | players | say | msg | msg-party | msg-admins | kick | ban [name] | unban | bans\n" "map | maps | name | password | welcome | listed on|off\n" - "tps 20|30|60|120 | voice on|off | voice-range <50-1000> | distances \n" + "tps 20|30|60|120 | voice on|off | voice-range <50-1000> | distances | crowd |off | rate \n" "placement everyone|admins|nobody | objects |off | clear-objects | noclip on|off | nobail on|off | boosts on|off | tuning on|off\n" "tpall [player] | tphere | votes [map|kick|tod on|off|] | vote-cancel\n" "map-pool [add|remove |clear] | rotation [|off]\n" "park | layer-sync on|off | layer default|on|off | tod \n" "activity-log on|off | announce-throwdowns on|off | parties [on|off] | party-size <2-8> | speed-check off|warn|kick\n" "score-check [off|warn|kick] | score-allow [|remove ]\n" - "admin add|remove | admins | update | quit"; + "reserved [slots | add|remove ] | admin add|remove | admins | update | quit"; } // namespace Host::Host(ServerConfig &config, SteamTransport &transport, Log log) @@ -137,6 +137,7 @@ unsigned Host::players() const { } bool Host::start(std::string &error) { + transport_.set_send_rate(static_cast(config_.send_rate * 1024)); if (!transport_.host(capacity())) { error = transport_.status().detail; return false; @@ -173,6 +174,13 @@ void Host::apply_layers() { } // ---- Sending --------------------------------------------------------------------------------- +namespace { +constexpr std::array traffic_names{"poses", "sound", "voice", "outfits", "objects", "other"}; +constexpr std::size_t traffic_kind(PacketKind kind) noexcept { + return kind == PacketKind::pose ? 0 : kind == PacketKind::audio ? 1 : kind == PacketKind::voice ? 2 + : kind == PacketKind::cosmetics ? 3 : kind == PacketKind::objects ? 4 : 5; +} +} // namespace bool Host::send_packet(Guest &g, const Packet &p, bool reliable, bool fresh, std::span raw, std::span wire) { auto update = raw.empty() ? g.sender.prepare(p) : g.sender.prepare(p, raw, wire); @@ -182,6 +190,10 @@ bool Host::send_packet(Guest &g, const Packet &p, bool reliable, bool fresh, std // A stream and its reliable delta references must always use the same lane. if (!transport_.send(g.member.id, update.bytes, reliable || update.establishes_baseline(), fresh, traffic_lane(p.kind))) return false; + for (auto *counted : {&g.traffic, &traffic_}) { + counted->total.out[traffic_kind(p.kind)] += update.bytes.size(); + counted->total.snapshots += update.establishes_baseline(); + } g.sender.sent(p, std::move(update)); return true; } @@ -193,8 +205,13 @@ void Host::send_required(Guest &g, const std::vector &bytes) { } void Host::broadcast(const Packet &packet, bool reliable, bool fresh, std::uint64_t except) { const auto *source = find(packet.source); - struct Encoded { Packet packet; std::vector raw, wire; }; + // Recipients holding the same delta reference share one patch and compression + // (DeltaCache): every update is built first, then all are sent and recorded. + struct Encoded { Packet packet; std::vector raw, wire; DeltaCache deltas; }; std::array encoded; + struct Outgoing { Guest *guest; std::uint64_t id; WireUpdate update; PoseDelivery *delivery; std::uint32_t interval; unsigned variant; }; + std::vector outgoing; + outgoing.reserve(guests_.size()); const bool gameplay = packet.kind == PacketKind::pose || packet.kind == PacketKind::audio || packet.kind == PacketKind::voice || packet.kind == PacketKind::cosmetics; for (auto &[id, guest] : guests_) { @@ -212,6 +229,23 @@ void Host::broadcast(const Packet &packet, bool reliable, bool fresh, std::uint6 continue; } } + // Not before they have been shown this player (introduce). + if (!p.unmet.empty() && (packet.kind == PacketKind::pose || packet.kind == PacketKind::audio || packet.kind == PacketKind::cosmetics) && + p.unmet.contains(packet.source)) + continue; + // A skater's sound goes to those who would hear it: within the full pose rate's reach, + // and in a crowd only from the nearest (the same players sent at the full rate). + // A game stops a sound it hears nothing more of after a second. + if (packet.kind == PacketKind::audio && source && source->latest_root && source->pose_arrival && + now_ - source->pose_arrival <= 1000000 && p.latest_root && p.pose_arrival && now_ - p.pose_arrival <= 1000000) { + float distance{}; + for (unsigned i = 0; i < 3; ++i) { + const auto d = p.latest_root->position[i] - source->latest_root->position[i]; + distance += d * d; + } + const auto reach = static_cast(config_.distances.half_rate_start); + if (distance > p.crowd.half || (config_.distances.valid() && distance > reach * reach)) continue; + } // Frequent gameplay streams skip the server once the receiver confirms a direct route. if (source && (packet.kind == PacketKind::pose || packet.kind == PacketKind::audio) && !needs_relay(p.direct_routes, p.route_reported, source->member, now_)) @@ -226,14 +260,17 @@ void Host::broadcast(const Packet &packet, bool reliable, bool fresh, std::uint6 [](const auto &a, const auto &b) { return a.last_sent < b.last_sent; }); delivery = &*entry; if (delivery->source != packet.source || delivery->epoch != packet.epoch) - *delivery = {packet.source, packet.epoch, 0, interval}; + *delivery = {packet.source, packet.epoch, 0, interval, 0, 0}; if (p.latest_root && p.pose_arrival && now_ >= p.pose_arrival && now_ - p.pose_arrival <= 1000000) { float distance{}; for (unsigned i = 0; i < 3; ++i) { const auto d = p.latest_root->position[i] - packet.pose.root.position[i]; distance += d * d; } - interval = pose_interval(distance, delivery->interval_us, config_.distances, config_.tps); + delivery->by_distance = pose_interval(distance, delivery->by_distance, config_.distances, config_.tps); + interval = crowd_interval(delivery->by_distance, distance, p.crowd); + } else { + delivery->by_distance = 0; } if (delivery->interval_us != interval) delivery->next_source_time = 0; delivery->interval_us = interval; @@ -247,12 +284,83 @@ void Host::broadcast(const Packet &packet, bool reliable, bool fresh, std::uint6 data.raw = encode(data.packet, true); data.wire = encode_wire_bytes(data.raw); } - const bool sent = send_packet(p, data.packet, reliable, fresh, data.raw, data.wire); - if (sent && delivery) { - delivery->last_sent = now_; - advance_pose_deadline(delivery->next_source_time, packet.time_us, interval); + outgoing.push_back({&p, id, p.sender.prepare(data.packet, data.raw, data.wire, data.deltas), delivery, interval, variant}); + } + for (auto &out : outgoing) { + // A packet that cannot be built for anyone is its source's fault, never this + // recipient's: nothing is sent, and the recipient is not treated as unreachable. + bool sent = true; + if (!out.update.bytes.empty()) { + const auto &sending = encoded[out.variant].packet; + const bool whole = out.update.establishes_baseline(); + sent = transport_.send(out.id, out.update.bytes, reliable || whole, fresh, traffic_lane(sending.kind)); + if (sent) { + for (auto *counted : {&out.guest->traffic, &traffic_}) { + counted->total.out[traffic_kind(sending.kind)] += out.update.bytes.size(); + counted->total.snapshots += whole; + } + out.guest->sender.sent(sending, std::move(out.update)); + } } - if (!sent && reliable && !fresh) transport_.disconnect(id, "Cannot deliver required session data. Join again."); + if (sent && out.delivery) { + out.delivery->last_sent = now_; + // The next slot on the source's own clock, not this long after this send: everyone + // sent a source at a slower rate is then sent the same packets of it, and shares + // their encoding and patch. + out.delivery->next_source_time = (packet.time_us / out.interval + 1) * out.interval; + } + // Refused by a full queue: a connection that is only behind keeps its player (a chat + // line or an outfit change is lost to them); one that takes nothing for a while does not. + if (reliable && !fresh) { + auto &since = out.guest->undelivered_since; + if (sent) since = 0; + else if (!since) since = now_; + else if (now_ - since > 15000000) transport_.disconnect(out.id, "Cannot deliver required session data. Join again."); + } + } +} +// A player who has just arrived in this world is shown the others a few at a time (introduce). +void Host::meet_later(Guest &guest) { + guest.unmet.clear(); + for (const auto &[id, other] : guests_) + if (other->handshaken && id != guest.member.id) guest.unmet.insert(id); + guest.next_introduction = now_; +} +// Shows each arriving player one more of the others: that player's outfit, after which their +// poses and sound follow. The nearest first, about seven a second, and none while what +// they have already been sent is still on its way. A busy server's eighty outfits and +// streams in one burst is what a joining game, building every skater at once, fell over on. +void Host::introduce() { + for (auto &[id, guest] : guests_) { + auto &g = *guest; + if (g.unmet.empty() || !g.handshaken || !g.world_ready || now_ < g.next_introduction) continue; + if (transport_.pending(id) > 96 * 1024) continue; + const Guest *next{}; + float nearest{}; + for (auto it = g.unmet.begin(); it != g.unmet.end();) { + const auto *other = find(*it); + if (!other || !other->handshaken) { + it = g.unmet.erase(it); + continue; + } + ++it; + float distance = std::numeric_limits::max(); + if (g.latest_root && other->latest_root) { + distance = 0; + for (unsigned i = 0; i < 3; ++i) { + const auto d = g.latest_root->position[i] - other->latest_root->position[i]; + distance += d * d; + } + } + if (!next || distance < nearest) { + next = other; + nearest = distance; + } + } + if (!next) continue; + g.unmet.erase(next->member.id); + g.next_introduction = now_ + 150000; + send_required(g, next->cosmetic_packet); } } void Host::send_roster() { @@ -355,6 +463,8 @@ void Host::change_map(std::string_view map) { for (auto &[id, guest] : guests_) { auto &g = *guest; auto old = std::exchange(g, Guest{}); + g.sender.set_refresh(8000000, true); // as for a new player (tick): a new world's streams start over + g.traffic = old.traffic; g.member = std::move(old.member); g.handshaken = old.handshaken; g.map_authorized = old.map_authorized; @@ -387,21 +497,22 @@ void Host::change_map(std::string_view map) { send_world_state(); roster_dirty_ = true; } -void Host::drop(std::uint64_t id, const std::string &reason) { +void Host::drop(std::uint64_t id, const std::string &reason, const std::string &detail) { transport_.disconnect(id, reason.c_str()); const auto found = guests_.find(id); if (found == guests_.end()) return; const auto name = guest_name(*found->second); if (found->second->handshaken) { roster_dirty_ = true; - log_(name + " left (" + reason + ")"); + log_(name + " left (" + reason + ")" + (detail.empty() ? std::string{} : " [" + detail + "]")); activity_.left(id); } else { // Never admitted: it held a player slot meanwhile, so it shows in the log, and an ID // that keeps failing waits longer each time before its connection is taken again. const auto failures = join_backoff_.failed(id, now_); log_("[join] " + std::to_string(id) + " did not finish joining (" + reason + ")" + - (failures > 1 ? ", attempt " + std::to_string(failures) : std::string{})); + (failures > 1 ? ", attempt " + std::to_string(failures) : std::string{}) + + (detail.empty() ? std::string{} : " [" + detail + "]")); } guests_.erase(found); party_left(id, name); @@ -448,6 +559,8 @@ void Host::receive(std::uint64_t peer, std::span bytes, std: if (!decoded || decoded->session != secret_ || decoded->source != peer) return drop(peer, "Join code, sender identity, or multiplayer protocol did not match."); const auto &p = *decoded; + link->traffic.total.in[traffic_kind(p.kind)] += bytes.size(); + traffic_.total.in[traffic_kind(p.kind)] += bytes.size(); switch (p.kind) { case PacketKind::world_state: return drop(peer, "Only the server may change the room's map."); // Sent once the player's game knows, and again if a mod enabled later changes it; not tied to @@ -474,9 +587,7 @@ void Host::receive(std::uint64_t peer, std::span bytes, std: log_("[map] " + guest_name(*link) + " finished loading (" + std::to_string((now_ - since) / 1000000) + " s)"); link->world_ready = p.world_ready; if (p.world_ready) link->travel_since = link->loading_since = 0; - if (arrived) - for (auto &[id, other] : guests_) - if (other->handshaken && id != peer) send_required(*link, other->cosmetic_packet); + if (arrived) meet_later(*link); return; } case PacketKind::map_request: { @@ -548,8 +659,7 @@ void Host::receive(std::uint64_t peer, std::span bytes, std: send_required(*link, encode_wire(packet(PacketKind::welcome, now_))); if (joined) { send_roster(); - for (auto &[id, other] : guests_) - if (other->handshaken && id != peer) send_required(*link, other->cosmetic_packet); + meet_later(*link); if (!config_.welcome.empty()) send_chat(config_.welcome, link); log_(guest_name(*link) + " joined (" + std::to_string(peer) + (is_admin(peer) ? ", admin" : "") + "), " + std::to_string(players()) + "/" + std::to_string(config_.max_players) + " players" + @@ -712,6 +822,9 @@ void Host::sync_objects() { for (auto &[id, guest] : guests_) { auto &peer = *guest; if (!peer.handshaken || !peer.world_ready || sources.empty()) continue; + // Objects wait for a connection that is behind: they must arrive, so they would only + // queue up behind the poses and push those out. + if (transport_.pending(id) > 128 * 1024) continue; auto &delivery = peer.object_delivery; const auto find_source = [&](std::uint64_t source, std::uint64_t epoch) { return std::find_if(sources.begin(), sources.end(), [&](const auto &s) { return s.id == source && s.epoch == epoch; }); @@ -740,9 +853,14 @@ void Host::sync_objects() { update.epoch = delivery.epoch; update.objects = delivery.chunks[delivery.next]; if (!send_packet(peer, update, true, false)) { - transport_.disconnect(id, "Cannot deliver shared object state. Join again."); + // Refused for now (a full queue): the same part is tried again. Only a + // connection that takes none for a long while is given up on. + if (!delivery.failing_since) delivery.failing_since = now_; + if (now_ - delivery.failing_since > 30000000) + transport_.disconnect(id, "Cannot deliver shared object state. Join again."); break; } + delivery.failing_since = 0; if (++delivery.next == delivery.chunks.size()) { delivery.sent[delivery.source] = {delivery.epoch, update.objects.revision}; delivery.chunks.clear(); @@ -751,10 +869,165 @@ void Host::sync_objects() { } } +// `net`: how the connections and the server's own loop are doing right now. With a player's +// name or SteamID64, that player alone. In-game admins get the first line: a reply has to fit +// one chat-sized message. +std::string Host::network_report(std::string_view player, bool console) { + const auto links = transport_.links(); + const auto &net = transport_.status(); + const auto kb = [](float bytes) { return std::to_string(static_cast(bytes / 1024.f + .5f)); }; + const auto percent = [](float quality) { return quality < 0 ? std::string("?") : std::to_string(static_cast(quality * 100.f + .5f)) + "%"; }; + const auto row = [&](const multiplayer::TransportLink &link) { + const auto *guest = find(link.id); + std::string name = guest ? guest_name(*guest) : std::to_string(link.id); + if (name.size() > 20) name.resize(20); + name.resize(21, ' '); + if (!link.measured) return " " + name + (link.connected ? "not measured yet" : "connecting"); + const auto silent = guest && guest->last_packet && now_ > guest->last_packet ? (now_ - guest->last_packet) / 100000 : 0; // tenths of a second + return " " + name + "ping " + std::to_string(link.ping_ms) + " ms, quality " + percent(link.quality_local) + " here / " + + percent(link.quality_remote) + " there, queued " + std::to_string(link.pending_bytes) + " B (" + + std::to_string(link.queue_us / 1000) + " ms), out " + kb(link.out_bps) + " of " + kb(static_cast(link.send_rate)) + + " KB/s, in " + kb(link.in_bps) + " KB/s, last heard " + std::to_string(silent / 10) + "." + std::to_string(silent % 10) + " s ago" + + (guest && !guest->handshaken ? ", joining" : guest && !guest->world_ready ? ", loading" : ""); + }; + // By what it carries, in KB/s: the last whole half minute, or the one so far. + const bool whole = traffic_window_us_ != 0; + const double seconds = whole ? static_cast(traffic_window_us_) / 1e6 + : std::max(1.0, static_cast(now_ - traffic_mark_) / 1e6); + const auto rate = [&](const Counted &c, bool out, std::size_t kind) { + const auto bytes = whole ? (out ? c.last.out[kind] : c.last.in[kind]) + : out ? c.total.out[kind] - c.mark.out[kind] : c.total.in[kind] - c.mark.in[kind]; + return static_cast(bytes) / 1024.0 / seconds; + }; + const auto number = [](double value) { + char text[32]; + std::snprintf(text, sizeof(text), value < 10 ? "%.1f" : "%.0f", value); + return std::string(text); + }; + const auto by_kind = [&](const Counted &c) { + std::string text = "KB/s out / in:"; + for (std::size_t kind = 0; kind < traffic_names.size(); ++kind) + text += std::string(kind ? ", " : " ") + traffic_names[kind] + " " + number(rate(c, true, kind)) + " / " + number(rate(c, false, kind)); + const auto snapshots = whole ? c.last.snapshots : c.total.snapshots - c.mark.snapshots; + return text + "; " + number(static_cast(snapshots) / seconds) + " whole states a second"; + }; + if (!player.empty()) { + const auto wanted = lower(player); + for (const auto &link : links) { + const auto *guest = find(link.id); + if (std::to_string(link.id) == player || (guest && lower(guest_name(*guest)).starts_with(wanted))) + return row(link).substr(2) + (guest ? "\n" + by_kind(guest->traffic) : std::string{}); + } + return "No connected player matches \"" + std::string(player) + "\"."; + } + float out{}, in{}; + std::uint64_t queued{}, longest_queue{}; + std::size_t waiting{}, poor{}; + int worst_ping{}; + for (const auto &link : links) { + if (!link.measured) continue; + out += link.out_bps; + in += link.in_bps; + queued += static_cast(std::max(0, link.pending_bytes)); + longest_queue = std::max(longest_queue, link.queue_us); + waiting += link.queue_us > 50000; + poor += (link.quality_local >= 0 && link.quality_local < .9f) || (link.quality_remote >= 0 && link.quality_remote < .9f); + worst_ping = std::max(worst_ping, link.ping_ms); + } + std::string text = std::to_string(players()) + " players (" + std::to_string(links.size()) + " connections), " + std::to_string(config_.tps) + + " TPS | " + kb(out) + " KB/s out, " + kb(in) + " KB/s in | worst ping " + std::to_string(worst_ping) + " ms | " + + std::to_string(queued / 1024) + " KB queued, longest wait " + std::to_string(longest_queue / 1000) + " ms, " + + std::to_string(waiting) + " waiting over 50 ms | " + std::to_string(poor) + " under 90% quality"; + if (!console) return text; + text += "\nloop: " + loop_report(); + text += "\n" + by_kind(traffic_); + { + // Who uploads most, and what most of it is: one player's outfit or objects can cost + // every other player's connection. + struct Upload { const Guest *guest; double total; std::size_t most; }; + std::vector uploads; + for (const auto &[id, guest] : guests_) { + Upload upload{guest.get(), 0, 0}; + for (std::size_t kind = 0; kind < traffic_names.size(); ++kind) { + upload.total += rate(guest->traffic, false, kind); + if (rate(guest->traffic, false, kind) > rate(guest->traffic, false, upload.most)) upload.most = kind; + } + uploads.push_back(upload); + } + std::stable_sort(uploads.begin(), uploads.end(), [](const auto &a, const auto &b) { return a.total > b.total; }); + if (!uploads.empty()) text += "\nmost uploaded:"; + for (std::size_t i = 0; i < std::min(uploads.size(), 6); ++i) + text += std::string(i ? ", " : " ") + guest_name(*uploads[i].guest) + " " + number(uploads[i].total) + " KB/s (" + + traffic_names[uploads[i].most] + " " + number(rate(uploads[i].guest->traffic, false, uploads[i].most)) + ")"; + } + text += "\nsends: " + std::to_string(net.send_failures) + " failed, " + std::to_string(net.skipped) + " skipped, " + + std::to_string(net.dropped) + " dropped (since the server started)"; + // The connections doing worst first: a long queue, poor delivery, silence, then ping. + auto order = links; + const auto badness = [&](const multiplayer::TransportLink &link) { + const auto *guest = find(link.id); + const double silent = guest && guest->handshaken && now_ > guest->last_packet ? static_cast(now_ - guest->last_packet) / 1000.0 : 0; + const double lost = 1.0 - std::min(link.quality_local < 0 ? 1.f : link.quality_local, link.quality_remote < 0 ? 1.f : link.quality_remote); + return static_cast(link.queue_us) / 1000.0 + lost * 2000.0 + silent + link.ping_ms; + }; + std::stable_sort(order.begin(), order.end(), [&](const auto &a, const auto &b) { return badness(a) > badness(b); }); + const std::size_t shown = std::min(order.size(), 12); + if (shown) text += "\nworst " + std::to_string(shown) + " of " + std::to_string(order.size()) + " (net for one):"; + for (std::size_t i = 0; i < shown; ++i) text += "\n" + row(order[i]); + return text; +} + // ---- Tick ------------------------------------------------------------------------------------ +namespace { +std::uint64_t clock_us() { + return static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count()); +} +} // namespace +std::string Host::loop_report() const { + // The last whole minute when there is one, or the minute so far. + const auto &l = last_loop_.passes ? last_loop_ : loop_; + const auto seconds = std::max(1, last_loop_.passes ? 60 : (clock_us() - l.since) / 1000000); + const auto ms = [](std::uint64_t us) { + char text[32]; + std::snprintf(text, sizeof(text), "%.1f", static_cast(us) / 1000.0); + return std::string(text); + }; + return std::to_string(l.passes / seconds) + " passes/s, " + + std::to_string(l.passes ? l.busy_us * 100 / (seconds * 1000000) : 0) + "% busy, longest pass " + ms(l.longest_pass_us) + + " ms, longest gap " + ms(l.longest_gap_us) + " ms (worst since start: " + ms(worst_loop_.longest_pass_us) + " ms pass, " + + ms(worst_loop_.longest_gap_us) + " ms gap)"; +} void Host::tick(std::uint64_t now) { if (!running_) return; now_ = now; + // Timed on a clock of its own, from the start of one pass to the start of the next. + struct Timed { + Host &host; + std::uint64_t started = clock_us(); + explicit Timed(Host &h) : host(h) { + auto &l = host.loop_; + if (!l.since) l.since = started; + if (host.pass_started_) { + const auto gap = started - host.pass_started_; + l.longest_gap_us = std::max(l.longest_gap_us, gap); + host.worst_loop_.longest_gap_us = std::max(host.worst_loop_.longest_gap_us, gap); + } + host.pass_started_ = started; + } + ~Timed() { + auto &l = host.loop_; + const auto took = clock_us() - started; + ++l.passes; + l.busy_us += took; + l.longest_pass_us = std::max(l.longest_pass_us, took); + host.worst_loop_.longest_pass_us = std::max(host.worst_loop_.longest_pass_us, took); + if (started - l.since >= 60000000) { + host.last_loop_ = l; + l = {}; + } + } + } timed{*this}; transport_.poll(); const auto links = transport_.status().peers; const auto linked = [&](std::uint64_t id) { @@ -762,7 +1035,11 @@ void Host::tick(std::uint64_t now) { }; for (auto it = guests_.begin(); it != guests_.end();) { const auto id = (it++)->first; - if (!linked(id)) drop(id, "Disconnected."); + if (linked(id)) continue; + // Steam closed it: the log says how (the player's game closing it, or the connection + // failing at one end or the other), which "Disconnected." alone does not. + const auto how = transport_.take_closed(id); + drop(id, "Disconnected.", how); } for (const auto &link : links) { if (kicked_.contains(link.id)) { @@ -784,11 +1061,15 @@ void Host::tick(std::uint64_t now) { continue; } if (!guest) { - if (!individual_steam_id(link.id) || guests_.size() >= config_.max_players) { - transport_.disconnect(link.id, "The server is full."); + if (!individual_steam_id(link.id) || !may_join(config_, link.id, guests_.size())) { + // Full for them: for everyone, or with only the reserved slots left. + transport_.disconnect(link.id, guests_.size() >= config_.max_players || !individual_steam_id(link.id) + ? "The server is full." + : "The server is full: the slots that are left are reserved."); continue; } auto created = std::make_unique(); + created->sender.set_refresh(8000000, true); created->member.id = link.id; created->last_packet = now_; guest = created.get(); @@ -798,6 +1079,7 @@ void Host::tick(std::uint64_t now) { } for (const auto &message : transport_.receive()) receive(message.peer, message.bytes, message.arrived); receive_cosmetics(); + introduce(); for (auto it = guests_.begin(); it != guests_.end();) { auto &g = *(it++)->second; // Authorized arrivals get time for loading; this deadline is absolute, so @@ -807,11 +1089,63 @@ void Host::tick(std::uint64_t now) { if (now_ - g.travel_since > 180000000) drop(g.member.id, "Could not finish loading the new map."); continue; } + // The connection is still open but nothing came over it: what Steam measures of it says + // whether the link went bad or the player's game just stopped sending. if ((!g.handshaken && g.connected_at && now_ - g.connected_at > handshake_timeout) || (g.handshaken && now_ - g.last_packet > 10000000)) - drop(g.member.id, "Timed out waiting for gameplay data."); + drop(g.member.id, "Timed out waiting for gameplay data.", + "nothing for " + std::to_string((now_ - g.last_packet) / 1000000) + " s; " + transport_.link_report(g.member.id)); + } + // Every half minute, what was sent and received in it (net). + if (now_ >= traffic_mark_ + 30000000) { + const auto roll = [](Counted &c) { + for (std::size_t i = 0; i < c.total.out.size(); ++i) { + c.last.out[i] = c.total.out[i] - c.mark.out[i]; + c.last.in[i] = c.total.in[i] - c.mark.in[i]; + } + c.last.snapshots = c.total.snapshots - c.mark.snapshots; + c.mark = c.total; + }; + roll(traffic_); + for (auto &[id, guest] : guests_) roll(guest->traffic); + traffic_window_us_ = traffic_mark_ ? now_ - traffic_mark_ : 0; + traffic_mark_ = now_; + } + // In a crowd, how far the full and half pose rates reach for each player (crowd_limits). + if (now_ >= next_crowd_) { + next_crowd_ = now_ + 500000; + std::vector squared; + for (auto &[id, guest] : guests_) { + auto &g = *guest; + g.crowd = {}; + if (!g.latest_root || !config_.crowd_budget || guests_.size() * config_.tps <= config_.crowd_budget) continue; + squared.clear(); + for (const auto &[other_id, other] : guests_) { + if (other_id == id || !other->handshaken || !other->world_ready) continue; + float distance{}; + if (other->latest_root) + for (unsigned i = 0; i < 3; ++i) { + const auto d = g.latest_root->position[i] - other->latest_root->position[i]; + distance += d * d; + } + squared.push_back(distance); // not placed yet: counted as beside them + } + g.crowd = crowd_limits(squared, config_.tps, config_.crowd_budget); + } } tick_parties(); + // Once a minute while anyone is on: how the connections are doing as a whole. A queue that + // grows with the player count, or sends that fail, is the server not getting data out. + if (config_.activity_log && now_ >= next_network_log_ && !guests_.empty()) { + next_network_log_ = now_ + 60000000; + const auto &net = transport_.status(); + if (net.telemetry) + log_("[network] " + std::to_string(guests_.size()) + " players, " + std::to_string(static_cast(net.outgoing_bps / 1024.f + .5f)) + + " KB/s out, " + std::to_string(static_cast(net.incoming_bps / 1024.f + .5f)) + " KB/s in, worst ping " + + std::to_string(net.ping_ms) + " ms, " + std::to_string(net.pending_bytes / 1024) + " KB queued (longest " + + std::to_string(net.queue_us / 1000) + " ms), " + std::to_string(net.send_failures) + " failed sends, " + + std::to_string(net.skipped) + " skipped, " + std::to_string(net.dropped) + " dropped | loop " + loop_report()); + } if (roster_dirty_ || now_ - last_roster_ > 2000000) send_roster(); for (auto &[id, guest] : guests_) if (guest->handshaken) { @@ -876,6 +1210,7 @@ std::string Host::command(std::string_view line, std::uint64_t admin) { std::to_string(config_.max_players) + " players | " + std::to_string(config_.tps) + " TPS | voice " + (voice_policy_.allowed ? "on" : "off") + " (" + std::to_string(static_cast(config_.voice_range)) + " m) | password " + (password_ ? "on" : "off") + " | code " + invite(); + if (name == "net") return network_report(argument, console); if (name == "players") { std::string text = std::to_string(players()) + " players"; for (const auto &[id, guest] : guests_) @@ -1123,6 +1458,29 @@ std::string Host::command(std::string_view line, std::uint64_t admin) { return changed(remove ? "Scoring " + scoring_text(*fingerprint) + " is no longer accepted." : "Scoring " + scoring_text(*fingerprint) + " is accepted like the game's own."); } + if (name == "rate") { + // rate : what the server may send each player, from now on and to those on. + const auto value = number(argument); + if (!value || *value < 128 || *value > 16384) + return "rate <128-16384> (KB/s for each player, now " + std::to_string(config_.send_rate) + ")"; + config_.send_rate = static_cast(*value); + const bool applied = transport_.set_send_rate(static_cast(config_.send_rate * 1024)); + return changed("Each player is sent at most " + std::to_string(config_.send_rate) + " KB/s" + + (applied ? "." : ": players who join from now on. Steam did not change the connections already open.")); + } + if (name == "crowd") { + // crowd |off: the most one player is sent (crowd_limits). + auto value = number(argument); + if (argument == "off") value = 0; + if (!value || *value > max_crowd_budget || !valid_crowd_budget(static_cast(*value))) + return "crowd <" + std::to_string(min_crowd_budget) + "-" + std::to_string(max_crowd_budget) + ">|off (now " + + (config_.crowd_budget ? std::to_string(config_.crowd_budget) : std::string("off")) + ")"; + config_.crowd_budget = static_cast(*value); + next_crowd_ = 0; + return changed(*value ? "Each player is sent at most " + std::to_string(*value) + " poses a second: about " + + std::to_string(*value / config_.tps) + " players near them at the full rate." + : std::string("No crowd limit: every player near is sent at the full rate.")); + } if (name == "party-size") { const auto value = number(argument); if (!value || *value < 2 || *value > 8) return "party-size <2-8> (now " + std::to_string(config_.party_size) + ")"; @@ -1387,6 +1745,39 @@ std::string Host::command(std::string_view line, std::uint64_t admin) { return changed(found->label + " set to " + std::string(mode) + (config_.world_layer_sync ? "." : ". Turn on layer-sync to apply it to everyone.")); } + if (name == "reserved") { + // reserved | reserved slots | reserved add|remove + if (!console) return "Only the server console manages reserved slots."; + const auto [sub, who] = split(argument); + if (sub.empty()) { + std::string text = std::to_string(config_.reserved_slots) + " of " + std::to_string(config_.max_players) + + " slots are reserved for " + std::to_string(config_.reserved.size()) + " players and the admins"; + for (const auto id : config_.reserved) { + const auto *guest = find(id); + text += "\n " + std::to_string(id) + (guest ? " " + guest_name(*guest) : std::string{}); + } + return text; + } + if (sub == "slots") { + const auto slots = number(who); + if (!slots || *slots >= config_.max_players) return "reserved slots <0-" + std::to_string(config_.max_players - 1) + ">"; + config_.reserved_slots = static_cast(*slots); + return changed(*slots ? std::to_string(*slots) + " slots are kept for reserved players and admins." + : std::string("No slots are reserved.")); + } + auto *guest = target(who); + const auto id = guest ? guest->member.id : number(who).value_or(0); + if (!individual_steam_id(id) || (sub != "add" && sub != "remove")) return "reserved | reserved slots | reserved add|remove "; + const bool listed = std::find(config_.reserved.begin(), config_.reserved.end(), id) != config_.reserved.end(); + if (sub == "add") { + if (!listed && config_.reserved.size() >= 1024) return "The reserved list is full."; + if (!listed) config_.reserved.push_back(id); + return changed(std::to_string(id) + " has a reserved slot." + + (config_.reserved_slots ? std::string{} : " No slots are reserved yet: reserved slots .")); + } + std::erase(config_.reserved, id); + return changed(std::to_string(id) + " no longer has a reserved slot."); + } if (name == "admins" || name == "admin") { if (!console) return "Only the server console manages admins."; const auto [sub, who] = split(argument); @@ -1461,9 +1852,7 @@ bool Host::check_speed(Guest &guest, std::uint64_t sent) { guest.speed_normal_since = 0; roster_dirty_ = true; send_chat("The server measured your game running faster than normal: throwdowns and challenges are off for you until it's back to normal speed.", &guest); - for (auto &[id, other] : guests_) - if (other->handshaken && is_admin(id) && other.get() != &guest) - send_chat(std::string(text) + " They were taken out of throwdowns and challenges.", other.get()); + // Only the player is told in the game; for the admins it is in the log. return true; } if (!guest.speeding) return true; diff --git a/Server/server_host.h b/Server/server_host.h index 1a60b1b2..a354c9be 100644 --- a/Server/server_host.h +++ b/Server/server_host.h @@ -52,8 +52,18 @@ class Host { return ++messages <= burst; } }; + // What is sent and received, in bytes, by what it carries (traffic_kind): poses, sound, + // voice, outfits, objects, the rest. `last` is the last whole half minute. + struct Traffic { + std::array out{}, in{}; + std::uint64_t snapshots{}; // whole states sent reliably; the rest of a stream is differences + }; + struct Counted { + Traffic total, mark, last; + }; struct Guest { Member member; + Counted traffic; std::uint64_t password_challenge{}; bool handshaken{}, map_authorized{}, world_ready = true; std::uint64_t last_map_offer{}, travel_since{}, connected_at{}, last_packet{}; @@ -71,6 +81,7 @@ class Host { std::optional latest_root; std::uint64_t pose_arrival{}; std::array pose_delivery; + CrowdLimits crowd; // how far the full and half rates reach for them in a crowd std::vector direct_routes; std::uint64_t route_reported{}; std::uint32_t route_sequence{}; @@ -91,6 +102,12 @@ class Host { bool scoring_flagged{}; // out of linked activities (config score_check) ChatBudget scoring_budget; std::uint64_t bans_sent{}; // the ban list revision this admin has + std::uint64_t undelivered_since{}; // Steam has refused what they must be sent since + // The players they have not been shown yet (introduce): on joining, and after a map + // change, a player is sent the others a few a second, nearest first, instead of every + // outfit and every stream at once. Nothing of a player in here is sent to them. + std::set unmet; + std::uint64_t next_introduction{}; bool maps_sent{}; // this player has the server's map list (send_maps) // The owner's own upload, and what the server shares of it. ObjectState objects, shared; @@ -101,6 +118,7 @@ class Host { std::vector chunks; std::uint64_t source{}, epoch{}; std::size_t next{}, cursor{}; + std::uint64_t failing_since{}; // Steam has refused their object updates since } object_delivery; }; @@ -137,6 +155,21 @@ class Host { bool roster_dirty_ = true, running_{}; std::uint64_t bans_revision_ = 1; // bumped whenever the ban list or the admins change std::uint64_t now_{}, last_roster_{}, last_world_state_{}, next_object_update_{}, travel_started_{}; + std::uint64_t next_crowd_{}; // when the crowd limits are worked out again + std::uint64_t next_network_log_{}; // the log's once-a-minute line about the connections + // How the server's own loop is keeping up, for `net` and that line: passes of tick() in a + // minute, the time spent in them, the longest one and the longest wait between two. A long + // pass or gap is the server itself stalling, which every player feels at once. + struct Loop { + std::uint64_t since{}, passes{}, busy_us{}, longest_pass_us{}, longest_gap_us{}; + } loop_, last_loop_, worst_loop_; + std::uint64_t pass_started_{}; + Counted traffic_; + std::uint64_t traffic_mark_{}, traffic_window_us_{}; // when `mark` was taken, and how long `last` covers + void meet_later(Guest &guest); + void introduce(); + std::string loop_report() const; + std::string network_report(std::string_view player, bool console); Guest *find(std::uint64_t id); Packet packet(PacketKind kind, std::uint64_t now); @@ -147,7 +180,8 @@ class Host { bool is_banned(std::uint64_t id) const; void save(); - void drop(std::uint64_t id, const std::string &reason); + // `detail` is for the log alone: the player is told `reason`. + void drop(std::uint64_t id, const std::string &reason, const std::string &detail = {}); bool send_packet(Guest &, const Packet &, bool reliable, bool fresh, std::span raw = {}, std::span wire = {}); void send_required(Guest &, const std::vector &bytes); diff --git a/Server/speed_check.h b/Server/speed_check.h index 18fbace9..9bad51c6 100644 --- a/Server/speed_check.h +++ b/Server/speed_check.h @@ -14,7 +14,9 @@ class SpeedCheck { static constexpr std::uint64_t block_us = 2000000; // one minimum per block static constexpr std::uint64_t window_us = 20000000; // speed measured across this static constexpr std::uint64_t gap_us = 2500000; // longer without poses (loading): start over - static constexpr double limit = 1.06; // a real clock never runs this far ahead + // Half as fast again and more is flagged. A little under 1.5, since a game at exactly 1.5x + // measures a hair either side of it; nothing but a speed hack comes near. + static constexpr double limit = 1.45; static constexpr unsigned strikes_needed = 3; // consecutive fast measurements (one per block) // One pose: the sender's timestamp and when it arrived, both in microseconds. Returns true