From c98c3f32076a1a035a2fa11257cabd3dd738587c Mon Sep 17 00:00:00 2001 From: AyeZeeBB Date: Wed, 7 Oct 2026 14:43:45 -0500 Subject: [PATCH] Keep busy dedicated servers playable; hold stale skaters instead of rebuilding them No protocol change: a server built from this still takes 1.1.4 players. Server: what each player is sent - A crowd limit: each player is sent at most crowd_budget poses a second (default 600, 0 for none; "crowd |off"). Everyone at 5 a second at least, the nearest at the full rate, the next nearest at 10. Before, everyone within full-rate distance was sent at the full rate, which a crowd of 40 turned into more than a connection carries - A skater's sound goes only to players within the full-rate distance, and in a crowd only from the nearest; it went to everyone - A player who joins, or arrives after a map change, is shown the others about seven a second, nearest first, and none while what they were already sent is still on its way. Before, every outfit and every stream went at once (and to everyone at once after a map change) - Whole pose and sound states every 8 seconds instead of 2, on the source's own clock, and slower pose rates sent on that clock too, so recipients share one patch and compression (DeltaCache) instead of one each. The settings survive a map change - Object updates wait for a connection that is behind Server: Steam connections - send_rate (default 900 KB/s, "rate ") is set as both Steam's minimum and maximum, for open connections and each new one. With only the maximum raised a connection stayed at Steam's 256 KB/s; set high, what was lost was resent until the connection was full of resends (about 1.1 MB/s reaches a player through the relays) - A 4 MB send buffer, and a refused must-arrive message no longer ends the player's session at once: 15 seconds of refusals do (30 for object updates). A full queue used to drop everyone who was behind together Server: seeing what is going on - "net [player]": every connection's ping, delivery, queue and rate, the traffic by kind (poses, sound, voice, outfits, objects), who uploads most, and how busy the server's loop is - A [network] line in the log every minute - A leave or failed join says how it ended: closed by the player's game with Steam's reason, connection lost, or closed by the server and why - The queue time no longer prints Steam's "unknown" as a number Server: other - Reserved slots: reserved_slots and reserved in ReSkateServer.json, and the "reserved" command; admins always have one - The speed check flags a game at about 1.5x and over (was 1.06x), and tells only that player; for the admins it is in the log Game - A remote skater with no pose for a second stays where it was for up to eight seconds instead of being destroyed and rebuilt three seconds later. On a busy server one slow frame did that to many players at once, and the rebuilding made the next frames slower still - Every waiting message is read each frame (at most 128 were) - On an official server the role tags (Dev, Staff, Homie, Creator) show for every player, not only the few the game is connected to directly; community servers keep the old rule - The server browser searches Steam at most every 15 seconds (45 after an empty answer) and keeps the last list for 15 minutes when Steam returns nothing, up from 2 - The game's own connections send at up to 900 KB/s --- Engine/Game/Multiplayer/tick_settings.h | 6 + Extension/Multiplayer/Net/delta_codec.cpp | 3 +- Extension/Multiplayer/Net/delta_codec.h | 12 + Extension/Multiplayer/Net/protocol.h | 2 + .../Multiplayer/Remote/playback_buffers.cpp | 3 + Extension/Multiplayer/Session/room.h | 35 ++ Extension/Multiplayer/Session/session.cpp | 8 +- .../Multiplayer/Session/session_internal.h | 8 +- .../Multiplayer/Session/session_receive.cpp | 3 +- Extension/Multiplayer/Steam/steam_lanes.h | 4 + .../Steam/steam_server_browser.cpp | 11 +- .../Multiplayer/Steam/steam_transport.cpp | 183 ++++++-- Extension/Multiplayer/Steam/steam_transport.h | 24 + .../Multiplayer/Test/bandwidth_tests.cpp | 38 ++ Server/README.txt | 32 +- Server/Test/server_config_tests.cpp | 39 ++ Server/Test/speed_check_tests.cpp | 7 +- Server/server_config.cpp | 24 + Server/server_config.h | 11 + Server/server_host.cpp | 443 ++++++++++++++++-- Server/server_host.h | 36 +- Server/speed_check.h | 4 +- 22 files changed, 865 insertions(+), 71 deletions(-) diff --git a/Engine/Game/Multiplayer/tick_settings.h b/Engine/Game/Multiplayer/tick_settings.h index 3f534404..43fa9b37 100644 --- a/Engine/Game/Multiplayer/tick_settings.h +++ b/Engine/Game/Multiplayer/tick_settings.h @@ -18,4 +18,10 @@ constexpr bool valid_pose_interval(std::uint32_t interval) noexcept { if (interval == multiplayer_pose_interval(rate)) return true; return false; } +// The most poses a second one player is sent on a dedicated server, whatever the crowd +// around them (Extension/Multiplayer/Session/room.h, crowd_limits); 0: no limit. +inline constexpr unsigned crowd_pose_budget = 600, min_crowd_budget = 300, max_crowd_budget = 20000; +constexpr bool valid_crowd_budget(unsigned budget) noexcept { + return !budget || (budget >= min_crowd_budget && budget <= max_crowd_budget); +} } // namespace dingosdk diff --git a/Extension/Multiplayer/Net/delta_codec.cpp b/Extension/Multiplayer/Net/delta_codec.cpp index 2ad5c1d9..1d71a4c6 100644 --- a/Extension/Multiplayer/Net/delta_codec.cpp +++ b/Extension/Multiplayer/Net/delta_codec.cpp @@ -43,7 +43,8 @@ WireUpdate DeltaSender::build(const Packet &p, std::span raw } const auto found = bases_.find({p.source, p.kind}); if (found == bases_.end() || found->second.epoch != p.epoch || found->second.world != p.world || p.time_us < found->second.time || - (p.kind != PacketKind::cosmetics && p.time_us - found->second.time >= 2000000)) { + (p.kind != PacketKind::cosmetics && + (aligned_ ? p.time_us / refresh_us_ != found->second.time / refresh_us_ : p.time_us - found->second.time >= refresh_us_))) { if (wire.size() + 4 > max_packet) return out; out.bytes.assign(wire.begin(), wire.end()); diff --git a/Extension/Multiplayer/Net/delta_codec.h b/Extension/Multiplayer/Net/delta_codec.h index 7ca5fbfc..456ced83 100644 --- a/Extension/Multiplayer/Net/delta_codec.h +++ b/Extension/Multiplayer/Net/delta_codec.h @@ -32,6 +32,8 @@ class DeltaSender { }; std::map bases_; std::uint64_t clock_{}; + std::uint64_t refresh_us_ = 2000000; + bool aligned_{}; WireUpdate build(const Packet &, std::span raw, std::span wire, DeltaCache *cache) const; @@ -44,6 +46,16 @@ class DeltaSender { WireUpdate prepare(const Packet &, std::span raw, std::span wire, DeltaCache &cache) const; void sent(const Packet &, WireUpdate &&); + // How old a pose or sound reference may get before a whole state replaces it. Each one + // is sent reliably, so Steam resends it until it arrives: a dedicated server, with a + // stream for every pair of players, keeps them rarer than the two seconds a game does. + // `aligned`: on the source's own clock, at each multiple of the time, instead of that + // long after this recipient's last one. Every recipient of a source then takes the same + // packet as its reference, and one patch serves them all (DeltaCache). + void set_refresh(std::uint64_t microseconds, bool aligned = false) { + refresh_us_ = microseconds; + aligned_ = aligned; + } }; class DeltaReceiver { struct Base { diff --git a/Extension/Multiplayer/Net/protocol.h b/Extension/Multiplayer/Net/protocol.h index 25c3540e..85792c60 100644 --- a/Extension/Multiplayer/Net/protocol.h +++ b/Extension/Multiplayer/Net/protocol.h @@ -322,6 +322,8 @@ class PoseBuffer { bool sample(std::uint64_t now_us, Pose &out) const; std::optional sample(std::uint64_t now_us) const; bool sample_remote(std::uint64_t now_us, Pose &out); + // Whether a pose arrived within this long: sampling gives up after a second without one. + bool heard_within(std::uint64_t now_us, std::uint64_t age_us) const; std::optional sample_remote(std::uint64_t now_us); PosePlayback playback() const { return playback_; } void clear(); diff --git a/Extension/Multiplayer/Remote/playback_buffers.cpp b/Extension/Multiplayer/Remote/playback_buffers.cpp index 7d1ac485..79f461d9 100644 --- a/Extension/Multiplayer/Remote/playback_buffers.cpp +++ b/Extension/Multiplayer/Remote/playback_buffers.cpp @@ -314,6 +314,9 @@ void PoseBuffer::correct(Pose &pose, std::uint64_t now) const { } offset_pose(pose, offset); } +bool PoseBuffer::heard_within(std::uint64_t now, std::uint64_t age) const { + return !frames_.empty() && (now < frames_.back().arrival || now - frames_.back().arrival <= age); +} bool PoseBuffer::sample_remote(std::uint64_t now, Pose &pose) { if (playback_at_ && (now < playback_at_ || now - playback_at_ > 200000)) { correction_start_ = correction_end_ = 0; diff --git a/Extension/Multiplayer/Session/room.h b/Extension/Multiplayer/Session/room.h index f1688390..49958abf 100644 --- a/Extension/Multiplayer/Session/room.h +++ b/Extension/Multiplayer/Session/room.h @@ -1,7 +1,9 @@ #pragma once #include "Extension/Multiplayer/Net/protocol.h" #include +#include #include +#include #include namespace dingosdk::multiplayer { @@ -76,7 +78,40 @@ struct PoseDelivery { std::uint64_t source{}, epoch{}, last_sent{}; std::uint32_t interval_us = 50000; std::uint64_t next_source_time{}; + std::uint32_t by_distance{}; // what the distance alone asked for, before any crowd limit }; +// A crowd in one place: every player there is within full-rate distance of every other, and +// what one player is sent grows with the crowd until their connection cannot carry it. The +// default budget is what was seen to arrive through Steam's relays (about 1 MB/s a player, +// whatever the send rate): nobody near is slowed until about 20 players are in one place at +// 30 TPS; past that the farthest of them are. A +// player is sent at most `budget` poses a second: everyone at the low rate at least, the +// nearest at the full rate with two thirds of what is left, the next nearest at the half +// rate with the rest. These are the squared distances beyond which a player is sent at the +// half and at the low rate; `squared` (the distances to the others) is sorted here. +struct CrowdLimits { + float half = std::numeric_limits::infinity(), low = std::numeric_limits::infinity(); +}; +inline CrowdLimits crowd_limits(std::span squared, unsigned tps, unsigned budget = crowd_pose_budget) { + const unsigned full = 1000000U / multiplayer_pose_interval(tps), half = 10, low = 5; + const auto count = static_cast(squared.size()); + if (count * full <= budget) return {}; + std::sort(squared.begin(), squared.end()); + const unsigned spare = budget > count * low ? budget - count * low : 0; + const unsigned at_full = std::min(count, spare * 2 / 3 / (full - low)); + const unsigned at_half = std::min(count - at_full, (spare - at_full * (full - low)) / (half - low)); + CrowdLimits limits; + // Between two players the limit falls halfway, so that neither sits on it. + const auto after = [&](unsigned sent) { + return sent >= count ? std::numeric_limits::infinity() : sent ? (squared[sent - 1] + squared[sent]) / 2 : -1.f; + }; + limits.half = after(at_full); + limits.low = after(at_full + at_half); + return limits; +} +inline std::uint32_t crowd_interval(std::uint32_t interval, float distance_squared, const CrowdLimits &limits) noexcept { + return distance_squared > limits.low ? 200000U : distance_squared > limits.half ? std::max(interval, 100000U) : interval; +} // Preserve the timer phase across variable client frames/packet arrivals. Only // the newest state is sent; elapsed slots are skipped without catch-up packets. inline void advance_pose_deadline(std::uint64_t &next, std::uint64_t now, std::uint64_t interval) { diff --git a/Extension/Multiplayer/Session/session.cpp b/Extension/Multiplayer/Session/session.cpp index 697b2100..08304d2c 100644 --- a/Extension/Multiplayer/Session/session.cpp +++ b/Extension/Multiplayer/Session/session.cpp @@ -503,13 +503,19 @@ void render(Session &s, std::uintptr_t client, const NativeFrame &local, std::ui if (local.ready && !p.render_failed) { const bool sampled = s.mode == Mode::echo ? p.poses.sample(now, p.render_pose) : p.poses.sample_remote(now, p.render_pose); + // Nothing new from them for over a second (a stall here, at the host or on the way): + // their skater stays where it was for a while longer. Taking it down and building + // it again three seconds later is the most expensive thing a frame can do, and on + // a busy server one slow frame made it happen to everyone at once, which made + // the next frames slower still. + const bool held = !sampled && was_visible && p.poses.heard_within(now, 8000000); // Showing a player without an actor spawns one (native_skater_spawn.cpp). const bool spawning = sampled && p.appearance.value() && !was_visible && !remote_skater_entity(); if (spawning && now < p.next_spawn) { p.native_status = "Waiting to show the player again."; } else if (spawning && native_pass) { p.native_status = "Waiting for another player's skater to finish spawning."; - } else if (sampled && p.appearance.value()) { + } else if ((sampled || held) && p.appearance.value()) { if (spawning) { native_pass = true; p.applied_cosmetics = 0; // the new actor wears no recipe yet diff --git a/Extension/Multiplayer/Session/session_internal.h b/Extension/Multiplayer/Session/session_internal.h index 3229f371..343bd1fb 100644 --- a/Extension/Multiplayer/Session/session_internal.h +++ b/Extension/Multiplayer/Session/session_internal.h @@ -1,6 +1,7 @@ #pragma once #include "Engine/Game/Multiplayer/chat_rate.h" #include "session.h" +#include "Extension/Multiplayer/developer_identity.h" #include "Extension/Customization/developer_hoodie.h" #include "Extension/Customization/developer_board.h" #include "Extension/Multiplayer/Remote/native_skater.h" @@ -324,9 +325,12 @@ inline bool dedicated_host(const Session &s) { return s.mode == Mode::join && ga // Whether Steam itself vouches for this player's identity to this PC: a host's guests and a // guest's host are connected directly, and so is another guest once the direct handshake is // done. Anyone else is known only from the host's roster, which a host can fill as it likes, -// so what rests on who a player is (the developer and friend marks) waits for this. +// so what rests on who a player is (the developer and friend marks) waits for this. An +// official server is the exception: Steam vouches for it to this PC and for each player to +// it, and it is ours, so its roster is taken at its word. On a busy one a player is +// connected directly to only the few nearest, and the rest would show no tag. inline bool steam_vouched(const Session &s, const Peer &peer) { - return s.mode != Mode::join || peer.member.id == s.host_id || peer.direct_ready; + return s.mode != Mode::join || peer.member.id == s.host_id || peer.direct_ready || official_server(s.host_id); } // Whether a player shows the tag the backend gives them, and whether the items that come with // it animate: not until their appearance has arrived, and not when it says they turned that diff --git a/Extension/Multiplayer/Session/session_receive.cpp b/Extension/Multiplayer/Session/session_receive.cpp index b1cda91e..3353b013 100644 --- a/Extension/Multiplayer/Session/session_receive.cpp +++ b/Extension/Multiplayer/Session/session_receive.cpp @@ -73,7 +73,8 @@ constexpr std::size_t chat_proofs_kept = 8; // Whether a line the host passed on as this player's would show a badge that rests on who // they are: another guest's, known to Steam, on one of the backend's lists and showing it. bool owes_proof(const Session &s, const Peer &sender) { - return s.mode == Mode::join && sender.member.id != s.host_id && steam_vouched(s, sender) && shows_tag(sender) && + // Their own copy only comes over a direct connection: without one there is none to wait for. + return s.mode == Mode::join && sender.member.id != s.host_id && sender.direct_ready && shows_tag(sender) && identity_mark(sender.member.id); } std::string chat_name(Session &s, const Peer &sender) { diff --git a/Extension/Multiplayer/Steam/steam_lanes.h b/Extension/Multiplayer/Steam/steam_lanes.h index 4eb11b68..e7fbb105 100644 --- a/Extension/Multiplayer/Steam/steam_lanes.h +++ b/Extension/Multiplayer/Steam/steam_lanes.h @@ -53,6 +53,10 @@ inline std::uint64_t lane_queue_time(const SteamNetConnectionRealTimeStatus_t &i if (time == std::numeric_limits::max()) return 0; return static_cast(std::max(0, time)); } +// A whole connection's queue time, with the same care for Steam's "cannot estimate" value. +inline std::uint64_t queue_time(const SteamNetConnectionRealTimeStatus_t &info) { + return lane_queue_time(info, {}, TrafficLane::gameplay); +} inline bool lane_congested(const SteamNetConnectionRealTimeStatus_t &info, std::span lanes, TrafficLane lane) { if (info.m_eState != k_ESteamNetworkingConnectionState_Connected) return false; diff --git a/Extension/Multiplayer/Steam/steam_server_browser.cpp b/Extension/Multiplayer/Steam/steam_server_browser.cpp index 4b0f07ae..50c37a92 100644 --- a/Extension/Multiplayer/Steam/steam_server_browser.cpp +++ b/Extension/Multiplayer/Steam/steam_server_browser.cpp @@ -159,6 +159,10 @@ const MultiplayerLobby *SteamServerBrowser::find(std::uint64_t id) const { } void SteamServerBrowser::refresh(std::uint64_t now) { if (!api().open()) return; + // Steam stops answering a game that asks often (an empty list, for minutes): one search at a + // time, a quarter of a minute apart, and three quarters after one that came back empty. + // Opening or refreshing the browser sooner shows what the last search found. + if (started_ && (!searches_.empty() || now - started_ < (internet_listed_ ? 15000000 : 45000000))) return; servers_ = api().servers(); if (!servers_) return; release(); @@ -248,11 +252,12 @@ void SteamServerBrowser::tick(std::uint64_t now) { if (done) { release(); // An internet list with servers in it is the whole truth: what it left out is gone. - // An empty one is Steam not answering (see the header): keep what was listed in the - // last two minutes. + // An empty one is Steam not answering (see the header), and it has stayed that way for + // minutes for a player who searched often: keep what was listed in the last quarter + // of an hour. A server that closed meanwhile only fails to join. const auto before = found_.size(); std::erase_if(found_, [&](const auto &entry) { - return entry.second.search != search_ && (internet_listed_ || now - entry.second.listed > 120000000); + return entry.second.search != search_ && (internet_listed_ || now - entry.second.listed > 900000000); }); if (found_.size() != before) read(now); std::string list; diff --git a/Extension/Multiplayer/Steam/steam_transport.cpp b/Extension/Multiplayer/Steam/steam_transport.cpp index 16ccf320..40153c6f 100644 --- a/Extension/Multiplayer/Steam/steam_transport.cpp +++ b/Extension/Multiplayer/Steam/steam_transport.cpp @@ -94,6 +94,39 @@ struct SteamTransport::Impl { bool recheck{}; // a status callback named this connection: read its state this poll }; std::map links; + // How the connections Steam closed ended (take_closed). A handful at most wait here. + std::map closed; + // What each connection may send a second, and Steam's call that changes an open one. + int send_rate = connection_send_rate, listen_rate = connection_send_rate; // now, and when the listen socket opened + void *utils{}; + bool (*set_config)(void *, ESteamNetworkingConfigValue, ESteamNetworkingConfigScope, intptr_t, + ESteamNetworkingConfigDataType, const void *){}; + // Gives one open connection the current send rate. A connection a listen socket + // accepts starts with the rate the socket was opened with, so each one is given it. + bool apply_rate(HSteamNetConnection handle) { + if (!handle || !set_config || !utils) return false; + bool ok = true; + const auto set = [&](ESteamNetworkingConfigValue option, int32 value) { + ok &= set_config(utils, option, k_ESteamNetworkingConfig_Connection, static_cast(handle), + k_ESteamNetworkingConfig_Int32, &value); + }; + // The minimum never above the maximum on the way, whatever the rate was before. + set(k_ESteamNetworkingConfig_SendRateMin, min_send_rate); + set(k_ESteamNetworkingConfig_SendRateMax, send_rate); + set(k_ESteamNetworkingConfig_SendRateMin, send_rate); + return ok; + } + // "ping 48 ms, quality 99%/97%, 0 B queued": what was last measured of a link. + static std::string measured(const Link &link) { + if (!link.status_ok) return "link not measured"; + const auto &s = link.status; + const auto percent = [](float quality) { return quality < 0 ? std::string("?") : std::to_string(static_cast(quality * 100.f + .5f)) + "%"; }; + return "ping " + std::to_string(s.m_nPing) + " ms, quality " + percent(s.m_flConnectionQualityLocal) + " here / " + + percent(s.m_flConnectionQualityRemote) + " there, " + std::to_string(s.m_cbPendingReliable + s.m_cbPendingUnreliable) + + " B queued (" + std::to_string(queue_time(s) / 1000) + " ms), " + + std::to_string(static_cast(s.m_flOutBytesPerSec / 1024.f + .5f)) + " KB/s out, " + + std::to_string(static_cast(s.m_flInBytesPerSec / 1024.f + .5f)) + " KB/s in"; + } std::uint64_t frame{}; ULONGLONG next_measure{}, next_sweep{}; struct Name { @@ -146,12 +179,20 @@ struct SteamTransport::Impl { reinterpret_cast(&changed)); return option; } - std::array options() { - std::array out{}; + std::array options() { + std::array out{}; out[0] = callback_option(); - // Raise this connection's ceiling; Steam retains congestion control and - // its default minimum. Do not alter the game's global Steam settings. - out[1].SetInt32(k_ESteamNetworkingConfig_SendRateMax, 1024 * 1024); + // This connection's send rate. Steam asks for the minimum and the maximum to be set + // to the same value: with only the maximum raised a connection stays at the 256 KB/s + // default, which a busy server fills for every player. Do not alter the game's + // global Steam settings. + listen_rate = send_rate; + out[1].SetInt32(k_ESteamNetworkingConfig_SendRateMax, send_rate); + out[3].SetInt32(k_ESteamNetworkingConfig_SendRateMin, send_rate); + // Room for what must arrive. Steam refuses a reliable message once 512 KB wait, its + // default, and a busy connection already holds half of that in poses: the refusal + // then ends the session for a player whose connection was only slow. + out[4].SetInt32(k_ESteamNetworkingConfig_SendBufferSize, 4 * 1024 * 1024); // Every connection goes through Steam's relays, which is how players who are not // Steam friends already connect: no player's IP address is shared with the others // in a session, whatever each one's own Steam setting is. @@ -303,6 +344,12 @@ bool SteamTransport::bind(void *library, void *sockets, void *networking_utils) } catch (...) { p.lanes = {}; } + p.utils = networking_utils; + try { + p.set_config = symbol(module, "SteamAPI_ISteamNetworkingUtils_SetConfigValue"); + } catch (...) { + p.set_config = nullptr; // open connections keep their rate; new ones get the new one + } symbol(module, "SteamAPI_ISteamNetworkingUtils_InitRelayNetworkAccess")(networking_utils); symbol( @@ -407,6 +454,9 @@ void SteamTransport::disconnect(std::uint64_t id, const char *reason) { return; const auto handle = found->second.handle; p.links.erase(found); + // Closed from this side, unless Steam's own account of the end is already kept. + if (p.closed.size() >= 256) p.closed.clear(); + p.closed.try_emplace(id, std::string("closed by this side: ") + (reason ? reason : "")); if (handle && p.close) p.close(p.sockets, handle, 1000, reason, false); p.state.detail = reason; @@ -476,6 +526,7 @@ bool SteamTransport::connect_peer(std::uint64_t id) { auto options = p.options(); const auto connection = p.connect(p.sockets, &identity, virtual_port, static_cast(options.size()), options.data()); + p.closed.erase(id); if (connection) p.links.emplace(id, Impl::Link{connection, GetTickCount64(), false, name(id), p.lanes.setup(p.sockets, connection)}); @@ -539,8 +590,10 @@ void SteamTransport::poll() { p.close(p.sockets, event.m_hConn, 4003, "Cannot accept connection", false); continue; } + p.closed.erase(id); p.links.emplace(id, Impl::Link{event.m_hConn, now, false, name(id), p.lanes.setup(p.sockets, event.m_hConn)}); + if (p.send_rate != p.listen_rate) p.apply_rate(event.m_hConn); } // Connected links report changes through the status callback (above), so read their // state only then, plus once a second in case a callback was dropped. @@ -565,6 +618,13 @@ void SteamTransport::poll() { } else if (info.m_eState == k_ESteamNetworkingConnectionState_ClosedByPeer || info.m_eState == k_ESteamNetworkingConnectionState_ProblemDetectedLocally) { const auto detail = bounded(info.m_szEndDebug, sizeof(info.m_szEndDebug)); + // Steam's end reasons: 1xxx the application closed it, 3xxx a problem at this end, + // 4xxx one at the other end (4001: it stopped answering), 5xxx anything else. + if (p.closed.size() >= 256) p.closed.clear(); + p.closed[id] = std::string(info.m_eState == k_ESteamNetworkingConnectionState_ClosedByPeer ? "closed by their game" + : "connection lost") + + ", Steam reason " + std::to_string(info.m_eEndReason) + (detail.empty() ? std::string{} : " \"" + detail + "\"") + + "; last " + Impl::measured(link); disconnect(id, detail.empty() ? "Steam connection closed." : detail.c_str()); } else if (now - link.connecting_since > 20000) disconnect(id, "Steam connection timed out."); @@ -575,6 +635,61 @@ void SteamTransport::poll() { } p.publish_links(); } +std::string SteamTransport::take_closed(std::uint64_t id) { + auto &p = *impl_; + const auto found = p.closed.find(id); + if (found == p.closed.end()) return {}; + auto text = std::move(found->second); + p.closed.erase(found); + return text; +} +int SteamTransport::send_rate() const { return impl_->send_rate; } +bool SteamTransport::set_send_rate(int rate) { + auto &p = *impl_; + if (rate < min_send_rate || rate > max_send_rate) return false; + p.send_rate = rate; + bool all = true; + for (auto &[id, link] : p.links) + if (link.handle) all &= p.apply_rate(link.handle); + return all; +} +std::int64_t SteamTransport::pending(std::uint64_t id) const { + const auto found = impl_->links.find(id); + if (found == impl_->links.end() || !found->second.status_ok) return 0; + return std::int64_t{found->second.status.m_cbPendingReliable} + found->second.status.m_cbPendingUnreliable; +} +std::string SteamTransport::link_report(std::uint64_t id) { + auto &p = *impl_; + const auto found = p.links.find(id); + if (found == p.links.end()) return "no connection"; + if (found->second.handle && found->second.connected) p.read_status(found->second); + return Impl::measured(found->second); +} +std::vector SteamTransport::links() { + auto &p = *impl_; + std::vector out; + out.reserve(p.links.size()); + for (auto &[id, link] : p.links) { + TransportLink row; + row.id = id; + row.connected = link.connected; + if (link.handle && link.connected) p.read_status(link); + if (link.connected && link.status_ok) { + const auto &s = link.status; + row.measured = true; + row.ping_ms = s.m_nPing; + row.pending_bytes = s.m_cbPendingReliable + s.m_cbPendingUnreliable; + row.send_rate = s.m_nSendRateBytesPerSecond; + row.quality_local = s.m_flConnectionQualityLocal; + row.quality_remote = s.m_flConnectionQualityRemote; + row.out_bps = s.m_flOutBytesPerSec; + row.in_bps = s.m_flInBytesPerSec; + row.queue_us = queue_time(s); + } + out.push_back(row); + } + return out; +} std::string SteamTransport::name(std::uint64_t id) { auto &p = *impl_; if (!p.friends || !id) @@ -674,33 +789,41 @@ std::vector SteamTransport::receive() { const auto &link = (it++)->second; if (!link.connected) continue; - SteamNetworkingMessage_t *messages[128]{}; - const int count = p.receive_messages(p.sockets, link.handle, messages, 128); - if (count < 0) { - disconnect(id, "Steam receive failed."); - continue; - } - for (int i = 0; i < count; ++i) { - auto *message = messages[i]; - if (!message) - continue; - struct Release { - Impl &p; - SteamNetworkingMessage_t *m; - ~Release() { p.release_message(m); } - } release{p, message}; - if (message->m_cbSize <= 0 || message->m_cbSize > static_cast(max_packet) || - !message->m_pData) { - ++p.state.dropped; - ++p.state.invalid_messages; - continue; + // Everything that has arrived, not one batch a call: a dedicated server sends a busy + // session's player a few thousand messages a second, and what a slow frame leaves + // behind would wait in Steam, later and later, taking memory until it is read. + bool failed{}; + for (unsigned batch = 0; batch < 64 && !failed; ++batch) { + SteamNetworkingMessage_t *messages[128]{}; + const int count = p.receive_messages(p.sockets, link.handle, messages, 128); + if (count < 0) { + failed = true; + break; } - const auto *data = static_cast(message->m_pData); - result.push_back({id, {data, data + message->m_cbSize}, - message->m_usecTimeReceived > 0 ? static_cast(message->m_usecTimeReceived) : 0}); - ++p.state.received; - p.state.received_bytes += static_cast(message->m_cbSize); + for (int i = 0; i < count; ++i) { + auto *message = messages[i]; + if (!message) + continue; + struct Release { + Impl &p; + SteamNetworkingMessage_t *m; + ~Release() { p.release_message(m); } + } release{p, message}; + if (message->m_cbSize <= 0 || message->m_cbSize > static_cast(max_packet) || + !message->m_pData) { + ++p.state.dropped; + ++p.state.invalid_messages; + continue; + } + const auto *data = static_cast(message->m_pData); + result.push_back({id, {data, data + message->m_cbSize}, + message->m_usecTimeReceived > 0 ? static_cast(message->m_usecTimeReceived) : 0}); + ++p.state.received; + p.state.received_bytes += static_cast(message->m_cbSize); + } + if (count < 128) break; } + if (failed) disconnect(id, "Steam receive failed."); } return result; } diff --git a/Extension/Multiplayer/Steam/steam_transport.h b/Extension/Multiplayer/Steam/steam_transport.h index d035b136..8f9fb97e 100644 --- a/Extension/Multiplayer/Steam/steam_transport.h +++ b/Extension/Multiplayer/Steam/steam_transport.h @@ -13,6 +13,15 @@ struct TransportPeer { std::uint64_t id{}; bool connected{}; }; +// What Steam measures of one connection right now (SteamTransport::links). +struct TransportLink { + std::uint64_t id{}; + bool connected{}, measured{}; + int ping_ms{}, pending_bytes{}, send_rate{}; // send_rate: bytes a second Steam will let out to them + float quality_local = -1, quality_remote = -1; // 0-1 of packets delivered each way; -1 unknown + float out_bps{}, in_bps{}; + std::uint64_t queue_us{}; // how long a message sent now would wait +}; struct TransportMessage { std::uint64_t peer{}; std::vector bytes; @@ -44,6 +53,8 @@ struct TransportStatus { std::uint64_t queue_us{}, skipped{}, send_failures{}, invalid_messages{}, sent_bytes{}, received_bytes{}, raw_sent_bytes{}; }; +// What each connection may send a second (bytes): a crowded server's players, mostly. +inline constexpr int connection_send_rate = 900 * 1024, min_send_rate = 128 * 1024, max_send_rate = 16 * 1024 * 1024; class SteamTransport { public: SteamTransport(); @@ -69,6 +80,19 @@ class SteamTransport { std::vector receive(); std::string name(std::uint64_t id); const TransportStatus &status() const; + // For a host's log. How Steam says a connection it closed ended, once (empty when this + // side closed it, or it is still open): who closed it, Steam's reason code and words, and + // the last ping and quality measured. And a connected player's link right now. + std::string take_closed(std::uint64_t id); + std::string link_report(std::uint64_t id); + // Bytes waiting to go out to a player, as last measured (a few times a second). + std::int64_t pending(std::uint64_t id) const; + // What each connection may send a second from now on, the open ones included. False when + // out of range, or when an open connection could not be changed (new ones still get it). + bool set_send_rate(int bytes_per_second); + int send_rate() const; + // Every connection, measured now: one Steam call each, so for a command, not every tick. + std::vector links(); private: bool bind(void *steam_api, void *sockets, void *networking_utils); diff --git a/Extension/Multiplayer/Test/bandwidth_tests.cpp b/Extension/Multiplayer/Test/bandwidth_tests.cpp index 6af0044b..ba2588a4 100644 --- a/Extension/Multiplayer/Test/bandwidth_tests.cpp +++ b/Extension/Multiplayer/Test/bandwidth_tests.cpp @@ -135,6 +135,44 @@ void rate_checks() { pose_interval(49*49, 100000) == 50000, "Near-rate hysteresis failed"); check(pose_interval(171*171, 100000) == 200000 && pose_interval(160*160, 200000) == 200000 && pose_interval(149*149, 200000) == 100000, "Far-rate hysteresis failed"); + { + // A crowd: no limit while everyone fits at the full rate, then the nearest at full, + // the next at half, the rest at low, within the budget. + std::vector few(20), crowd(49), packed(127); + for (std::size_t i = 0; i < packed.size(); ++i) { + const auto d = static_cast((packed.size() - i) * (packed.size() - i)); // farthest first + packed[i] = d; + if (i < crowd.size()) crowd[i] = d; + if (i < few.size()) few[i] = d; + } + const auto none = crowd_limits(few, 30); + // The default leaves twenty players in one place at the full rate, each to each. + std::vector twenty(crowd.begin(), crowd.begin() + 19); + check(crowd_limits(twenty, 30).half == none.half && dingosdk::valid_crowd_budget(0) && + dingosdk::valid_crowd_budget(dingosdk::crowd_pose_budget) && !dingosdk::valid_crowd_budget(50), + "The default crowd budget slowed twenty players, or a bad one was valid"); + check(crowd_interval(33333, 1e9f, none) == 33333, "A small group was limited"); + const auto count = [](std::span all, const CrowdLimits &limits, unsigned tps) { + unsigned sent{}; + for (const auto d : all) sent += 1000000U / crowd_interval(dingosdk::multiplayer_pose_interval(tps), d, limits); + return sent; + }; + for (const unsigned tps : {20U, 30U, 60U, 120U}) { + const auto limits = crowd_limits(crowd, tps, 600); + const auto sent = count(crowd, limits, tps); + check(sent <= 602 && sent > 450, "A crowd was not sent within the budget"); + check(crowd_interval(dingosdk::multiplayer_pose_interval(tps), crowd.front(), limits) == dingosdk::multiplayer_pose_interval(tps) && + crowd_interval(dingosdk::multiplayer_pose_interval(tps), crowd.back(), limits) == 200000, + "A crowd's nearest and farthest were not sent at the full and low rates"); + check(limits.half < limits.low, "A crowd had no half-rate ring"); + } + // Too many for the budget even at the low rate: all at the low rate, which is the floor. + const auto all_low = crowd_limits(packed, 30, 600); + check(count(packed, all_low, 30) == packed.size() * 5, "An over-full crowd was not sent at the low rate"); + // Distance still slows what the crowd limit would send faster. + check(crowd_interval(200000, 0, none) == 200000 && crowd_interval(100000, 0, crowd_limits(crowd, 30, 600)) == 100000, + "A crowd limit sped a far player up"); + } for (auto interval : {8333U, 16666U, 33333U, 50000U, 100000U, 200000U}) { PoseBuffer buffer; auto p = fixture(); p.pose_interval_us = interval; diff --git a/Server/README.txt b/Server/README.txt index 4c6d3b50..170323ac 100644 --- a/Server/README.txt +++ b/Server/README.txt @@ -55,6 +55,27 @@ map_rotation_minutes Minutes on each map before the server moves to the next warning; the clock waits while nobody is on, and starts over whenever the map changes (by a vote or an admin too). max_players 1-249. +send_rate The most the server sends one player, in KB/s (default 900, + 128-16384). About 1100 is what reaches a player through + Steam's relays: set higher, what is lost is resent until the + connection is full of resends and half of everything is + lost. Console: rate , also for the players already on. +crowd_budget The most position updates a second one player is sent + (default 600, 0 for no limit). Players near each other are + sent at the full rate (tps); this only matters once more are + in one place than budget / tps, about 20 at 30 TPS. Then the + nearest stay at the full rate and the farthest of the crowd + drop to 10 and 5 a second, instead of everyone's connection + filling up. About 1000 is what reaches a player through + Steam's relays; more than that was seen to lose half. + Console: crowd |off. +reserved_slots How many of those are kept for the players in "reserved" and + the admins (default 0). With 64 players and 4 reserved slots, + anyone can join until 60 are on; the last 4 only they can + take. The browser still shows 64, and everyone else is told + the remaining slots are reserved. +reserved SteamID64s of the players with a reserved slot, e.g. + ["76561198000000000"]. Admins always have one. password Empty for anyone; otherwise players type it to join. welcome A chat line sent to each player as they join. listed false hides the server; players then need the code. @@ -95,7 +116,7 @@ speed_check Catch players whose game runs faster than normal (Cheat Engine's speedhack and the like), measured from the timing of what their game sends: "warn" (default) takes them out of throwdowns and coop challenges until their speed is normal - again and tells the admins, "kick" removes them from the + again and tells them so (the log has it for the admins), "kick" removes them from the server, "off" does not check. score_check Catch players whose mods change how many points tricks score (per-trick points, the scoring multipliers, the @@ -148,6 +169,15 @@ and change voice, distances, placement and kicks from the Multiplayer menu. help A short list of every command. status Name, map, players, code. players Connected players and their SteamID64s. + reserved Who has a reserved slot and how many are kept. + reserved slots Keep n slots for them (console only), 0 for none. + reserved add|remove (console only) + net [player] How the connections are doing right now: traffic, + queues, the server's own loop timing, and the twelve + connections doing worst. With a player, that one's + ping, delivery, queue and when it was last heard from. + The log also gets a [network] line every minute and, + when a player's connection ends, how Steam says it ended. say Chat as the server (console only). msg Private message, shown to them as "[DM from ] ...". Name start (one word) or SteamID64. diff --git a/Server/Test/server_config_tests.cpp b/Server/Test/server_config_tests.cpp index 96072bed..40a3b0a3 100644 --- a/Server/Test/server_config_tests.cpp +++ b/Server/Test/server_config_tests.cpp @@ -60,6 +60,45 @@ int run() { check(token_refused(tokened), "A steam_token with other characters was accepted"); tokened.steam_token = std::string(65, 'A'); check(token_refused(tokened), "An overlong steam_token was accepted"); + { + check(has("crowd_budget") && config.crowd_budget == 600, "crowd_budget is not a new setting of 600"); + check(has("send_rate") && config.send_rate == 900, "send_rate is not a new setting of 900"); + ServerConfig rate; + rate.send_rate = 64; + check(config_error(rate).find("send_rate") != std::string::npos, "A send rate too low to play with was accepted"); + ServerConfig crowd; + crowd.crowd_budget = 50; + check(config_error(crowd).find("crowd_budget") != std::string::npos, "A crowd budget too small to play with was accepted"); + } + // Reserved slots: none unless set; the last of them only the listed players and admins take. + { + check(has("reserved_slots") && has("reserved") && config.reserved_slots == 0 && config.reserved.empty(), + "Reserved slots are not new settings that reserve nothing"); + ServerConfig slots; + slots.max_players = 4; + const std::uint64_t vip = 76561198000000001ULL, admin = 76561198000000002ULL, anyone = 76561198000000003ULL; + check(may_join(slots, anyone, 3) && !may_join(slots, anyone, 4) && !may_join(slots, vip, 4), "A server with nothing reserved did not fill up plainly"); + slots.reserved_slots = 2; + slots.reserved = {vip}; + slots.admins = {admin}; + check(may_join(slots, anyone, 1) && !may_join(slots, anyone, 2) && !may_join(slots, anyone, 3), "Reserved slots were given to anyone"); + check(may_join(slots, vip, 2) && may_join(slots, vip, 3) && may_join(slots, admin, 3), "A reserved player or an admin was kept out of a reserved slot"); + check(!may_join(slots, vip, 4) && !may_join(slots, admin, 4), "A full server let someone in"); + const auto refused = [](const ServerConfig &c) { return config_error(c).find("reserved") != std::string::npos; }; + check(!refused(slots), "Valid reserved slots were refused"); + slots.reserved_slots = 4; + check(refused(slots), "A server with every slot reserved was accepted"); + slots.reserved_slots = 2; + slots.reserved.push_back(42); + check(refused(slots), "A reserved entry that is not a player was accepted"); + auto kept = reloaded; + kept.reserved_slots = 3; + kept.reserved = {vip}; + save_config(kept); + const auto back = load_config(file); + check(back.reserved_slots == 3 && back.reserved == std::vector{vip}, "Reserved slots were not kept"); + save_config(reloaded); + } // object_limit: 100 unless set (0 is no limit), and kept on a rewrite. check(has("object_limit") && config.object_limit == 100, "object_limit is not a new setting of 100"); auto limited = reloaded; diff --git a/Server/Test/speed_check_tests.cpp b/Server/Test/speed_check_tests.cpp index 34bb5726..ad25a03e 100644 --- a/Server/Test/speed_check_tests.cpp +++ b/Server/Test/speed_check_tests.cpp @@ -56,8 +56,9 @@ int main() { check(fast.flagged && std::abs(fast.speed - 1.5) < 0.02, "A 1.5x speedhack was not caught"); const auto slight = run(1.03, 120, false); check(!slight.flagged, "A 3% difference was flagged"); - const auto subtle = run(1.1, 60, false); - check(subtle.flagged, "A 1.1x speedhack was not caught"); + // Under half as fast again is left alone; from there up it is caught. + check(!run(1.1, 60, false).flagged && !run(1.3, 60, false).flagged, "A game under 1.5x speed was flagged"); + check(run(2.0, 60, false).flagged, "A 2x speedhack was not caught"); const auto slow = run(0.5, 60, false); check(!slow.flagged, "A slowed-down client was flagged"); // Loading (no poses for a while) starts the measurement over. @@ -65,5 +66,5 @@ int main() { for (int i = 0; i < 30 * 15; ++i) gaps.sample(1000000 + i * 50000, 1000000 + i * 33333); gaps.sample(1000000 + 30 * 15 * 50000, 1000000 + 30 * 15 * 33333 + 5000000); check(!gaps.flagged() && gaps.speed() == 1.0, "A pause did not restart the measurement"); - std::cout << "Speed check: real time, lag, 1.5x and 1.1x speedhacks, small drift, slow motion and pauses passed.\n"; + std::cout << "Speed check: real time, lag, 1.5x and 2x speedhacks, 1.1x and 1.3x left alone, small drift, slow motion and pauses passed.\n"; } diff --git a/Server/server_config.cpp b/Server/server_config.cpp index 518f3cb8..b109db62 100644 --- a/Server/server_config.cpp +++ b/Server/server_config.cpp @@ -23,6 +23,12 @@ Json to_json(const ServerConfig &c) { root["map_pool"] = std::move(pool); root["map_rotation_minutes"] = c.map_rotation; root["max_players"] = c.max_players; + root["reserved_slots"] = c.reserved_slots; + root["crowd_budget"] = c.crowd_budget; + root["send_rate"] = c.send_rate; + auto reserved = Json::array(); + for (const auto id : c.reserved) reserved.push_back(std::to_string(id)); // as strings, like the admins + root["reserved"] = std::move(reserved); root["password"] = c.password; root["welcome"] = c.welcome; root["listed"] = c.listed; @@ -124,6 +130,11 @@ ServerConfig load_config(const std::filesystem::path &file, std::vector parse_scoring(std::string_view text) { if (!value) return std::nullopt; return value; } +bool may_join(const ServerConfig &config, std::uint64_t id, std::size_t on) noexcept { + if (on >= config.max_players) return false; + const auto listed = [&](const std::vector &ids) { return std::find(ids.begin(), ids.end(), id) != ids.end(); }; + return on + config.reserved_slots < config.max_players || listed(config.reserved) || listed(config.admins); +} std::string config_error(const ServerConfig &c) { using namespace multiplayer; if (!valid_server_name(c.name)) return std::string("name must be ") + server_name_rule + "."; + for (const auto id : c.reserved) + if (!individual_steam_id(id)) return "reserved must be SteamID64s (17 digits starting 7656119)."; + if (c.reserved_slots >= c.max_players) return "reserved_slots must be less than max_players, so that anyone can join at all."; + if (c.reserved.size() > 1024) return "reserved holds at most 1024 players."; + if (c.send_rate < 128 || c.send_rate > 16384) return "send_rate must be 128 to 16384 (KB/s for each player)."; + if (!valid_crowd_budget(c.crowd_budget)) + return "crowd_budget must be 0 (no limit) or " + std::to_string(min_crowd_budget) + " to " + + std::to_string(max_crowd_budget) + "."; if (c.steam_token.size() > 64 || !std::all_of(c.steam_token.begin(), c.steam_token.end(), [](unsigned char ch) { return std::isalnum(ch); })) return "steam_token must be a game server login token (letters and digits), or empty to sign in anonymously."; if (c.map.empty() || !valid_map_destination(map_destination(c.map))) diff --git a/Server/server_config.h b/Server/server_config.h index 61d54208..a46dc361 100644 --- a/Server/server_config.h +++ b/Server/server_config.h @@ -36,6 +36,14 @@ struct ServerConfig { std::vector map_pool; // maps for votes and the rotation, in order; empty: every map unsigned map_rotation = 0; // minutes per map before the next pool map (0: off) unsigned max_players = 16; // players; the server itself is not one + // Of those, how many are kept for the players in `reserved` and the admins: everyone else + // is told the server is full once only these are left. 0: none are kept. + unsigned reserved_slots = 0; + // The most poses a second one player is sent (crowd_limits); 0: no limit. + unsigned crowd_budget = crowd_pose_budget; + // What the server may send each player, in KB/s (128-16384). + unsigned send_rate = 900; + std::vector reserved; std::string password; // empty: anyone may join std::string welcome; // sent to each player as they join bool listed = true; // shown in the in-game server browser @@ -88,6 +96,9 @@ struct ServerConfig { // their names go to `added` ("votes.seconds" for a nested one). ServerConfig load_config(const std::filesystem::path &file, std::vector *added = nullptr); void save_config(const ServerConfig &config); +// Whether a player who is not yet on may join a server with `on` players on it: anyone while +// an unreserved slot is free, then only the reserved players and the admins until it is full. +bool may_join(const ServerConfig &config, std::uint64_t id, std::size_t on) noexcept; // Why `config` cannot run, or empty. std::string config_error(const ServerConfig &config); // A scoring fingerprint as the config and console write it (16 hex digits), and read back diff --git a/Server/server_host.cpp b/Server/server_host.cpp index 56c9afe1..8c843e85 100644 --- a/Server/server_host.cpp +++ b/Server/server_host.cpp @@ -42,16 +42,16 @@ std::uint64_t nonce() { return value; } constexpr std::string_view help_text = - "status | players | say | msg | msg-party | msg-admins | kick | ban [name] | unban | bans\n" + "status | net [player] | players | say | msg | msg-party | msg-admins | kick | ban [name] | unban | bans\n" "map | maps | name | password | welcome | listed on|off\n" - "tps 20|30|60|120 | voice on|off | voice-range <50-1000> | distances \n" + "tps 20|30|60|120 | voice on|off | voice-range <50-1000> | distances | crowd |off | rate \n" "placement everyone|admins|nobody | objects |off | clear-objects | noclip on|off | nobail on|off | boosts on|off | tuning on|off\n" "tpall [player] | tphere | votes [map|kick|tod on|off|] | vote-cancel\n" "map-pool [add|remove |clear] | rotation [|off]\n" "park | layer-sync on|off | layer default|on|off | tod \n" "activity-log on|off | announce-throwdowns on|off | parties [on|off] | party-size <2-8> | speed-check off|warn|kick\n" "score-check [off|warn|kick] | score-allow [|remove ]\n" - "admin add|remove | admins | update | quit"; + "reserved [slots | add|remove ] | admin add|remove | admins | update | quit"; } // namespace Host::Host(ServerConfig &config, SteamTransport &transport, Log log) @@ -137,6 +137,7 @@ unsigned Host::players() const { } bool Host::start(std::string &error) { + transport_.set_send_rate(static_cast(config_.send_rate * 1024)); if (!transport_.host(capacity())) { error = transport_.status().detail; return false; @@ -173,6 +174,13 @@ void Host::apply_layers() { } // ---- Sending --------------------------------------------------------------------------------- +namespace { +constexpr std::array traffic_names{"poses", "sound", "voice", "outfits", "objects", "other"}; +constexpr std::size_t traffic_kind(PacketKind kind) noexcept { + return kind == PacketKind::pose ? 0 : kind == PacketKind::audio ? 1 : kind == PacketKind::voice ? 2 + : kind == PacketKind::cosmetics ? 3 : kind == PacketKind::objects ? 4 : 5; +} +} // namespace bool Host::send_packet(Guest &g, const Packet &p, bool reliable, bool fresh, std::span raw, std::span wire) { auto update = raw.empty() ? g.sender.prepare(p) : g.sender.prepare(p, raw, wire); @@ -182,6 +190,10 @@ bool Host::send_packet(Guest &g, const Packet &p, bool reliable, bool fresh, std // A stream and its reliable delta references must always use the same lane. if (!transport_.send(g.member.id, update.bytes, reliable || update.establishes_baseline(), fresh, traffic_lane(p.kind))) return false; + for (auto *counted : {&g.traffic, &traffic_}) { + counted->total.out[traffic_kind(p.kind)] += update.bytes.size(); + counted->total.snapshots += update.establishes_baseline(); + } g.sender.sent(p, std::move(update)); return true; } @@ -193,8 +205,13 @@ void Host::send_required(Guest &g, const std::vector &bytes) { } void Host::broadcast(const Packet &packet, bool reliable, bool fresh, std::uint64_t except) { const auto *source = find(packet.source); - struct Encoded { Packet packet; std::vector raw, wire; }; + // Recipients holding the same delta reference share one patch and compression + // (DeltaCache): every update is built first, then all are sent and recorded. + struct Encoded { Packet packet; std::vector raw, wire; DeltaCache deltas; }; std::array encoded; + struct Outgoing { Guest *guest; std::uint64_t id; WireUpdate update; PoseDelivery *delivery; std::uint32_t interval; unsigned variant; }; + std::vector outgoing; + outgoing.reserve(guests_.size()); const bool gameplay = packet.kind == PacketKind::pose || packet.kind == PacketKind::audio || packet.kind == PacketKind::voice || packet.kind == PacketKind::cosmetics; for (auto &[id, guest] : guests_) { @@ -212,6 +229,23 @@ void Host::broadcast(const Packet &packet, bool reliable, bool fresh, std::uint6 continue; } } + // Not before they have been shown this player (introduce). + if (!p.unmet.empty() && (packet.kind == PacketKind::pose || packet.kind == PacketKind::audio || packet.kind == PacketKind::cosmetics) && + p.unmet.contains(packet.source)) + continue; + // A skater's sound goes to those who would hear it: within the full pose rate's reach, + // and in a crowd only from the nearest (the same players sent at the full rate). + // A game stops a sound it hears nothing more of after a second. + if (packet.kind == PacketKind::audio && source && source->latest_root && source->pose_arrival && + now_ - source->pose_arrival <= 1000000 && p.latest_root && p.pose_arrival && now_ - p.pose_arrival <= 1000000) { + float distance{}; + for (unsigned i = 0; i < 3; ++i) { + const auto d = p.latest_root->position[i] - source->latest_root->position[i]; + distance += d * d; + } + const auto reach = static_cast(config_.distances.half_rate_start); + if (distance > p.crowd.half || (config_.distances.valid() && distance > reach * reach)) continue; + } // Frequent gameplay streams skip the server once the receiver confirms a direct route. if (source && (packet.kind == PacketKind::pose || packet.kind == PacketKind::audio) && !needs_relay(p.direct_routes, p.route_reported, source->member, now_)) @@ -226,14 +260,17 @@ void Host::broadcast(const Packet &packet, bool reliable, bool fresh, std::uint6 [](const auto &a, const auto &b) { return a.last_sent < b.last_sent; }); delivery = &*entry; if (delivery->source != packet.source || delivery->epoch != packet.epoch) - *delivery = {packet.source, packet.epoch, 0, interval}; + *delivery = {packet.source, packet.epoch, 0, interval, 0, 0}; if (p.latest_root && p.pose_arrival && now_ >= p.pose_arrival && now_ - p.pose_arrival <= 1000000) { float distance{}; for (unsigned i = 0; i < 3; ++i) { const auto d = p.latest_root->position[i] - packet.pose.root.position[i]; distance += d * d; } - interval = pose_interval(distance, delivery->interval_us, config_.distances, config_.tps); + delivery->by_distance = pose_interval(distance, delivery->by_distance, config_.distances, config_.tps); + interval = crowd_interval(delivery->by_distance, distance, p.crowd); + } else { + delivery->by_distance = 0; } if (delivery->interval_us != interval) delivery->next_source_time = 0; delivery->interval_us = interval; @@ -247,12 +284,83 @@ void Host::broadcast(const Packet &packet, bool reliable, bool fresh, std::uint6 data.raw = encode(data.packet, true); data.wire = encode_wire_bytes(data.raw); } - const bool sent = send_packet(p, data.packet, reliable, fresh, data.raw, data.wire); - if (sent && delivery) { - delivery->last_sent = now_; - advance_pose_deadline(delivery->next_source_time, packet.time_us, interval); + outgoing.push_back({&p, id, p.sender.prepare(data.packet, data.raw, data.wire, data.deltas), delivery, interval, variant}); + } + for (auto &out : outgoing) { + // A packet that cannot be built for anyone is its source's fault, never this + // recipient's: nothing is sent, and the recipient is not treated as unreachable. + bool sent = true; + if (!out.update.bytes.empty()) { + const auto &sending = encoded[out.variant].packet; + const bool whole = out.update.establishes_baseline(); + sent = transport_.send(out.id, out.update.bytes, reliable || whole, fresh, traffic_lane(sending.kind)); + if (sent) { + for (auto *counted : {&out.guest->traffic, &traffic_}) { + counted->total.out[traffic_kind(sending.kind)] += out.update.bytes.size(); + counted->total.snapshots += whole; + } + out.guest->sender.sent(sending, std::move(out.update)); + } } - if (!sent && reliable && !fresh) transport_.disconnect(id, "Cannot deliver required session data. Join again."); + if (sent && out.delivery) { + out.delivery->last_sent = now_; + // The next slot on the source's own clock, not this long after this send: everyone + // sent a source at a slower rate is then sent the same packets of it, and shares + // their encoding and patch. + out.delivery->next_source_time = (packet.time_us / out.interval + 1) * out.interval; + } + // Refused by a full queue: a connection that is only behind keeps its player (a chat + // line or an outfit change is lost to them); one that takes nothing for a while does not. + if (reliable && !fresh) { + auto &since = out.guest->undelivered_since; + if (sent) since = 0; + else if (!since) since = now_; + else if (now_ - since > 15000000) transport_.disconnect(out.id, "Cannot deliver required session data. Join again."); + } + } +} +// A player who has just arrived in this world is shown the others a few at a time (introduce). +void Host::meet_later(Guest &guest) { + guest.unmet.clear(); + for (const auto &[id, other] : guests_) + if (other->handshaken && id != guest.member.id) guest.unmet.insert(id); + guest.next_introduction = now_; +} +// Shows each arriving player one more of the others: that player's outfit, after which their +// poses and sound follow. The nearest first, about seven a second, and none while what +// they have already been sent is still on its way. A busy server's eighty outfits and +// streams in one burst is what a joining game, building every skater at once, fell over on. +void Host::introduce() { + for (auto &[id, guest] : guests_) { + auto &g = *guest; + if (g.unmet.empty() || !g.handshaken || !g.world_ready || now_ < g.next_introduction) continue; + if (transport_.pending(id) > 96 * 1024) continue; + const Guest *next{}; + float nearest{}; + for (auto it = g.unmet.begin(); it != g.unmet.end();) { + const auto *other = find(*it); + if (!other || !other->handshaken) { + it = g.unmet.erase(it); + continue; + } + ++it; + float distance = std::numeric_limits::max(); + if (g.latest_root && other->latest_root) { + distance = 0; + for (unsigned i = 0; i < 3; ++i) { + const auto d = g.latest_root->position[i] - other->latest_root->position[i]; + distance += d * d; + } + } + if (!next || distance < nearest) { + next = other; + nearest = distance; + } + } + if (!next) continue; + g.unmet.erase(next->member.id); + g.next_introduction = now_ + 150000; + send_required(g, next->cosmetic_packet); } } void Host::send_roster() { @@ -355,6 +463,8 @@ void Host::change_map(std::string_view map) { for (auto &[id, guest] : guests_) { auto &g = *guest; auto old = std::exchange(g, Guest{}); + g.sender.set_refresh(8000000, true); // as for a new player (tick): a new world's streams start over + g.traffic = old.traffic; g.member = std::move(old.member); g.handshaken = old.handshaken; g.map_authorized = old.map_authorized; @@ -387,21 +497,22 @@ void Host::change_map(std::string_view map) { send_world_state(); roster_dirty_ = true; } -void Host::drop(std::uint64_t id, const std::string &reason) { +void Host::drop(std::uint64_t id, const std::string &reason, const std::string &detail) { transport_.disconnect(id, reason.c_str()); const auto found = guests_.find(id); if (found == guests_.end()) return; const auto name = guest_name(*found->second); if (found->second->handshaken) { roster_dirty_ = true; - log_(name + " left (" + reason + ")"); + log_(name + " left (" + reason + ")" + (detail.empty() ? std::string{} : " [" + detail + "]")); activity_.left(id); } else { // Never admitted: it held a player slot meanwhile, so it shows in the log, and an ID // that keeps failing waits longer each time before its connection is taken again. const auto failures = join_backoff_.failed(id, now_); log_("[join] " + std::to_string(id) + " did not finish joining (" + reason + ")" + - (failures > 1 ? ", attempt " + std::to_string(failures) : std::string{})); + (failures > 1 ? ", attempt " + std::to_string(failures) : std::string{}) + + (detail.empty() ? std::string{} : " [" + detail + "]")); } guests_.erase(found); party_left(id, name); @@ -448,6 +559,8 @@ void Host::receive(std::uint64_t peer, std::span bytes, std: if (!decoded || decoded->session != secret_ || decoded->source != peer) return drop(peer, "Join code, sender identity, or multiplayer protocol did not match."); const auto &p = *decoded; + link->traffic.total.in[traffic_kind(p.kind)] += bytes.size(); + traffic_.total.in[traffic_kind(p.kind)] += bytes.size(); switch (p.kind) { case PacketKind::world_state: return drop(peer, "Only the server may change the room's map."); // Sent once the player's game knows, and again if a mod enabled later changes it; not tied to @@ -474,9 +587,7 @@ void Host::receive(std::uint64_t peer, std::span bytes, std: log_("[map] " + guest_name(*link) + " finished loading (" + std::to_string((now_ - since) / 1000000) + " s)"); link->world_ready = p.world_ready; if (p.world_ready) link->travel_since = link->loading_since = 0; - if (arrived) - for (auto &[id, other] : guests_) - if (other->handshaken && id != peer) send_required(*link, other->cosmetic_packet); + if (arrived) meet_later(*link); return; } case PacketKind::map_request: { @@ -548,8 +659,7 @@ void Host::receive(std::uint64_t peer, std::span bytes, std: send_required(*link, encode_wire(packet(PacketKind::welcome, now_))); if (joined) { send_roster(); - for (auto &[id, other] : guests_) - if (other->handshaken && id != peer) send_required(*link, other->cosmetic_packet); + meet_later(*link); if (!config_.welcome.empty()) send_chat(config_.welcome, link); log_(guest_name(*link) + " joined (" + std::to_string(peer) + (is_admin(peer) ? ", admin" : "") + "), " + std::to_string(players()) + "/" + std::to_string(config_.max_players) + " players" + @@ -712,6 +822,9 @@ void Host::sync_objects() { for (auto &[id, guest] : guests_) { auto &peer = *guest; if (!peer.handshaken || !peer.world_ready || sources.empty()) continue; + // Objects wait for a connection that is behind: they must arrive, so they would only + // queue up behind the poses and push those out. + if (transport_.pending(id) > 128 * 1024) continue; auto &delivery = peer.object_delivery; const auto find_source = [&](std::uint64_t source, std::uint64_t epoch) { return std::find_if(sources.begin(), sources.end(), [&](const auto &s) { return s.id == source && s.epoch == epoch; }); @@ -740,9 +853,14 @@ void Host::sync_objects() { update.epoch = delivery.epoch; update.objects = delivery.chunks[delivery.next]; if (!send_packet(peer, update, true, false)) { - transport_.disconnect(id, "Cannot deliver shared object state. Join again."); + // Refused for now (a full queue): the same part is tried again. Only a + // connection that takes none for a long while is given up on. + if (!delivery.failing_since) delivery.failing_since = now_; + if (now_ - delivery.failing_since > 30000000) + transport_.disconnect(id, "Cannot deliver shared object state. Join again."); break; } + delivery.failing_since = 0; if (++delivery.next == delivery.chunks.size()) { delivery.sent[delivery.source] = {delivery.epoch, update.objects.revision}; delivery.chunks.clear(); @@ -751,10 +869,165 @@ void Host::sync_objects() { } } +// `net`: how the connections and the server's own loop are doing right now. With a player's +// name or SteamID64, that player alone. In-game admins get the first line: a reply has to fit +// one chat-sized message. +std::string Host::network_report(std::string_view player, bool console) { + const auto links = transport_.links(); + const auto &net = transport_.status(); + const auto kb = [](float bytes) { return std::to_string(static_cast(bytes / 1024.f + .5f)); }; + const auto percent = [](float quality) { return quality < 0 ? std::string("?") : std::to_string(static_cast(quality * 100.f + .5f)) + "%"; }; + const auto row = [&](const multiplayer::TransportLink &link) { + const auto *guest = find(link.id); + std::string name = guest ? guest_name(*guest) : std::to_string(link.id); + if (name.size() > 20) name.resize(20); + name.resize(21, ' '); + if (!link.measured) return " " + name + (link.connected ? "not measured yet" : "connecting"); + const auto silent = guest && guest->last_packet && now_ > guest->last_packet ? (now_ - guest->last_packet) / 100000 : 0; // tenths of a second + return " " + name + "ping " + std::to_string(link.ping_ms) + " ms, quality " + percent(link.quality_local) + " here / " + + percent(link.quality_remote) + " there, queued " + std::to_string(link.pending_bytes) + " B (" + + std::to_string(link.queue_us / 1000) + " ms), out " + kb(link.out_bps) + " of " + kb(static_cast(link.send_rate)) + + " KB/s, in " + kb(link.in_bps) + " KB/s, last heard " + std::to_string(silent / 10) + "." + std::to_string(silent % 10) + " s ago" + + (guest && !guest->handshaken ? ", joining" : guest && !guest->world_ready ? ", loading" : ""); + }; + // By what it carries, in KB/s: the last whole half minute, or the one so far. + const bool whole = traffic_window_us_ != 0; + const double seconds = whole ? static_cast(traffic_window_us_) / 1e6 + : std::max(1.0, static_cast(now_ - traffic_mark_) / 1e6); + const auto rate = [&](const Counted &c, bool out, std::size_t kind) { + const auto bytes = whole ? (out ? c.last.out[kind] : c.last.in[kind]) + : out ? c.total.out[kind] - c.mark.out[kind] : c.total.in[kind] - c.mark.in[kind]; + return static_cast(bytes) / 1024.0 / seconds; + }; + const auto number = [](double value) { + char text[32]; + std::snprintf(text, sizeof(text), value < 10 ? "%.1f" : "%.0f", value); + return std::string(text); + }; + const auto by_kind = [&](const Counted &c) { + std::string text = "KB/s out / in:"; + for (std::size_t kind = 0; kind < traffic_names.size(); ++kind) + text += std::string(kind ? ", " : " ") + traffic_names[kind] + " " + number(rate(c, true, kind)) + " / " + number(rate(c, false, kind)); + const auto snapshots = whole ? c.last.snapshots : c.total.snapshots - c.mark.snapshots; + return text + "; " + number(static_cast(snapshots) / seconds) + " whole states a second"; + }; + if (!player.empty()) { + const auto wanted = lower(player); + for (const auto &link : links) { + const auto *guest = find(link.id); + if (std::to_string(link.id) == player || (guest && lower(guest_name(*guest)).starts_with(wanted))) + return row(link).substr(2) + (guest ? "\n" + by_kind(guest->traffic) : std::string{}); + } + return "No connected player matches \"" + std::string(player) + "\"."; + } + float out{}, in{}; + std::uint64_t queued{}, longest_queue{}; + std::size_t waiting{}, poor{}; + int worst_ping{}; + for (const auto &link : links) { + if (!link.measured) continue; + out += link.out_bps; + in += link.in_bps; + queued += static_cast(std::max(0, link.pending_bytes)); + longest_queue = std::max(longest_queue, link.queue_us); + waiting += link.queue_us > 50000; + poor += (link.quality_local >= 0 && link.quality_local < .9f) || (link.quality_remote >= 0 && link.quality_remote < .9f); + worst_ping = std::max(worst_ping, link.ping_ms); + } + std::string text = std::to_string(players()) + " players (" + std::to_string(links.size()) + " connections), " + std::to_string(config_.tps) + + " TPS | " + kb(out) + " KB/s out, " + kb(in) + " KB/s in | worst ping " + std::to_string(worst_ping) + " ms | " + + std::to_string(queued / 1024) + " KB queued, longest wait " + std::to_string(longest_queue / 1000) + " ms, " + + std::to_string(waiting) + " waiting over 50 ms | " + std::to_string(poor) + " under 90% quality"; + if (!console) return text; + text += "\nloop: " + loop_report(); + text += "\n" + by_kind(traffic_); + { + // Who uploads most, and what most of it is: one player's outfit or objects can cost + // every other player's connection. + struct Upload { const Guest *guest; double total; std::size_t most; }; + std::vector uploads; + for (const auto &[id, guest] : guests_) { + Upload upload{guest.get(), 0, 0}; + for (std::size_t kind = 0; kind < traffic_names.size(); ++kind) { + upload.total += rate(guest->traffic, false, kind); + if (rate(guest->traffic, false, kind) > rate(guest->traffic, false, upload.most)) upload.most = kind; + } + uploads.push_back(upload); + } + std::stable_sort(uploads.begin(), uploads.end(), [](const auto &a, const auto &b) { return a.total > b.total; }); + if (!uploads.empty()) text += "\nmost uploaded:"; + for (std::size_t i = 0; i < std::min(uploads.size(), 6); ++i) + text += std::string(i ? ", " : " ") + guest_name(*uploads[i].guest) + " " + number(uploads[i].total) + " KB/s (" + + traffic_names[uploads[i].most] + " " + number(rate(uploads[i].guest->traffic, false, uploads[i].most)) + ")"; + } + text += "\nsends: " + std::to_string(net.send_failures) + " failed, " + std::to_string(net.skipped) + " skipped, " + + std::to_string(net.dropped) + " dropped (since the server started)"; + // The connections doing worst first: a long queue, poor delivery, silence, then ping. + auto order = links; + const auto badness = [&](const multiplayer::TransportLink &link) { + const auto *guest = find(link.id); + const double silent = guest && guest->handshaken && now_ > guest->last_packet ? static_cast(now_ - guest->last_packet) / 1000.0 : 0; + const double lost = 1.0 - std::min(link.quality_local < 0 ? 1.f : link.quality_local, link.quality_remote < 0 ? 1.f : link.quality_remote); + return static_cast(link.queue_us) / 1000.0 + lost * 2000.0 + silent + link.ping_ms; + }; + std::stable_sort(order.begin(), order.end(), [&](const auto &a, const auto &b) { return badness(a) > badness(b); }); + const std::size_t shown = std::min(order.size(), 12); + if (shown) text += "\nworst " + std::to_string(shown) + " of " + std::to_string(order.size()) + " (net for one):"; + for (std::size_t i = 0; i < shown; ++i) text += "\n" + row(order[i]); + return text; +} + // ---- Tick ------------------------------------------------------------------------------------ +namespace { +std::uint64_t clock_us() { + return static_cast( + std::chrono::duration_cast(std::chrono::steady_clock::now().time_since_epoch()).count()); +} +} // namespace +std::string Host::loop_report() const { + // The last whole minute when there is one, or the minute so far. + const auto &l = last_loop_.passes ? last_loop_ : loop_; + const auto seconds = std::max(1, last_loop_.passes ? 60 : (clock_us() - l.since) / 1000000); + const auto ms = [](std::uint64_t us) { + char text[32]; + std::snprintf(text, sizeof(text), "%.1f", static_cast(us) / 1000.0); + return std::string(text); + }; + return std::to_string(l.passes / seconds) + " passes/s, " + + std::to_string(l.passes ? l.busy_us * 100 / (seconds * 1000000) : 0) + "% busy, longest pass " + ms(l.longest_pass_us) + + " ms, longest gap " + ms(l.longest_gap_us) + " ms (worst since start: " + ms(worst_loop_.longest_pass_us) + " ms pass, " + + ms(worst_loop_.longest_gap_us) + " ms gap)"; +} void Host::tick(std::uint64_t now) { if (!running_) return; now_ = now; + // Timed on a clock of its own, from the start of one pass to the start of the next. + struct Timed { + Host &host; + std::uint64_t started = clock_us(); + explicit Timed(Host &h) : host(h) { + auto &l = host.loop_; + if (!l.since) l.since = started; + if (host.pass_started_) { + const auto gap = started - host.pass_started_; + l.longest_gap_us = std::max(l.longest_gap_us, gap); + host.worst_loop_.longest_gap_us = std::max(host.worst_loop_.longest_gap_us, gap); + } + host.pass_started_ = started; + } + ~Timed() { + auto &l = host.loop_; + const auto took = clock_us() - started; + ++l.passes; + l.busy_us += took; + l.longest_pass_us = std::max(l.longest_pass_us, took); + host.worst_loop_.longest_pass_us = std::max(host.worst_loop_.longest_pass_us, took); + if (started - l.since >= 60000000) { + host.last_loop_ = l; + l = {}; + } + } + } timed{*this}; transport_.poll(); const auto links = transport_.status().peers; const auto linked = [&](std::uint64_t id) { @@ -762,7 +1035,11 @@ void Host::tick(std::uint64_t now) { }; for (auto it = guests_.begin(); it != guests_.end();) { const auto id = (it++)->first; - if (!linked(id)) drop(id, "Disconnected."); + if (linked(id)) continue; + // Steam closed it: the log says how (the player's game closing it, or the connection + // failing at one end or the other), which "Disconnected." alone does not. + const auto how = transport_.take_closed(id); + drop(id, "Disconnected.", how); } for (const auto &link : links) { if (kicked_.contains(link.id)) { @@ -784,11 +1061,15 @@ void Host::tick(std::uint64_t now) { continue; } if (!guest) { - if (!individual_steam_id(link.id) || guests_.size() >= config_.max_players) { - transport_.disconnect(link.id, "The server is full."); + if (!individual_steam_id(link.id) || !may_join(config_, link.id, guests_.size())) { + // Full for them: for everyone, or with only the reserved slots left. + transport_.disconnect(link.id, guests_.size() >= config_.max_players || !individual_steam_id(link.id) + ? "The server is full." + : "The server is full: the slots that are left are reserved."); continue; } auto created = std::make_unique(); + created->sender.set_refresh(8000000, true); created->member.id = link.id; created->last_packet = now_; guest = created.get(); @@ -798,6 +1079,7 @@ void Host::tick(std::uint64_t now) { } for (const auto &message : transport_.receive()) receive(message.peer, message.bytes, message.arrived); receive_cosmetics(); + introduce(); for (auto it = guests_.begin(); it != guests_.end();) { auto &g = *(it++)->second; // Authorized arrivals get time for loading; this deadline is absolute, so @@ -807,11 +1089,63 @@ void Host::tick(std::uint64_t now) { if (now_ - g.travel_since > 180000000) drop(g.member.id, "Could not finish loading the new map."); continue; } + // The connection is still open but nothing came over it: what Steam measures of it says + // whether the link went bad or the player's game just stopped sending. if ((!g.handshaken && g.connected_at && now_ - g.connected_at > handshake_timeout) || (g.handshaken && now_ - g.last_packet > 10000000)) - drop(g.member.id, "Timed out waiting for gameplay data."); + drop(g.member.id, "Timed out waiting for gameplay data.", + "nothing for " + std::to_string((now_ - g.last_packet) / 1000000) + " s; " + transport_.link_report(g.member.id)); + } + // Every half minute, what was sent and received in it (net). + if (now_ >= traffic_mark_ + 30000000) { + const auto roll = [](Counted &c) { + for (std::size_t i = 0; i < c.total.out.size(); ++i) { + c.last.out[i] = c.total.out[i] - c.mark.out[i]; + c.last.in[i] = c.total.in[i] - c.mark.in[i]; + } + c.last.snapshots = c.total.snapshots - c.mark.snapshots; + c.mark = c.total; + }; + roll(traffic_); + for (auto &[id, guest] : guests_) roll(guest->traffic); + traffic_window_us_ = traffic_mark_ ? now_ - traffic_mark_ : 0; + traffic_mark_ = now_; + } + // In a crowd, how far the full and half pose rates reach for each player (crowd_limits). + if (now_ >= next_crowd_) { + next_crowd_ = now_ + 500000; + std::vector squared; + for (auto &[id, guest] : guests_) { + auto &g = *guest; + g.crowd = {}; + if (!g.latest_root || !config_.crowd_budget || guests_.size() * config_.tps <= config_.crowd_budget) continue; + squared.clear(); + for (const auto &[other_id, other] : guests_) { + if (other_id == id || !other->handshaken || !other->world_ready) continue; + float distance{}; + if (other->latest_root) + for (unsigned i = 0; i < 3; ++i) { + const auto d = g.latest_root->position[i] - other->latest_root->position[i]; + distance += d * d; + } + squared.push_back(distance); // not placed yet: counted as beside them + } + g.crowd = crowd_limits(squared, config_.tps, config_.crowd_budget); + } } tick_parties(); + // Once a minute while anyone is on: how the connections are doing as a whole. A queue that + // grows with the player count, or sends that fail, is the server not getting data out. + if (config_.activity_log && now_ >= next_network_log_ && !guests_.empty()) { + next_network_log_ = now_ + 60000000; + const auto &net = transport_.status(); + if (net.telemetry) + log_("[network] " + std::to_string(guests_.size()) + " players, " + std::to_string(static_cast(net.outgoing_bps / 1024.f + .5f)) + + " KB/s out, " + std::to_string(static_cast(net.incoming_bps / 1024.f + .5f)) + " KB/s in, worst ping " + + std::to_string(net.ping_ms) + " ms, " + std::to_string(net.pending_bytes / 1024) + " KB queued (longest " + + std::to_string(net.queue_us / 1000) + " ms), " + std::to_string(net.send_failures) + " failed sends, " + + std::to_string(net.skipped) + " skipped, " + std::to_string(net.dropped) + " dropped | loop " + loop_report()); + } if (roster_dirty_ || now_ - last_roster_ > 2000000) send_roster(); for (auto &[id, guest] : guests_) if (guest->handshaken) { @@ -876,6 +1210,7 @@ std::string Host::command(std::string_view line, std::uint64_t admin) { std::to_string(config_.max_players) + " players | " + std::to_string(config_.tps) + " TPS | voice " + (voice_policy_.allowed ? "on" : "off") + " (" + std::to_string(static_cast(config_.voice_range)) + " m) | password " + (password_ ? "on" : "off") + " | code " + invite(); + if (name == "net") return network_report(argument, console); if (name == "players") { std::string text = std::to_string(players()) + " players"; for (const auto &[id, guest] : guests_) @@ -1123,6 +1458,29 @@ std::string Host::command(std::string_view line, std::uint64_t admin) { return changed(remove ? "Scoring " + scoring_text(*fingerprint) + " is no longer accepted." : "Scoring " + scoring_text(*fingerprint) + " is accepted like the game's own."); } + if (name == "rate") { + // rate : what the server may send each player, from now on and to those on. + const auto value = number(argument); + if (!value || *value < 128 || *value > 16384) + return "rate <128-16384> (KB/s for each player, now " + std::to_string(config_.send_rate) + ")"; + config_.send_rate = static_cast(*value); + const bool applied = transport_.set_send_rate(static_cast(config_.send_rate * 1024)); + return changed("Each player is sent at most " + std::to_string(config_.send_rate) + " KB/s" + + (applied ? "." : ": players who join from now on. Steam did not change the connections already open.")); + } + if (name == "crowd") { + // crowd |off: the most one player is sent (crowd_limits). + auto value = number(argument); + if (argument == "off") value = 0; + if (!value || *value > max_crowd_budget || !valid_crowd_budget(static_cast(*value))) + return "crowd <" + std::to_string(min_crowd_budget) + "-" + std::to_string(max_crowd_budget) + ">|off (now " + + (config_.crowd_budget ? std::to_string(config_.crowd_budget) : std::string("off")) + ")"; + config_.crowd_budget = static_cast(*value); + next_crowd_ = 0; + return changed(*value ? "Each player is sent at most " + std::to_string(*value) + " poses a second: about " + + std::to_string(*value / config_.tps) + " players near them at the full rate." + : std::string("No crowd limit: every player near is sent at the full rate.")); + } if (name == "party-size") { const auto value = number(argument); if (!value || *value < 2 || *value > 8) return "party-size <2-8> (now " + std::to_string(config_.party_size) + ")"; @@ -1387,6 +1745,39 @@ std::string Host::command(std::string_view line, std::uint64_t admin) { return changed(found->label + " set to " + std::string(mode) + (config_.world_layer_sync ? "." : ". Turn on layer-sync to apply it to everyone.")); } + if (name == "reserved") { + // reserved | reserved slots | reserved add|remove + if (!console) return "Only the server console manages reserved slots."; + const auto [sub, who] = split(argument); + if (sub.empty()) { + std::string text = std::to_string(config_.reserved_slots) + " of " + std::to_string(config_.max_players) + + " slots are reserved for " + std::to_string(config_.reserved.size()) + " players and the admins"; + for (const auto id : config_.reserved) { + const auto *guest = find(id); + text += "\n " + std::to_string(id) + (guest ? " " + guest_name(*guest) : std::string{}); + } + return text; + } + if (sub == "slots") { + const auto slots = number(who); + if (!slots || *slots >= config_.max_players) return "reserved slots <0-" + std::to_string(config_.max_players - 1) + ">"; + config_.reserved_slots = static_cast(*slots); + return changed(*slots ? std::to_string(*slots) + " slots are kept for reserved players and admins." + : std::string("No slots are reserved.")); + } + auto *guest = target(who); + const auto id = guest ? guest->member.id : number(who).value_or(0); + if (!individual_steam_id(id) || (sub != "add" && sub != "remove")) return "reserved | reserved slots | reserved add|remove "; + const bool listed = std::find(config_.reserved.begin(), config_.reserved.end(), id) != config_.reserved.end(); + if (sub == "add") { + if (!listed && config_.reserved.size() >= 1024) return "The reserved list is full."; + if (!listed) config_.reserved.push_back(id); + return changed(std::to_string(id) + " has a reserved slot." + + (config_.reserved_slots ? std::string{} : " No slots are reserved yet: reserved slots .")); + } + std::erase(config_.reserved, id); + return changed(std::to_string(id) + " no longer has a reserved slot."); + } if (name == "admins" || name == "admin") { if (!console) return "Only the server console manages admins."; const auto [sub, who] = split(argument); @@ -1461,9 +1852,7 @@ bool Host::check_speed(Guest &guest, std::uint64_t sent) { guest.speed_normal_since = 0; roster_dirty_ = true; send_chat("The server measured your game running faster than normal: throwdowns and challenges are off for you until it's back to normal speed.", &guest); - for (auto &[id, other] : guests_) - if (other->handshaken && is_admin(id) && other.get() != &guest) - send_chat(std::string(text) + " They were taken out of throwdowns and challenges.", other.get()); + // Only the player is told in the game; for the admins it is in the log. return true; } if (!guest.speeding) return true; diff --git a/Server/server_host.h b/Server/server_host.h index 1a60b1b2..a354c9be 100644 --- a/Server/server_host.h +++ b/Server/server_host.h @@ -52,8 +52,18 @@ class Host { return ++messages <= burst; } }; + // What is sent and received, in bytes, by what it carries (traffic_kind): poses, sound, + // voice, outfits, objects, the rest. `last` is the last whole half minute. + struct Traffic { + std::array out{}, in{}; + std::uint64_t snapshots{}; // whole states sent reliably; the rest of a stream is differences + }; + struct Counted { + Traffic total, mark, last; + }; struct Guest { Member member; + Counted traffic; std::uint64_t password_challenge{}; bool handshaken{}, map_authorized{}, world_ready = true; std::uint64_t last_map_offer{}, travel_since{}, connected_at{}, last_packet{}; @@ -71,6 +81,7 @@ class Host { std::optional latest_root; std::uint64_t pose_arrival{}; std::array pose_delivery; + CrowdLimits crowd; // how far the full and half rates reach for them in a crowd std::vector direct_routes; std::uint64_t route_reported{}; std::uint32_t route_sequence{}; @@ -91,6 +102,12 @@ class Host { bool scoring_flagged{}; // out of linked activities (config score_check) ChatBudget scoring_budget; std::uint64_t bans_sent{}; // the ban list revision this admin has + std::uint64_t undelivered_since{}; // Steam has refused what they must be sent since + // The players they have not been shown yet (introduce): on joining, and after a map + // change, a player is sent the others a few a second, nearest first, instead of every + // outfit and every stream at once. Nothing of a player in here is sent to them. + std::set unmet; + std::uint64_t next_introduction{}; bool maps_sent{}; // this player has the server's map list (send_maps) // The owner's own upload, and what the server shares of it. ObjectState objects, shared; @@ -101,6 +118,7 @@ class Host { std::vector chunks; std::uint64_t source{}, epoch{}; std::size_t next{}, cursor{}; + std::uint64_t failing_since{}; // Steam has refused their object updates since } object_delivery; }; @@ -137,6 +155,21 @@ class Host { bool roster_dirty_ = true, running_{}; std::uint64_t bans_revision_ = 1; // bumped whenever the ban list or the admins change std::uint64_t now_{}, last_roster_{}, last_world_state_{}, next_object_update_{}, travel_started_{}; + std::uint64_t next_crowd_{}; // when the crowd limits are worked out again + std::uint64_t next_network_log_{}; // the log's once-a-minute line about the connections + // How the server's own loop is keeping up, for `net` and that line: passes of tick() in a + // minute, the time spent in them, the longest one and the longest wait between two. A long + // pass or gap is the server itself stalling, which every player feels at once. + struct Loop { + std::uint64_t since{}, passes{}, busy_us{}, longest_pass_us{}, longest_gap_us{}; + } loop_, last_loop_, worst_loop_; + std::uint64_t pass_started_{}; + Counted traffic_; + std::uint64_t traffic_mark_{}, traffic_window_us_{}; // when `mark` was taken, and how long `last` covers + void meet_later(Guest &guest); + void introduce(); + std::string loop_report() const; + std::string network_report(std::string_view player, bool console); Guest *find(std::uint64_t id); Packet packet(PacketKind kind, std::uint64_t now); @@ -147,7 +180,8 @@ class Host { bool is_banned(std::uint64_t id) const; void save(); - void drop(std::uint64_t id, const std::string &reason); + // `detail` is for the log alone: the player is told `reason`. + void drop(std::uint64_t id, const std::string &reason, const std::string &detail = {}); bool send_packet(Guest &, const Packet &, bool reliable, bool fresh, std::span raw = {}, std::span wire = {}); void send_required(Guest &, const std::vector &bytes); diff --git a/Server/speed_check.h b/Server/speed_check.h index 18fbace9..9bad51c6 100644 --- a/Server/speed_check.h +++ b/Server/speed_check.h @@ -14,7 +14,9 @@ class SpeedCheck { static constexpr std::uint64_t block_us = 2000000; // one minimum per block static constexpr std::uint64_t window_us = 20000000; // speed measured across this static constexpr std::uint64_t gap_us = 2500000; // longer without poses (loading): start over - static constexpr double limit = 1.06; // a real clock never runs this far ahead + // Half as fast again and more is flagged. A little under 1.5, since a game at exactly 1.5x + // measures a hair either side of it; nothing but a speed hack comes near. + static constexpr double limit = 1.45; static constexpr unsigned strikes_needed = 3; // consecutive fast measurements (one per block) // One pose: the sender's timestamp and when it arrived, both in microseconds. Returns true