Drone: other players' drones are heard where they fly

Each remote drone's motor is synthesised and played through the game's positioned voice sound,
the one proximity chat uses, so it is placed in the world and muffled behind walls. The throttle
is worked out from how the drone accelerates, so the drone packet and protocol stay the same.
Full volume only within 1.5 m, heard out to 120 m; the DRONE page's sound volume sets the level.

Extension/Multiplayer/Remote/drone_sound.cpp

@@ -0,0 +1,144 @@
+#include "drone_sound.h"
+#include "native_prop.h"
+#include "Engine/Core/Log/logging.h"
+#include "Extension/Multiplayer/Voice/native_voice.h"
+#include "Extension/Skater/client_source_spawn.h"
+#include "Extension/Skater/drone_audio.h"
+#include "Extension/Skater/drone_beacon.h"
+#include <algorithm>
+#include <chrono>
+#include <cmath>
+#include <format>
+#include <map>
+#include <mutex>
+#include <vector>
+
+namespace dingosdk::multiplayer {
+namespace {
+using Clock = std::chrono::steady_clock;
+// How far away a drone is heard: a 5-inch at full power carries well past a skate spot.
+constexpr float hearing_distance = 120.0f;
+// Other players' drones a little under the pilot's own motor at the same volume setting.
+constexpr float remote_level = 0.3f;
+// Full volume only this close; from there it falls off with distance (about -6 dB each time the
+// distance doubles), faded to nothing over the last quarter of the hearing distance.
+constexpr float full_volume_distance = 1.5f;
+float drone_gain(const std::array<float, 3> &ear, const std::array<float, 3> &at) {
+    float squared = 0.0f;
+    for (std::size_t i = 0; i < 3; ++i) squared += (ear[i] - at[i]) * (ear[i] - at[i]);
+    const float distance = std::sqrt(squared);
+    if (!std::isfinite(distance) || distance >= hearing_distance) return 0.0f;
+    float gain = full_volume_distance / std::max(distance, full_volume_distance);
+    const float fade_start = hearing_distance * 0.75f;
+    if (distance > fade_start) gain *= 1.0f - (distance - fade_start) / (hearing_distance - fade_start);
+    return gain;
+}
+constexpr std::uint64_t drone_source_tag = 0x52534b4400000000ULL; // "RSKD"
+// The dronemodeltest prop's sound (players are keyed by their nonzero ids).
+constexpr std::uint64_t test_drone = 0;
+// Sound made ahead of time for a drone just heard, so the first blocks do not run dry.
+constexpr double preroll_seconds = 0.06;
+// At most this much sound made in one frame (a long hitch is not caught up).
+constexpr float longest_frame = 0.1f;
+
+struct Voice {
+    drone::audio::MotorSynth synth{static_cast<int>(native_voice_rate)};
+    double owed{}; // samples due and not yet made (the fraction carries to the next frame)
+};
+struct Sounds {
+    NativeVoicePlayback playback{drone_source_tag};
+    std::map<std::uint64_t, Voice> voices;
+    bool started{}, announced{};
+    Clock::time_point last{}, test_started{};
+    std::mutex status_mutex;
+    std::string status = "No other player's drone is in the air.";
+};
+// Never destroyed: its sounds belong to the game, which is gone by then.
+Sounds &sounds() {
+    static auto *s = new Sounds;
+    return *s;
+}
+void set_status(Sounds &s, std::string text) {
+    std::lock_guard lock(s.status_mutex);
+    s.status = std::move(text);
+}
+struct Source {
+    std::uint64_t id;
+    std::array<float, 3> position;
+    float throttle, speed;
+};
+} // namespace
+
+void tick_drone_sounds(std::uintptr_t base, std::uintptr_t client, bool ready, bool show) noexcept {
+    auto &s = sounds();
+    try {
+        const auto now = Clock::now();
+        if (!s.started) {
+            s.playback.reset(1);
+            s.started = true;
+            s.last = now;
+        }
+        const float elapsed = std::clamp(std::chrono::duration<float>(now - s.last).count(), 0.0f, longest_frame);
+        s.last = now;
+        const float volume = drone::knobs().value[drone::kVolume].load(std::memory_order_relaxed);
+        std::vector<Source> sources;
+        std::array<float, 16> camera{};
+        if (ready && volume > 0.001f && read_local_camera_transform(base, client, camera)) {
+            if (show)
+                for (const auto &drone : drone::beacon::snapshot())
+                    sources.push_back({drone.id, drone.position, drone.throttle, drone.speed});
+            if (const auto at = prop_test_position()) {
+                // Going round its circle with the throttle rising and falling, as a drone would.
+                if (!s.voices.contains(test_drone)) s.test_started = now;
+                const float t = std::chrono::duration<float>(now - s.test_started).count();
+                sources.push_back({test_drone, *at, 0.35f + 0.25f * std::sin(t * 1.3f), 3.1f});
+            }
+        }
+        const std::array<float, 3> ear{camera[12], camera[13], camera[14]};
+        std::erase_if(s.voices, [&](const auto &item) {
+            return std::none_of(sources.begin(), sources.end(), [&](const Source &x) { return x.id == item.first; });
+        });
+        std::vector<std::int16_t> samples;
+        for (const auto &source : sources) {
+            const float gain = remote_level * volume * drone_gain(ear, source.position);
+            if (!(gain > 0.0f)) {
+                s.voices.erase(source.id);
+                continue; // its sound is released once it has had no position for a moment
+            }
+            auto [found, added] = s.voices.try_emplace(source.id);
+            auto &voice = found->second;
+            if (added) voice.owed = preroll_seconds * native_voice_rate;
+            voice.owed += static_cast<double>(elapsed) * native_voice_rate;
+            const auto count = static_cast<int>(std::min(std::floor(voice.owed), static_cast<double>(native_voice_rate) * longest_frame));
+            voice.owed = std::max(0.0, voice.owed - count);
+            s.playback.position(source.id, 1, source.position, gain);
+            if (count <= 0) continue;
+            samples.resize(static_cast<std::size_t>(count));
+            voice.synth.render(samples.data(), count, source.throttle, source.speed, 1.0f);
+            s.playback.submit(source.id, 1, {reinterpret_cast<const std::uint8_t *>(samples.data()), samples.size() * 2});
+        }
+        s.playback.process(base);
+        if (!s.voices.empty() && !s.announced && s.playback.available()) {
+            s.announced = true;
+            logging::log(logging::Level::info, logging::Channel::runtime,
+                         "Other players' drones are heard through the game's positioned voice sound.");
+        }
+        if (s.voices.empty())
+            set_status(s, "No other player's drone is in the air within earshot.");
+        else if (!s.playback.available())
+            set_status(s, std::format("{} drone(s) within earshot, but the game's positioned voice sound is not ready, so they are silent.",
+                                      s.voices.size()));
+        else
+            set_status(s, std::format("{} drone(s) heard.", s.voices.size()));
+    } catch (const std::exception &e) {
+        s.voices.clear();
+        set_status(s, std::string("Drone sounds stopped: ") + e.what());
+    }
+}
+
+std::string drone_sounds_status() {
+    auto &s = sounds();
+    std::lock_guard lock(s.status_mutex);
+    return s.status;
+}
+} // namespace dingosdk::multiplayer

Extension/Multiplayer/Remote/drone_sound.h

@@ -0,0 +1,15 @@
+#pragma once
+#include <cstdint>
+#include <string>
+
+// Other players' FPV drones heard where they fly. Each drone's motor is synthesised here
+// (Skater/drone_audio.h, at its estimated throttle and speed: drone_beacon.h) and played through
+// the game's positioned voice sound, the one proximity chat uses (Voice/native_voice.h), so the
+// game places it, muffles it behind walls and gives it the level's reverb.
+namespace dingosdk::multiplayer {
+// The client update thread, every frame (in a level or not: sounds stop when `ready` or `show`
+// goes false). `show`: other players' drones are shown (DRONE page, LOBBY tab). The
+// dronemodeltest prop, while it runs, is heard as a drone too.
+void tick_drone_sounds(std::uintptr_t base, std::uintptr_t client, bool ready, bool show) noexcept;
+std::string drone_sounds_status();
+} // namespace dingosdk::multiplayer

Extension/Multiplayer/Voice/native_voice.cpp

@@ -81,7 +81,7 @@ struct NativeVoicePlayback::Impl {
     std::mutex mutex;
     std::map<std::uint64_t, Request> requests;
     std::map<std::uint64_t, Sound> sounds;
-    std::uint64_t generation{}, serial{};
+    std::uint64_t generation{}, serial{}, source_tag{};
     std::atomic<bool> ready{};
     Address base{}, manager{}, provider{}, asset{};
     DWORD thread{};
@@ -141,7 +141,7 @@ struct NativeVoicePlayback::Impl {
                 if (!request.pcm.samples()) continue;
                 Sound sound;
                 sound.serial = request.serial;
-                sound.source = 0x52534b5600000000ULL | request.serial;
+                sound.source = source_tag | request.serial;
                 if (!create_sound(base, asset, &sound.handle)) throw std::runtime_error("Native voice sound creation failed");
                 found = sounds.emplace(id, sound).first;
                 if (!set_input(base, &found->second.handle, base + voice::source_input_type, &found->second.source))
@@ -168,7 +168,9 @@ struct NativeVoicePlayback::Impl {
         if (failed) throw std::runtime_error("Native voice cleanup failed");
     }
 };
-NativeVoicePlayback::NativeVoicePlayback() : impl_(std::make_unique<Impl>()) {}
+NativeVoicePlayback::NativeVoicePlayback(std::uint64_t source_tag) : impl_(std::make_unique<Impl>()) {
+    impl_->source_tag = source_tag & 0xffffffff00000000ULL;
+}
 NativeVoicePlayback::~NativeVoicePlayback() = default;
 bool NativeVoicePlayback::available() const noexcept { return impl_->ready.load(); }
 void NativeVoicePlayback::reset(std::uint64_t generation) {

Extension/Multiplayer/Voice/native_voice.h

@@ -5,7 +5,10 @@
 namespace dingosdk::multiplayer {
 class NativeVoicePlayback {
 public:
-    NativeVoicePlayback();
+    // Players' voices; other sounds played this way (remote drones) take their own tag, so their
+    // streams never share an id with a voice's in the game's one sample provider.
+    static constexpr std::uint64_t voice_source_tag = 0x52534b5600000000ULL; // "RSKV"
+    explicit NativeVoicePlayback(std::uint64_t source_tag = voice_source_tag);
     ~NativeVoicePlayback();
     bool available() const noexcept;
     void reset(std::uint64_t generation);

Extension/Skater/Test/drone_flight_tests.cpp

@@ -1,6 +1,7 @@
 // The FPV drone's flight model (drone_flight.h): hover and height hold, thrust and top speed,
 // motor and rate lag, angle mode, crashes against a wall and landing on a floor, and a gamepad's
 // sticks. Flown in a world of one floor (y = 0) and one wall (x = 20).
+#include "Extension/Skater/drone_beacon.h"
 #include "Extension/Skater/drone_flight.h"
 #include <cstdio>
 using namespace dingosdk::drone;
@@ -223,6 +224,36 @@ int main() {
                   "without height hold the throttle does not change with the lean");
         }
     }
+    // Another player's drone, heard: the throttle it seems to be on, from how it moves.
+    {
+        namespace beacon = dingosdk::drone::beacon;
+        const float g = beacon::assumed_gravity, hover = beacon::assumed_hover;
+        CHECK(std::abs(beacon::estimate_throttle({0, 0, 0}, beacon::no_rotation) - hover) < 1e-4f, "a hovering drone is on its hover throttle");
+        CHECK(beacon::estimate_throttle({0, -g, 0}, beacon::no_rotation) < 1e-4f, "a falling drone is off the throttle");
+        CHECK(std::abs(beacon::estimate_throttle({0, g, 0}, beacon::no_rotation) - hover * std::pow(2.0f, 2.0f / 3.0f)) < 1e-3f,
+              "climbing at 1 g takes twice the hover thrust");
+        // Leant 45 degrees and holding its height: the thrust is 1/cos(45) of its weight.
+        const float half = 0.3926991f;
+        const beacon::Rotation leant{std::sin(half), 0, 0, std::cos(half)};
+        const auto up = beacon::basis_from_rotation(leant)[1];
+        const beacon::Position level{up[0] / up[1] * g, 0, up[2] / up[1] * g};
+        CHECK(std::abs(beacon::estimate_throttle(level, leant) - hover * std::pow(1.0f / std::cos(2 * half), 2.0f / 3.0f)) < 1e-3f,
+              "a leant drone holding its height is on more than its hover throttle");
+        // Through packets: still for a while, then climbing at 1 g.
+        const auto start = beacon::Clock::now();
+        const auto at_time = [&](int n) { return start + std::chrono::milliseconds(66 * n); };
+        int n = 0;
+        for (; n < 10; ++n) beacon::receive(7, "pilot", {0, 5, 0}, beacon::no_rotation, true, at_time(n));
+        auto heard = beacon::snapshot(at_time(n - 1));
+        CHECK(heard.size() == 1 && std::abs(heard[0].throttle - hover) < 0.01f && heard[0].speed < 0.01f, "a drone keeping still sounds like a hover");
+        for (int k = 0; k < 15; ++k, ++n) {
+            const float t = 0.066f * k;
+            beacon::receive(7, "pilot", {0, 5 + 0.5f * g * t * t, 0}, beacon::no_rotation, true, at_time(n));
+        }
+        heard = beacon::snapshot(at_time(n - 1));
+        CHECK(heard.size() == 1 && heard[0].throttle > 0.5f && heard[0].speed > 5.0f, "a drone climbing hard sounds busier and faster");
+        beacon::forget(7);
+    }
     if (fails) return 1;
     std::puts("Drone flight tests passed.");
     return 0;

Extension/Skater/console_commands.cpp

@@ -4,6 +4,7 @@
 #include "drone_flight.h"
 #include "drone_outfit.h"
 #include "Extension/Multiplayer/Remote/native_prop.h"
+#include "Extension/Multiplayer/Remote/drone_sound.h"
 #include <algorithm>
 #include <cstdlib>
 #include <format>
@@ -191,6 +192,7 @@ void register_movement_commands(Commands &registry) {
         if (name.empty() || name == "status") {
             out(multiplayer::prop_test_status());
             out("Other players' drones: " + multiplayer::drone_models_status());
+            out("Their sound: " + multiplayer::drone_sounds_status());
             return;
         }
         if (name == "stop") {

Extension/Skater/drone_audio.h

@@ -16,7 +16,11 @@ namespace dingosdk::drone::audio {
 
 class MotorSynth {
 public:
-    static constexpr int rate = 44100;
+    static constexpr int rate = 44100; // the pilot's own sound; other players' drones play at 16 kHz
+    explicit MotorSynth(int sample_rate = rate)
+        : rate_(sample_rate > 0 ? sample_rate : rate),
+          // the wind's low-pass at about 900 Hz whatever the rate (0.12 a sample at 44.1 kHz)
+          wind_k_(1.0f - std::exp(-6.2831853f * 900.0f / static_cast<float>(rate_))) {}
     // Fills `count` mono 16-bit samples. throttle 0..1, speed in m/s, volume 0..1; pitch scales the
     // motor note (1 = normal; small drones spin faster and sound higher).
     void render(std::int16_t* out, int count, float throttle, float speed, float volume, float pitch = 1.0f) {
@@ -25,7 +29,7 @@ public:
         const float vol = std::isfinite(volume) ? std::clamp(volume, 0.0f, 1.0f) : 0.0f;
         const float wind = std::isfinite(speed) ? std::clamp(speed / 60.0f, 0.0f, 1.0f) : 0.0f;
         constexpr float tau = 0.08f;                         // motors spin up/down over ~80 ms
-        const float k = 1.0f - std::exp(-1.0f / (tau * rate));
+        const float k = 1.0f - std::exp(-1.0f / (tau * static_cast<float>(rate_)));
         constexpr float detune[4] = {1.0f, 1.013f, 0.989f, 1.027f};
         constexpr float tpi = 6.283185307179586f;
         for (int i = 0; i < count; ++i) {
@@ -33,7 +37,7 @@ public:
             const float base = (110.0f + 520.0f * rpm_) * note; // fundamental, Hz
             float sample = 0.0f;
             for (int m = 0; m < 4; ++m) {
-                phase_[m] += base * detune[m] / rate;
+                phase_[m] += base * detune[m] / static_cast<float>(rate_);
                 phase_[m] -= std::floor(phase_[m]);
                 const float p = phase_[m] * tpi;
                 sample += std::sin(p) + 0.55f * std::sin(2 * p) + 0.35f * std::sin(3 * p) + 0.2f * std::sin(5 * p);
@@ -41,7 +45,7 @@ public:
             // cheap noise, low-passed more for the wind, brighter for the props
             seed_ = seed_ * 1664525u + 1013904223u;
             const float white = static_cast<float>(static_cast<std::int32_t>(seed_)) / 2147483648.0f;
-            lp_ += (white - lp_) * 0.12f;
+            lp_ += (white - lp_) * wind_k_;
             const float noise = (0.35f * rpm_ + 0.5f * wind) * lp_ + 0.12f * rpm_ * white;
             const float level = 0.03f + 0.07f * rpm_;         // idle whine is quieter than full power
             const float v = std::tanh((sample * level + noise * 0.25f) * 1.6f) * 0.85f * vol;
@@ -50,6 +54,8 @@ public:
     }
     float rpm() const { return rpm_; }
 private:
+    int rate_;
+    float wind_k_;
     float rpm_ = 0, lp_ = 0;
     float phase_[4] = {0, 0.25f, 0.5f, 0.75f};
     std::uint32_t seed_ = 12345u;

Extension/Skater/drone_beacon.h

@@ -140,7 +140,23 @@ struct Remote {
     Clock::duration glide{};      // how long the glide takes
     Rotation rotation = no_rotation;                     // in a snapshot: which way it faces now
     Rotation from_rotation = no_rotation, to_rotation = no_rotation;
+    // For its motor sound (Multiplayer/Remote/drone_sound.h): how fast it goes and the throttle it
+    // seems to be on, worked out from how it moves (estimate_throttle), as packets carry no stick.
+    Position velocity{};
+    float speed = 0.0f;
+    float throttle = 0.0f;
 };
+// The throttle a drone is on, from how it accelerates and which way its props point: thrust is the
+// acceleration it has besides gravity, along its up axis. Thrust goes with output^1.5 and hovering
+// takes `hover` (drone_flight.h), so output = hover * (thrust / g)^(2/3). Drag is left out: a fast
+// drone sounds a little less busy than it is.
+inline constexpr float assumed_hover = 0.35f, assumed_gravity = 9.81f;
+inline float estimate_throttle(const Position& acceleration, const Rotation& facing) {
+    const auto up = basis_from_rotation(facing)[1];
+    const float thrust = acceleration[0] * up[0] + (acceleration[1] + assumed_gravity) * up[1] + acceleration[2] * up[2];
+    if (!std::isfinite(thrust) || thrust <= 0.0f) return 0.0f;
+    return std::clamp(assumed_hover * std::pow(thrust / assumed_gravity, 2.0f / 3.0f), 0.0f, 1.0f);
+}
 inline float glide_fraction(const Remote& r, Clock::time_point now) {
     if (r.glide <= Clock::duration::zero()) return 1.0f;
     return std::clamp(std::chrono::duration<float>(now - r.seen) / std::chrono::duration<float>(r.glide), 0.0f, 1.0f);
@@ -176,6 +192,26 @@ inline void receive(std::uint64_t id, std::string name, const Position& position
             const auto gap = std::clamp<Clock::duration>(now - it->seen, std::chrono::milliseconds(20),
                                                          std::chrono::milliseconds(250));
             const bool snap = jump > snap_distance * snap_distance;
+            // Velocity and acceleration between the last two positions as they arrived; smoothed,
+            // as arrival times jitter. A long gap or a jump starts over.
+            const float dt = std::chrono::duration<float>(now - it->seen).count();
+            if (!snap && dt >= 0.02f && dt <= 0.5f) {
+                Position velocity{}, acceleration{};
+                float speed = 0.0f;
+                for (std::size_t i = 0; i < 3; ++i) {
+                    velocity[i] = (position[i] - it->to[i]) / dt;
+                    acceleration[i] = (velocity[i] - it->velocity[i]) / dt;
+                    speed += velocity[i] * velocity[i];
+                }
+                const float throttle = estimate_throttle(acceleration, facing);
+                it->throttle += (throttle - it->throttle) * 0.35f;
+                it->speed += (std::sqrt(speed) - it->speed) * 0.5f;
+                it->velocity = velocity;
+            } else {
+                it->velocity = {};
+                it->speed = 0.0f;
+                it->throttle = assumed_hover;
+            }
             it->from_rotation = snap ? facing : blend_rotation(it->from_rotation, it->to_rotation, glide_fraction(*it, now));
             it->to_rotation = facing;
             it->from = snap ? position : shown;
@@ -186,7 +222,9 @@ inline void receive(std::uint64_t id, std::string name, const Position& position
             return;
         }
     if (!active || r.list.size() >= max_remote) return;
-    r.list.push_back({id, std::move(name), position, now, position, position, {}, facing, facing, facing});
+    Remote added{id, std::move(name), position, now, position, position, {}, facing, facing, facing};
+    added.throttle = assumed_hover;
+    r.list.push_back(std::move(added));
 }
 // Someone left the session.
 inline void forget(std::uint64_t id) {

Runtime/client_tick.cpp

@@ -24,6 +24,7 @@
 #include "Extension/Multiplayer/Hud/native_party.h"
 #include "Extension/Multiplayer/Session/session.h"
 #include "Extension/Multiplayer/Remote/native_prop.h"
+#include "Extension/Multiplayer/Remote/drone_sound.h"
 #include "Extension/Boot/discord_presence.h"
 #include "Engine/Game/World/world_names.h"
 #include "Extension/Multiplayer/Hud/custom_nametags.h"
@@ -1291,6 +1292,9 @@ void tick(std::uintptr_t client, std::uintptr_t update) {
                 dingosdk::multiplayer::tick_drone_models(r.base, client,
                     dingosdk::drone::knobs().value[dingosdk::drone::kShowBeacons].load(std::memory_order_relaxed) >= 0.5f);
             }
+            // And heard where they fly (every frame, so their sounds stop on leaving a level).
+            dingosdk::multiplayer::tick_drone_sounds(r.base, client, multiplayer_ready,
+                dingosdk::drone::knobs().value[dingosdk::drone::kShowBeacons].load(std::memory_order_relaxed) >= 0.5f);
             if (auto notice = dingosdk::multiplayer::take_leave_notice(); !notice.empty())
                 dingosdk::overlay::notify(dingosdk::overlay::NoticeLevel::warning, "Map not installed", std::move(notice));
             // Kicked, banned or otherwise put out by the host or server: said in the middle of

cmake/Runtime.cmake

@@ -33,6 +33,7 @@ add_library(dingosdk_runtime SHARED
     Extension/Multiplayer/Remote/native_skater.cpp
     Extension/Multiplayer/Remote/native_skater_spawn.cpp
     Extension/Multiplayer/Remote/native_prop.cpp
+    Extension/Multiplayer/Remote/drone_sound.cpp
     Extension/Multiplayer/Remote/puppet_cost.cpp
     Extension/Multiplayer/Remote/native_cosmetics.cpp
     Extension/Multiplayer/Remote/native_audio.cpp