Compare commits

4 Commits
Author SHA1 Message Date
John 12bb423be1 Update remote play and voice chat status to working 2026-10-10 21:43:00 +02:00
JBuildr 12dc2ed2ea Nubix 0.4.0: headset voice chat 2026-10-09 18:23:06 +02:00
JBuildr abba3149bd README: showcase video 2026-10-07 21:04:00 +02:00
JBuildr 76d1caf532 README: header banner 2026-10-07 19:56:04 +02:00
26 changed files with 3434 additions and 88 deletions

No files matched your search

Binary file not shown.

After

Width:  |  Height:  |  Size: 251 KiB

+5 -4
View File
@@ -5,7 +5,7 @@
# Host: scripts/build-host.sh (or: cmake -S . -B build-host && cmake --build build-host)
# PS5: scripts/build-ps5.sh (inside docker "ps5dev" with the prospero toolchain)
cmake_minimum_required(VERSION 3.20)
project(nubix VERSION 0.3.0 LANGUAGES C CXX)
project(nubix VERSION 0.4.0 LANGUAGES C CXX)
set(CMAKE_CXX_STANDARD 17)
set(CMAKE_CXX_STANDARD_REQUIRED ON)
@@ -63,9 +63,10 @@ set(XC_PLATFORM_SOURCES
src/platform/gamepad.cpp
src/platform/net_compat.c)
if(XC_PS5)
list(APPEND XC_PLATFORM_SOURCES src/platform/ps5.cpp src/platform/ps5_shims.c)
# audio_in_ps5.cpp resolves libSceAudioIn at run time (dlopen), so no extra link library.
list(APPEND XC_PLATFORM_SOURCES src/platform/ps5.cpp src/platform/ps5_shims.c src/platform/audio_in_ps5.cpp)
else()
list(APPEND XC_PLATFORM_SOURCES src/platform/host.cpp)
list(APPEND XC_PLATFORM_SOURCES src/platform/host.cpp src/platform/audio_in_host.cpp)
endif()
add_library(xc_platform STATIC ${XC_PLATFORM_SOURCES})
target_link_libraries(xc_platform PUBLIC xc_core xc::sdl2)
@@ -125,7 +126,7 @@ if(NOT XC_PS5 AND XC_BUILD_TESTS)
file(GLOB XC_TEST_SOURCES CONFIGURE_DEPENDS "${CMAKE_SOURCE_DIR}/tests/*.cpp")
add_executable(xc-tests ${XC_TEST_SOURCES})
target_link_libraries(xc-tests PRIVATE xc_ui)
foreach(suite smoke protocol sdp rtp config ratelimit net)
foreach(suite smoke protocol sdp rtp config ratelimit net voice voice_sdp)
add_test(NAME ${suite} COMMAND xc-tests ${suite})
endforeach()
endif()
+48 -6
View File
@@ -1,6 +1,6 @@
# Nubix
**Play Xbox Cloud Gaming online on your jailbroken PS5.**
<p align="center">
<img src=".github/assets/header.png" alt="Nubix - Play Xbox Cloud Gaming online on your jailbroken PS5" width="100%">
</p>
An **unofficial**, open-source client for **Xbox Cloud Gaming** on jailbroken PS5 consoles.
It runs as a homebrew app (ELF payload) started from the
@@ -12,6 +12,16 @@ to your TV with a DualSense as the controller.
> Sony Interactive Entertainment. They are used here only to describe what this software works with.
> Read [Disclaimer](#disclaimer) before you sign in.
## Showcase
<p align="center">
<a href="https://www.youtube.com/watch?v=N1xeoeBogxQ">
<img src="https://img.youtube.com/vi/N1xeoeBogxQ/maxresdefault.jpg" alt="Nubix showcase video on YouTube" width="720">
</a>
<br>
<sub>▶️ <a href="https://www.youtube.com/watch?v=N1xeoeBogxQ">Watch the showcase on YouTube</a>: sign-in, cloud library and gameplay on a jailbroken PS5.</sub>
</p>
## Status
| Area | State |
@@ -20,7 +30,8 @@ to your TV with a DualSense as the controller.
| Cloud library (Game Pass, owned, free-to-play), filters, box art | ✅ works on PS5 |
| Queue / waiting for a server | ✅ works on PS5 |
| Cloud streaming (video, audio, controller, rumble) | ✅ works on PS5 |
| Remote play of your own Xbox console (xHome) | ⚠️ implemented, not tested |
| Remote play of your own Xbox console (xHome) | ✅ works on PS5 |
| Voice chat (headset mic, party/game chat) | ✅ works on PS5 |
| Native PS5 title / home-screen tile | ❌ not supported (start it from the websrv launcher) |
Video is decoded in software (FFmpeg). If the picture stutters, set the resolution to 720p.
@@ -92,7 +103,7 @@ load your library or stream.
## Installation
1. Download `nubix-v<version>-websrv.zip` (e.g. `nubix-v0.3.0-websrv.zip`) from the releases page.
1. Download `nubix-v<version>-websrv.zip` (e.g. `nubix-v0.4.0-websrv.zip`) from the releases page.
2. Unzip it and copy the `nubix` folder to `/data/homebrew/` on the PS5 (FTP, or a USB drive
under `/mnt/usb0/homebrew/`). You should end up with
`/data/homebrew/nubix/{eboot.elf, homebrew.js, sce_sys/icon0.png, assets/}`.
@@ -121,7 +132,7 @@ The PS5 system keeps the PS and Create buttons to itself, so the app never sees
| L1 / R1, L2 / R2, L3 / R3 | LB / RB, LT / RT, LS / RS |
| Options | Menu |
| Touchpad click | View |
| Hold **Options + touchpad** for 1 s | Stream menu (resume, Xbox button, statistics, refresh video, disconnect) |
| Hold **Options + touchpad** for 1 s | Stream menu (resume, Xbox button, mute/unmute microphone, statistics, refresh video, disconnect) |
In the menus: ✕ = select, ○ = back, L1/R1 = switch tabs, △ = filter.
@@ -131,8 +142,29 @@ In the menus: ✕ = select, ○ = back, L1/R1 = switch tabs, △ = filter.
- **Maximum bitrate**: 5–20 Mbps. Lower it on Wi-Fi or if the picture stutters.
- **Server region**: automatic or one of the regions Xbox offers your account.
- **Free-to-play fallback**: retry with the ad-supported offering when a title is not in your plan.
- **Voice chat**: on (default) or off, see [Voice chat](#voice-chat). Applies to the next stream.
- **Extra stick deadzone**, **stream statistics**, **sign out**.
## Voice chat
Talk and listen in Xbox party chat and in-game chat while you stream:
- Plug a headset into the DualSense (or use a USB headset), or use the DualSense's built-in
microphone. Your voice goes to the streamed console; other players' voices play together with
the game audio, on the headset when one is connected.
- When a stream starts, a toast says the microphone is on, and a small microphone badge appears in
the top-right corner (green level bar = live, red slash = muted).
- **Mute / unmute** in the stream menu (hold Options + touchpad for 1 s → *Mute microphone*). Muting
sends silence and stays in effect for later streams until you unmute or restart Nubix.
- **Settings → Voice chat → Off** turns the feature off completely (no microphone use, the stream is
set up exactly as before). Use it if a stream ever fails to start with voice chat on.
- Without a headset, the DualSense microphone may pick up the TV and other players may hear an
echo of the game: use a headset, or mute the microphone.
- No microphone found: the stream menu shows *Microphone unavailable* and the stream runs normally.
- Party or game chat works when the party belongs to the streamed session, i.e. you create or join
it from the Xbox guide inside the stream. **Known limitation:** a party that exists only in the
Xbox app or on a phone, without the streamed console in it, is not heard (not supported yet).
## Network behaviour
Nubix talks to the same Microsoft services as the Xbox web player, and paces itself
@@ -151,6 +183,13 @@ Nubix talks to the same Microsoft services as the Xbox web player, and paces its
- Sign-in, library or box art fail only with the protective DNS: start `tools/sony_dns.py --log-all`
and look for BLOCK lines while Nubix loads; please report them so the built-in list can be fixed.
- Close other running homebrew before starting Nubix (websrv can crash otherwise).
- Voice chat: the log shows every step under `voice:` (microphone open attempts, `voice: answer
chat=… chatStream=…`, renegotiation, `first mic RTP sent`). The statistics overlay shows
`mic <state> <kbps> | chat rx <packets>`. If a stream does not start with voice chat on, set
*Settings → Voice chat* to Off and please report the log.
- Voice chat and game sound play on the TV instead of the headset: on the PS5 set *Settings → Sound →
Audio Output → Output to Headphones* to *All Audio*. (Nubix plays everything through one output;
a separate voice-only output to the headset is a possible later addition.)
## Privacy and security
@@ -159,6 +198,9 @@ Nubix talks to the same Microsoft services as the Xbox web player, and paces its
file access to the console (for example an open FTP payload on your network) can copy it. Do not
leave FTP running on untrusted networks. *Settings → Sign out* deletes it; you can also sign out of
all devices in your Microsoft account's security settings.
- With voice chat on, your microphone audio is sent to the streamed Xbox session (and from there to
the people you chat with) only while a stream is running. Mute it in the stream menu or turn voice
chat off in Settings.
- The app only contacts Microsoft services (sign-in, Xbox Live, cloud gaming, Microsoft Store catalog)
plus Google's public STUN server (`stun.l.google.com`) to find its public address for WebRTC.
+2
View File
@@ -144,6 +144,7 @@ bool Config::load() {
readStr(s, "locale", settings.locale);
readBool(s, "f2pFallback", settings.f2pFallback);
readInt(s, "stickDeadzone", settings.stickDeadzone);
readBool(s, "voiceChat", settings.voiceChat);
if (clampSettings(settings)) XC_LOGW("config: out-of-range settings were reset");
}
auto tIt = root.find("tokens");
@@ -259,6 +260,7 @@ bool Config::save() {
{"resolution", settings.resolution}, {"bitrateKbps", settings.bitrateKbps},
{"region", settings.region}, {"locale", settings.locale},
{"f2pFallback", settings.f2pFallback}, {"stickDeadzone", settings.stickDeadzone},
{"voiceChat", settings.voiceChat},
};
root["tokens"] = {
{"msaRefresh", tokens.msaRefresh},
+1
View File
@@ -19,6 +19,7 @@ struct Settings {
std::string locale = "en-US"; // used for catalog + session start
bool f2pFallback = true; // retry cloud start with the F2P offering on OfferingDoesNotContainTitle
int stickDeadzone = 0; // extra radial deadzone in percent (0 = rely on server)
bool voiceChat = true; // negotiate in-stream voice chat + capture the headset/controller mic
};
// One gssv region of an offering, from /v2/login/user offeringSettings.regions[].
+48 -14
View File
@@ -408,9 +408,9 @@ int GssvClient::waitTimeSeconds(const std::string& titleId) {
return -1;
}
bool GssvClient::sendSdpOffer(const SessionInfo& s, const std::string& sdp, std::string& err) {
HttpResponse r =
Http::request("POST", sessionUrl(s, "sdp"), headers(), proto::sdpPostBody(sdp), kRequestTimeoutSec);
bool GssvClient::sendSdpOffer(const SessionInfo& s, const std::string& sdp, std::string& err, bool chatStream) {
HttpResponse r = Http::request("POST", sessionUrl(s, "sdp"), headers(), proto::sdpPostBody(sdp, chatStream),
kRequestTimeoutSec);
noteResponse(r);
if (!r.ok()) {
err = describeFailure("sdp offer", r);
@@ -419,8 +419,33 @@ bool GssvClient::sendSdpOffer(const SessionInfo& s, const std::string& sdp, std:
return true;
}
bool GssvClient::sendSdpRenegotiation(const SessionInfo& s, const std::string& sdp, std::string& err) {
HttpResponse r = Http::request("POST", sessionUrl(s, "sdp"), headers(), proto::sdpChatRenegotiationBody(sdp),
kRequestTimeoutSec);
noteResponse(r);
if (!r.ok()) {
err = describeFailure("sdp renegotiation", r);
return false;
}
if (!isPending(r) && !r.body.empty()) {
json parsed = json::parse(r.body, nullptr, false);
if (!parsed.is_discarded() && parsed.is_object() && isRealExchangeError(parsed)) {
err = "sdp renegotiation: " + snippet(parsed.dump(-1, ' ', false, json::error_handler_t::replace), 400);
return false;
}
}
return true;
}
bool GssvClient::pollSdpAnswer(const SessionInfo& s, std::string& answerSdp, bool& ready, std::string& err) {
answerSdp.clear();
SdpAnswerInfo info;
const bool ok = pollSdpAnswer(s, info, ready, err);
answerSdp = ready ? std::move(info.sdp) : std::string();
return ok;
}
bool GssvClient::pollSdpAnswer(const SessionInfo& s, SdpAnswerInfo& out, bool& ready, std::string& err) {
out = SdpAnswerInfo{};
ready = false;
HttpResponse r = Http::request("GET", sessionUrl(s, "sdp"), headers(), "", kRequestTimeoutSec);
noteResponse(r);
@@ -436,27 +461,36 @@ bool GssvClient::pollSdpAnswer(const SessionInfo& s, std::string& answerSdp, boo
}
if (isRealExchangeError(parsed)) {
err = "sdp exchange: " + snippet(parsed.dump(), 400);
out.rejected = true;
return false;
}
// exchangeResponse is a JSON *string* holding {"sdp":"v=0...","sdpType":"answer",...}.
json exchange;
auto it = parsed.find("exchangeResponse");
if (it == parsed.end() || it->is_null()) return true; // not answered yet
if (it->is_string()) exchange = json::parse(it->get<std::string>(), nullptr, false);
else exchange = *it;
if (exchange.is_discarded() || !exchange.is_object()) {
proto::SdpExchangeFields f;
const std::string raw = it->is_string() ? it->get<std::string>() : it->dump(-1, ' ', false, json::error_handler_t::replace);
if (!proto::parseExchangeFields(raw, f)) {
err = "sdp exchange: unparsable exchangeResponse";
return false;
}
const std::string status = jsonString(exchange, "status");
auto sdpIt = exchange.find("sdp");
if (sdpIt == exchange.end() || !sdpIt->is_string() || sdpIt->get<std::string>().empty()) {
err = "sdp exchange: answer has no sdp" + (status.empty() ? std::string() : " (status " + status + ")") +
": " + snippet(exchange.dump(), 400);
if (f.sdp.empty()) {
err = "sdp exchange: answer has no sdp" + (f.status.empty() ? std::string() : " (status " + f.status + ")") +
": " + f.summary;
out.rejected = true;
return false;
}
// xbox.com handleSdpAnswer only accepts status "success"; an absent status is tolerated.
if (!f.status.empty() && f.status != "success") {
err = "sdp exchange: status " + f.status + ": " + f.summary;
out.rejected = true;
return false;
}
XC_LOGI("sdp exchange fields: %s", f.summary.c_str());
// Keep the answer verbatim (CRLF intact) — re-serialising corrupted ice-ufrag/pwd (green-nx).
answerSdp = sdpIt->get<std::string>();
out.sdp = std::move(f.sdp);
out.chat = f.chat;
out.chatStream = f.chatStream;
out.summary = std::move(f.summary);
ready = true;
return true;
}
+20 -2
View File
@@ -27,6 +27,18 @@ struct SessionInfo {
std::string host;
};
// GET {sessionPath}/sdp answer with the exchangeResponse fields voice chat needs.
struct SdpAnswerInfo {
std::string sdp; // verbatim answer SDP (CRLF intact)
int chat = -1; // exchangeResponse.chat (-1 absent)
int chatStream = -1; // exchangeResponse.chatStream (-1 absent)
std::string summary; // proto::SdpExchangeFields::summary
// Set when pollSdpAnswer failed because the server refused the exchange itself (errorDetails,
// exchangeResponse.status other than "success", answer without sdp) rather than a transport
// or HTTP error. Polling again will not change the outcome.
bool rejected = false;
};
// Well-known gssv error codes (SessionInfo::errorCode).
constexpr const char* kErrOfferingDoesNotContainTitle = "OfferingDoesNotContainTitle";
@@ -72,11 +84,17 @@ public:
// Queue wait time in seconds for a cloud title (GET /v1/waittime/{titleId}); -1 if unknown.
int waitTimeSeconds(const std::string& titleId);
// POST {sessionPath}/sdp with proto::sdpPostBody(offer).
bool sendSdpOffer(const SessionInfo& s, const std::string& sdp, std::string& err);
// POST {sessionPath}/sdp with proto::sdpPostBody(offer, chatStream).
bool sendSdpOffer(const SessionInfo& s, const std::string& sdp, std::string& err, bool chatStream = false);
// POST {sessionPath}/sdp with proto::sdpChatRenegotiationBody(sdp) (voice chat second offer).
bool sendSdpRenegotiation(const SessionInfo& s, const std::string& sdp, std::string& err);
// GET {sessionPath}/sdp. ready=false while the server has not answered yet (HTTP 204).
// Forwards to the SdpAnswerInfo overload.
bool pollSdpAnswer(const SessionInfo& s, std::string& answerSdp, bool& ready, std::string& err);
// Same, returning every exchange field (logs "sdp exchange fields: ..." once per answer).
bool pollSdpAnswer(const SessionInfo& s, SdpAnswerInfo& out, bool& ready, std::string& err);
// POST {sessionPath}/ice with proto::icePostBody(candidateLines, ufrag).
bool sendIce(const SessionInfo& s, const std::vector<std::string>& candidateLines, const std::string& ufrag,
+109 -4
View File
@@ -17,6 +17,7 @@
#include <sys/socket.h>
#include <cstdio>
#include <cstdlib>
#include <cstring>
#include <mutex>
#include <random>
@@ -386,7 +387,7 @@ std::vector<std::string> expandTeredoCandidates(const std::vector<std::string>&
// ============================================================================================
std::string buildOffer(const std::string& ufrag, const std::string& pwd, const std::string& fingerprint, bool home,
const std::string& resolution) {
const std::string& resolution, const OfferMic* mic, int sessionVersion) {
const std::string u = trimmed(ufrag), p = trimmed(pwd);
std::string fp = trimmed(fingerprint);
if (startsWith(fp, "sha-256 ")) fp = trimmed(fp.substr(8));
@@ -406,7 +407,7 @@ std::string buildOffer(const std::string& ufrag, const std::string& pwd, const s
s += "\r\n";
};
L("v=0");
L("o=- 4611731400430051 2 IN IP4 127.0.0.1");
L("o=- 4611731400430051 " + std::to_string(sessionVersion) + " IN IP4 127.0.0.1");
L("s=-");
L("t=0 0");
L("a=group:BUNDLE video audio 0");
@@ -436,7 +437,20 @@ std::string buildOffer(const std::string& ufrag, const std::string& pwd, const s
L("a=rtcp-rsize");
L("a=rtpmap:111 opus/48000/2");
L("a=fmtp:111 minptime=10;useinbandfec=1;stereo=1");
L("a=recvonly");
if (mic && mic->ssrc != 0) {
// Voice chat: the microphone goes up on this m-line (xbox.com adds its mic track to the
// audio transceiver the same way). Attribute order matches libdatachannel's addSSRC().
const std::string ssrc = std::to_string(mic->ssrc);
const std::string msid = trimmed(mic->msid);
const std::string track = trimmed(mic->trackId).empty() ? msid : trimmed(mic->trackId);
const std::string cname = trimmed(mic->cname).empty() ? std::string("xc") + ssrc : trimmed(mic->cname);
L("a=sendrecv");
if (!msid.empty()) L("a=msid:" + msid + " " + track);
L("a=ssrc:" + ssrc + " cname:" + cname);
if (!msid.empty()) L("a=ssrc:" + ssrc + " msid:" + msid + " " + track);
} else {
L("a=recvonly");
}
// data channels (GreenOvercast; libdatachannel assigns the SCTP m-line mid "0")
L("m=application 9 UDP/DTLS/SCTP webrtc-datachannel");
L("c=IN IP4 0.0.0.0");
@@ -446,7 +460,7 @@ std::string buildOffer(const std::string& ufrag, const std::string& pwd, const s
return s;
}
std::string sdpPostBody(const std::string& offer) {
std::string sdpPostBody(const std::string& offer, bool chatStream) {
json body = {
{"messageType", "offer"},
{"sdp", offer},
@@ -465,9 +479,100 @@ std::string sdpPostBody(const std::string& offer) {
{"reliableinput", {{"minVersion", 9}, {"maxVersion", 9}}},
{"unreliableinput", {{"minVersion", 9}, {"maxVersion", 9}}}}},
};
// Media-stream chat (xbox.com ChatStreamManager): mic as RTP on the audio m-line.
if (chatStream) body["configuration"]["chatStream"] = {{"minVersion", 1}, {"maxVersion", 1}};
return body.dump(-1, ' ', false, kReplace);
}
std::string sdpChatRenegotiationBody(const std::string& offer) {
json body = {
{"messageType", "offer"},
{"requestId", "2"},
{"sdp", offer},
{"configuration", {{"isMediaStreamsChatRenegotiation", true}}},
};
return body.dump(-1, ' ', false, kReplace);
}
namespace {
// Integer value of an exchange field: numbers, bools and numeric strings; -1 otherwise.
int exchangeInt(const json& obj, const char* key) {
auto it = obj.find(key);
if (it == obj.end()) return -1;
if (it->is_number_integer() || it->is_number_unsigned()) {
const long long v = it->get<long long>();
return v < -1 ? -1 : v > 0x7FFFFFFF ? 0x7FFFFFFF : static_cast<int>(v);
}
if (it->is_number_float()) {
const double v = it->get<double>();
return v != v || v < -1.0 ? -1 : v > 2147483647.0 ? 0x7FFFFFFF : static_cast<int>(v);
}
if (it->is_boolean()) return it->get<bool>() ? 1 : 0;
if (it->is_string()) {
const std::string t = trimmed(it->get<std::string>());
if (t.empty() || t.size() > 9) return -1;
for (char c : t)
if (c < '0' || c > '9') return -1;
return std::atoi(t.c_str());
}
return -1;
}
} // namespace
bool parseExchangeFields(const std::string& exchangeJson, SdpExchangeFields& out) {
out = SdpExchangeFields{};
json j = json::parse(exchangeJson, nullptr, false);
// exchangeResponse is normally a JSON *string* holding the object.
if (!j.is_discarded() && j.is_string()) j = json::parse(j.get<std::string>(), nullptr, false);
if (j.is_discarded() || !j.is_object()) return false;
auto sdpIt = j.find("sdp");
if (sdpIt != j.end() && sdpIt->is_string()) out.sdp = sdpIt->get<std::string>();
auto stIt = j.find("status");
if (stIt != j.end() && stIt->is_string()) out.status = stIt->get<std::string>();
out.chat = exchangeInt(j, "chat");
out.chatStream = exchangeInt(j, "chatStream");
json rest = j;
rest.erase("sdp");
out.summary = rest.dump(-1, ' ', false, kReplace);
if (out.summary.size() > 400) out.summary.resize(400);
return true;
}
std::string sdpMediaDirection(const std::string& sdp, const std::string& mid) {
std::istringstream in(sdp);
std::string line;
bool inMedia = false, match = false;
std::string dir;
while (std::getline(in, line)) {
line = trimmed(line);
if (startsWith(line, "m=")) {
if (match) break;
inMedia = true;
dir.clear();
continue;
}
if (!inMedia) continue;
if (startsWith(line, "a=mid:")) {
if (trimmed(line.substr(6)) == mid) match = true;
} else if (line == "a=sendrecv" || line == "a=sendonly" || line == "a=recvonly" || line == "a=inactive") {
dir = line.substr(2);
}
}
if (!match) return std::string();
return dir.empty() ? std::string("sendrecv") : dir;
}
std::string messageSetPartyChatActive(bool partyChatActive, const std::string& cv) {
return json{{"type", "TransactionStart"},
{"content", json{{"partyChatActive", partyChatActive}}.dump(-1, ' ', false, kReplace)},
{"id", newUuid()},
{"target", target::kSetPartyChatActive},
{"cv", cv}}
.dump(-1, ' ', false, kReplace);
}
std::string icePostBody(const std::vector<std::string>& candidates, const std::string& ufrag) {
auto entry = [&ufrag](const std::string& cand) {
return json{{"candidate", cand}, {"sdpMid", "0"}, {"sdpMLineIndex", 0}, {"usernameFragment", ufrag}}.dump(-1, ' ', false, kReplace);
+45 -2
View File
@@ -146,6 +146,7 @@ constexpr const char* kServerInitiatedDisconnect = "/streaming/sessionLifetimeMa
constexpr const char* kShowMessageDialog = "/streaming/systemUi/messages/ShowMessageDialog";
constexpr const char* kShowVirtualKeyboard = "/streaming/systemUi/messages/ShowVirtualKeyboard";
constexpr const char* kTitleInfo = "/streaming/properties/titleinfo";
constexpr const char* kSetPartyChatActive = "/streaming/social/partyChatAudioCoordination/setPartyChatActive";
} // namespace target
// Parse a server->client input packet carrying a Vibration section. False if not vibration.
@@ -161,6 +162,14 @@ bool teredoDecode(const std::string& ipv6, std::string& ipv4, uint16_t& port);
// prefix as the source line) come right before the original Teredo line, decoded port first.
std::vector<std::string> expandTeredoCandidates(const std::vector<std::string>& candidates);
// Microphone send-stream identity announced in the audio m-line (voice chat).
struct OfferMic {
uint32_t ssrc = 0; // != 0
std::string cname; // e.g. "xc1a2b3c4d" (WebRtc::micInfo().cname)
std::string msid; // stream id (uuid)
std::string trackId; // track id (uuid)
};
// Hand-written browser-compatible offer (GreenOvercast buildCompatibleOffer / xCloud template):
// BUNDLE video audio 0, H.264 profiles, Opus, data channel m-line mid 0, with the given ICE
// credentials and sha-256 fingerprint ("AB:CD:..."). home selects xHome tweaks.
@@ -171,10 +180,44 @@ std::vector<std::string> expandTeredoCandidates(const std::vector<std::string>&
// home (any): profile-level-id=42e020;max-fs=3600;max-mbps=108000
// and Opus PT 111 "minptime=10;useinbandfec=1;stereo=1". All lines CRLF. Returns "" if
// ufrag, pwd or fingerprint is empty (a leading "sha-256 " on the fingerprint is stripped).
//
// mic == nullptr (or mic->ssrc == 0): audio m-line is recvonly, byte-identical to the original
// template. mic != nullptr: the audio m-line carries, in this order (replacing "a=recvonly"):
// a=mid:audio / a=rtcp-mux / a=rtcp-rsize / a=rtpmap:111 opus/48000/2 /
// a=fmtp:111 minptime=10;useinbandfec=1;stereo=1 / a=sendrecv /
// a=msid:<msid> <trackId> / a=ssrc:<ssrc> cname:<cname> / a=ssrc:<ssrc> msid:<msid> <trackId>
// (SSRC in decimal; an empty trackId falls back to msid, an empty msid omits the msid lines).
// sessionVersion: number in "o=- 4611731400430051 <v> IN IP4 127.0.0.1" (initial 2, renegotiation 3).
std::string buildOffer(const std::string& ufrag, const std::string& pwd, const std::string& fingerprint, bool home,
const std::string& resolution);
const std::string& resolution, const OfferMic* mic = nullptr, int sessionVersion = 2);
// JSON body for POST /sdp: {"messageType":"offer","sdp":..., "requestId":"1","configuration":{...}}.
std::string sdpPostBody(const std::string& offer);
// chatStream=true additionally puts "chatStream":{"minVersion":1,"maxVersion":1} into configuration
// (every other key unchanged, chatConfiguration kept); false gives the original body.
std::string sdpPostBody(const std::string& offer, bool chatStream = false);
// Second (chat media-stream renegotiation) offer, as xbox.com's ChatStreamManager posts it:
// {"messageType":"offer","requestId":"2","sdp":<offer>,"configuration":{"isMediaStreamsChatRenegotiation":true}}.
std::string sdpChatRenegotiationBody(const std::string& offer);
// Fields of the parsed exchangeResponse object (gssv GET /sdp). Missing ints = -1.
struct SdpExchangeFields {
std::string sdp;
std::string status;
int chat = -1;
int chatStream = -1;
std::string summary; // exchange JSON re-dumped WITHOUT the "sdp" key (for logs), max 400 chars
};
// Parse an exchangeResponse (JSON text of the object, or JSON text of a string holding one).
// False if unparsable / not an object. Numeric fields also accept bools and numeric strings.
bool parseExchangeFields(const std::string& exchangeJson, SdpExchangeFields& out);
// Direction attribute of the m-section with a=mid:<mid> in an SDP ("sendrecv","sendonly",
// "recvonly","inactive"; "" if the mid is absent; "sendrecv" if the section has none).
std::string sdpMediaDirection(const std::string& sdp, const std::string& mid);
// Not used yet (party-client phase 2): message-channel TransactionStart for
// /streaming/social/partyChatAudioCoordination/setPartyChatActive, content {"partyChatActive":b}.
std::string messageSetPartyChatActive(bool partyChatActive, const std::string& cv = "");
// JSON body for POST /ice (green-nx shape: stringified {candidate,sdpMid:"0",sdpMLineIndex:0,
// usernameFragment} entries + end-of-candidates). Candidate lines may carry an "a=" prefix
// and trailing CR/LF (both stripped); TCP candidates are skipped.
+41
View File
@@ -0,0 +1,41 @@
// Nubix — microphone capture abstraction (PS5 sceAudioIn / host SDL capture).
// Copyright (C) 2026 Nubix contributors
// SPDX-License-Identifier: GPL-3.0-only
#pragma once
#include <cstdint>
#include <memory>
#include <string>
namespace xc {
namespace platform {
// Mono s16 microphone source. One instance per stream; open/read/close from ONE thread.
// An instance may be re-opened after close().
class AudioIn {
public:
virtual ~AudioIn() = default;
// Open the device. On success rate (Hz) and grain (samples per read) are set. Logs every attempt.
virtual bool open(int& rate, int& grain) = 0;
// Read exactly one grain into buf (capacity >= grain). Blocks at most ~timeoutMs
// (PS5: blocks one hardware grain, timeout ignored). Returns samples (>0), 0 on timeout,
// <0 on fatal error (device gone).
virtual int read(int16_t* buf, int timeoutMs) = 0;
// 0 = live. PS5: sceAudioInGetSilentState bits (0x1 no device, 0x2 low priority,
// 0x8 unable format, others = user mute). Host: 0. Cheap enough to call every 500 ms.
virtual uint32_t silentState() = 0;
virtual void close() = 0; // idempotent
virtual const char* backend() const = 0; // "ps5-audioin", "sdl", "tone", "none"
// Raw backend code of the last failed read() (PS5: sceAudioInInput rc), 0 if none/unknown.
virtual int32_t lastError() const { return 0; }
};
// Silent-state bit: no input device for the user (headset / controller mic absent).
constexpr uint32_t kAudioInSilentNoDevice = 0x1;
// PS5: sceAudioIn backend. Host: XC_MIC_SOURCE=tone -> 440 Hz -20 dBFS synthetic,
// =none -> open() fails, else SDL capture (default device, 48000 Hz, mono, s16, 480-sample grain).
std::unique_ptr<AudioIn> createAudioIn();
} // namespace platform
} // namespace xc
+180
View File
@@ -0,0 +1,180 @@
// Nubix — host microphone sources: SDL capture, synthetic tone, "none".
// Copyright (C) 2026 Nubix contributors
// SPDX-License-Identifier: GPL-3.0-only
//
// XC_MIC_SOURCE selects the source: "tone" (440 Hz sine at -20 dBFS, paced in real time),
// "none" (open() fails: simulates a console without a microphone), anything else / unset:
// the default SDL capture device at 48 kHz mono s16 with a 480-sample (10 ms) grain.
#ifndef XC_PS5
#include "platform/audio_in.hpp"
#include <SDL.h>
#include <chrono>
#include <cmath>
#include <cstdlib>
#include <cstring>
#include <string>
#include <thread>
#include "core/log.hpp"
namespace xc {
namespace platform {
namespace {
constexpr int kRate = 48000;
constexpr int kGrain = 480; // 10 ms
// ---- XC_MIC_SOURCE=none ----------------------------------------------------------------------
class NoneAudioIn final : public AudioIn {
public:
bool open(int&, int&) override {
XC_LOGI("voice: host mic source 'none' (XC_MIC_SOURCE=none)");
return false;
}
int read(int16_t*, int) override { return -1; }
uint32_t silentState() override { return kAudioInSilentNoDevice; }
void close() override {}
const char* backend() const override { return "none"; }
};
// ---- XC_MIC_SOURCE=tone ----------------------------------------------------------------------
class ToneAudioIn final : public AudioIn {
public:
bool open(int& rate, int& grain) override {
rate = kRate;
grain = kGrain;
phase_ = 0.0;
next_ = Clock::now();
open_ = true;
XC_LOGI("voice: host mic source 'tone' (440 Hz, -20 dBFS, %d Hz, grain %d)", kRate, kGrain);
return true;
}
int read(int16_t* buf, int timeoutMs) override {
if (!open_) return -1;
// Pace like a real device: one grain per 10 ms of wall clock.
const auto now = Clock::now();
if (next_ > now) {
const auto wait = next_ - now;
if (wait > std::chrono::milliseconds(timeoutMs < 0 ? 0 : timeoutMs)) {
std::this_thread::sleep_for(std::chrono::milliseconds(timeoutMs < 0 ? 0 : timeoutMs));
return 0;
}
std::this_thread::sleep_until(next_);
} else if (now - next_ > std::chrono::milliseconds(200)) {
next_ = now; // fell far behind (debugger, suspend): do not burst
}
next_ += std::chrono::microseconds(1000000LL * kGrain / kRate);
const double amp = 32767.0 * 0.1; // -20 dBFS
const double step = 2.0 * M_PI * 440.0 / kRate;
for (int i = 0; i < kGrain; ++i) {
buf[i] = static_cast<int16_t>(std::lround(amp * std::sin(phase_)));
phase_ += step;
if (phase_ > 2.0 * M_PI) phase_ -= 2.0 * M_PI;
}
return kGrain;
}
uint32_t silentState() override { return 0; }
void close() override { open_ = false; }
const char* backend() const override { return "tone"; }
private:
using Clock = std::chrono::steady_clock;
bool open_ = false;
double phase_ = 0.0;
Clock::time_point next_{};
};
// ---- SDL capture -----------------------------------------------------------------------------
class SdlAudioIn final : public AudioIn {
public:
~SdlAudioIn() override { close(); }
bool open(int& rate, int& grain) override {
close();
if (SDL_InitSubSystem(SDL_INIT_AUDIO) != 0) {
XC_LOGW("voice: SDL_InitSubSystem(AUDIO) failed: %s", SDL_GetError());
return false;
}
sdlInited_ = true;
const int nDev = SDL_GetNumAudioDevices(1);
const char* name = nDev > 0 ? SDL_GetAudioDeviceName(0, 1) : nullptr;
XC_LOGI("voice: host SDL capture devices: %d (default '%s')", nDev, name ? name : "?");
SDL_AudioSpec want;
SDL_zero(want);
want.freq = kRate;
want.format = AUDIO_S16SYS;
want.channels = 1;
want.samples = kGrain;
want.callback = nullptr; // queue mode: SDL_DequeueAudio
SDL_AudioSpec have;
SDL_zero(have);
// No ALLOW_* flags: SDL converts to exactly s16 / 48 kHz / mono for us.
dev_ = SDL_OpenAudioDevice(nullptr, 1, &want, &have, 0);
if (dev_ == 0) {
XC_LOGW("voice: SDL_OpenAudioDevice(capture) failed: %s", SDL_GetError());
close();
return false;
}
SDL_PauseAudioDevice(dev_, 0);
rate = kRate;
grain = kGrain;
XC_LOGI("voice: host SDL capture open (%s, %d Hz, %d ch, %u samples)",
SDL_GetCurrentAudioDriver() ? SDL_GetCurrentAudioDriver() : "?", have.freq,
have.channels, static_cast<unsigned>(have.samples));
return true;
}
int read(int16_t* buf, int timeoutMs) override {
if (dev_ == 0) return -1;
const Uint32 need = static_cast<Uint32>(kGrain * sizeof(int16_t));
const auto deadline = std::chrono::steady_clock::now() + std::chrono::milliseconds(timeoutMs < 0 ? 0 : timeoutMs);
while (SDL_GetQueuedAudioSize(dev_) < need) {
if (SDL_GetAudioDeviceStatus(dev_) == SDL_AUDIO_STOPPED) return -1; // unplugged
if (std::chrono::steady_clock::now() >= deadline) return 0;
std::this_thread::sleep_for(std::chrono::milliseconds(2));
}
const Uint32 got = SDL_DequeueAudio(dev_, buf, need);
if (got < need) std::memset(reinterpret_cast<uint8_t*>(buf) + got, 0, need - got);
return kGrain;
}
uint32_t silentState() override { return 0; }
void close() override {
if (dev_ != 0) {
SDL_CloseAudioDevice(dev_);
dev_ = 0;
}
if (sdlInited_) {
SDL_QuitSubSystem(SDL_INIT_AUDIO); // reference counted by SDL
sdlInited_ = false;
}
}
const char* backend() const override { return "sdl"; }
private:
SDL_AudioDeviceID dev_ = 0;
bool sdlInited_ = false;
};
} // namespace
std::unique_ptr<AudioIn> createAudioIn() {
const char* env = std::getenv("XC_MIC_SOURCE");
const std::string src = env ? env : "";
if (src == "tone") return std::make_unique<ToneAudioIn>();
if (src == "none") return std::make_unique<NoneAudioIn>();
return std::make_unique<SdlAudioIn>();
}
} // namespace platform
} // namespace xc
#endif // !XC_PS5
+246
View File
@@ -0,0 +1,246 @@
// Nubix — PS5 microphone capture via libSceAudioIn (headset / DualSense mic).
// Copyright (C) 2026 Nubix contributors
// SPDX-License-Identifier: GPL-3.0-only
//
// libSceAudioIn is not among the payload's DT_NEEDED modules and the SDK ships no stub for it,
// so it is resolved at run time: dlopen("libSceAudioIn.sprx") first, then
// sceKernelLoadStartModule + sceKernelDlsym. The PS4-era ABI is used:
// int32_t sceAudioInOpen(userId, type, index, len, freq, param) (HqOpen: same shape)
// int32_t sceAudioInInput(handle, dst) blocks for one grain
// int32_t sceAudioInGetSilentState(handle)
// Format/type/rate combinations differ between firmware and device, so open() walks a small
// table and logs every attempt; the first one accepted wins.
#ifdef XC_PS5
#include "platform/audio_in.hpp"
#include <dlfcn.h>
#include <chrono>
#include <cstdint>
#include <cstdio>
#include <mutex>
#include "core/log.hpp"
extern "C" {
int sceKernelLoadStartModule(const char* path, size_t argc, const void* argv, uint32_t flags, void* opt, int* res);
int sceKernelDlsym(int handle, const char* symbol, void** addr);
int sceUserServiceInitialize(const void* params);
int sceUserServiceGetForegroundUser(int32_t* userId);
int sceUserServiceGetInitialUser(int32_t* userId);
}
namespace xc {
namespace platform {
namespace {
using OpenFn = int32_t (*)(int32_t userId, uint32_t type, uint32_t index, uint32_t len, uint32_t freq, uint32_t param);
using InputFn = int32_t (*)(int32_t handle, void* dst);
using CloseFn = int32_t (*)(int32_t handle);
using SilentFn = int32_t (*)(int32_t handle);
using InitFn = int32_t (*)(void);
constexpr const char* kModuleName = "libSceAudioIn.sprx";
constexpr const char* kModulePath = "/system/common/lib/libSceAudioIn.sprx";
constexpr int32_t kUserSystem = 0xFF; // never open for SYSTEM
constexpr uint32_t kParamS16Mono = 0;
struct AudioInApi {
OpenFn open = nullptr;
OpenFn hqOpen = nullptr;
InputFn input = nullptr;
CloseFn close = nullptr;
SilentFn silent = nullptr;
InitFn init = nullptr;
bool ok() const { return (open || hqOpen) && input && close; }
};
std::once_flag g_loadOnce;
AudioInApi g_api;
void* symDl(void* h, const char* name) { return h ? dlsym(h, name) : nullptr; }
void* symKernel(int h, const char* name) {
void* p = nullptr;
if (h < 0 || sceKernelDlsym(h, name, &p) != 0) return nullptr;
return p;
}
template <typename Lookup>
void resolveAll(AudioInApi& api, Lookup&& sym) {
api.open = reinterpret_cast<OpenFn>(sym("sceAudioInOpen"));
api.hqOpen = reinterpret_cast<OpenFn>(sym("sceAudioInHqOpen"));
api.input = reinterpret_cast<InputFn>(sym("sceAudioInInput"));
api.close = reinterpret_cast<CloseFn>(sym("sceAudioInClose"));
api.silent = reinterpret_cast<SilentFn>(sym("sceAudioInGetSilentState"));
api.init = reinterpret_cast<InitFn>(sym("sceAudioInInit"));
}
// Resolve the module once per process. Every outcome is logged.
void loadApi() {
AudioInApi api;
if (void* h = dlopen(kModuleName, RTLD_LAZY)) {
resolveAll(api, [h](const char* n) { return symDl(h, n); });
if (api.ok()) {
XC_LOGI("voice: ps5 libSceAudioIn via %s (%s)", "dlopen", "ok");
} else {
XC_LOGW("voice: ps5 libSceAudioIn via %s (%s)", "dlopen", "FAILED symbols");
api = AudioInApi{};
}
} else {
const char* e = dlerror();
XC_LOGW("voice: ps5 libSceAudioIn via %s (%s: %s)", "dlopen", "FAILED", e ? e : "?");
}
if (!api.ok()) {
int res = 0;
const int h = sceKernelLoadStartModule(kModulePath, 0, nullptr, 0, nullptr, &res);
if (h >= 0) {
resolveAll(api, [h](const char* n) { return symKernel(h, n); });
if (api.ok()) {
XC_LOGI("voice: ps5 libSceAudioIn via %s (%s)", "sceKernelLoadStartModule", "ok");
} else {
XC_LOGW("voice: ps5 libSceAudioIn via %s (%s)", "sceKernelLoadStartModule", "FAILED symbols");
api = AudioInApi{};
}
} else {
char what[32];
std::snprintf(what, sizeof(what), "FAILED 0x%08x", static_cast<unsigned>(h));
XC_LOGW("voice: ps5 libSceAudioIn via %s (%s)", "sceKernelLoadStartModule", what);
}
}
if (api.ok() && api.init) {
const int32_t rc = api.init();
XC_LOGD("voice: ps5 sceAudioInInit -> 0x%08x", static_cast<unsigned>(rc));
}
g_api = api;
}
bool userId(int32_t& uid) {
static std::once_flag initOnce;
std::call_once(initOnce, [] { sceUserServiceInitialize(nullptr); }); // already-initialised is fine
int32_t u = -1;
if (sceUserServiceGetForegroundUser(&u) == 0 && u >= 0 && u != kUserSystem) {
XC_LOGI("voice: ps5 user id 0x%x (%s)", static_cast<unsigned>(u), "foreground");
uid = u;
return true;
}
u = -1;
if (sceUserServiceGetInitialUser(&u) == 0 && u >= 0 && u != kUserSystem) {
XC_LOGI("voice: ps5 user id 0x%x (%s)", static_cast<unsigned>(u), "initial");
uid = u;
return true;
}
XC_LOGW("voice: ps5 no foreground or initial user");
return false;
}
struct OpenAttempt {
uint32_t type, len, freq;
};
// type 0 = VOICE_CHAT, 1 = GENERAL; all S16 mono.
constexpr OpenAttempt kAttempts[] = {
{0, 256, 16000}, {1, 256, 48000}, {1, 480, 48000}, {0, 256, 48000}, {1, 256, 16000},
};
class Ps5AudioIn final : public AudioIn {
public:
~Ps5AudioIn() override { close(); }
bool open(int& rate, int& grain) override {
close();
std::call_once(g_loadOnce, loadApi);
if (!g_api.ok()) return false;
int32_t uid = -1;
if (!userId(uid)) return false;
const struct {
const char* name;
OpenFn fn;
} variants[] = {{"Open", g_api.open}, {"HqOpen", g_api.hqOpen}};
for (const auto& v : variants) {
if (!v.fn) continue;
// HqOpen's argument layout is unconfirmed (assumed equal to Open); the capture buffer
// has headroom for a larger grain, so a mismatch cannot overrun it.
if (v.fn == g_api.hqOpen) XC_LOGI("voice: ps5 every Open attempt failed, trying HqOpen (experimental)");
for (const OpenAttempt& a : kAttempts) {
const int32_t h = v.fn(uid, a.type, 0, a.len, a.freq, kParamS16Mono);
XC_LOGI("voice: ps5 %s(user=0x%x type=%u len=%u freq=%u) -> 0x%08x", v.name,
static_cast<unsigned>(uid), a.type, a.len, a.freq, static_cast<unsigned>(h));
if (h >= 0) {
handle_ = h;
grain_ = static_cast<int>(a.len);
rate = static_cast<int>(a.freq);
grain = grain_;
firstInput_ = true;
lastErr_ = 0;
lastErrLog_ = {};
lastSilent_ = 0xFFFFFFFFu;
XC_LOGI("voice: ps5 mic open: %d Hz grain %d handle 0x%08x", rate, grain,
static_cast<unsigned>(h));
return true;
}
}
}
return false;
}
int read(int16_t* buf, int /*timeoutMs*/) override {
if (handle_ < 0) return -1;
const int32_t rc = g_api.input(handle_, buf);
if (firstInput_) {
firstInput_ = false;
XC_LOGI("voice: ps5 first sceAudioInInput -> %d", static_cast<int>(rc));
}
if (rc >= 0) return grain_;
lastErr_ = rc;
const auto now = std::chrono::steady_clock::now();
if (lastErrLog_ == std::chrono::steady_clock::time_point{} || now - lastErrLog_ >= std::chrono::seconds(5)) {
lastErrLog_ = now;
XC_LOGW("voice: ps5 sceAudioInInput -> 0x%08x", static_cast<unsigned>(rc));
}
return -1;
}
uint32_t silentState() override {
if (handle_ < 0 || !g_api.silent) return 0;
const int32_t rc = g_api.silent(handle_);
// Negative = error code, not a bit set: treat as "no device" so zeros flow.
const uint32_t s = rc < 0 ? kAudioInSilentNoDevice : static_cast<uint32_t>(rc);
if (s != lastSilent_) {
XC_LOGI("voice: ps5 silent state 0x%x -> 0x%x",
lastSilent_ == 0xFFFFFFFFu ? 0u : lastSilent_, s);
lastSilent_ = s;
}
return s;
}
void close() override {
if (handle_ >= 0) {
const int32_t rc = g_api.close(handle_);
XC_LOGI("voice: ps5 mic closed (handle 0x%08x -> 0x%08x)", static_cast<unsigned>(handle_),
static_cast<unsigned>(rc));
handle_ = -1;
}
}
const char* backend() const override { return "ps5-audioin"; }
int32_t lastError() const override { return lastErr_; }
private:
int32_t handle_ = -1;
int grain_ = 0;
bool firstInput_ = true;
int32_t lastErr_ = 0;
std::chrono::steady_clock::time_point lastErrLog_{};
uint32_t lastSilent_ = 0xFFFFFFFFu;
};
} // namespace
std::unique_ptr<AudioIn> createAudioIn() { return std::make_unique<Ps5AudioIn>(); }
} // namespace platform
} // namespace xc
#endif // XC_PS5
+203 -2
View File
@@ -12,8 +12,13 @@
// Ring, prebuffer, servo and resampler design are ported from green-nx
// (src/switch/stream/audio_player.cpp, GPL-3.0); the output side uses SDL instead of audout.
//
// Threads: pushRtp()/pushOpus()/reset() may be called from any thread (normally the
// libdatachannel track callback); the SDL audio thread only touches the ring under ringMu.
// Chat voice: a second, independent Opus stream (party / game chat delivered on its own SSRC)
// has its own decoder and a small ring (40 ms prebuffer, 120 ms cap, no servo); fill() sums it
// into the game output with int16 saturation before the output mute is applied.
//
// Threads: pushRtp()/pushOpus()/pushVoiceOpus()/reset() may be called from any thread (normally
// the libdatachannel track callback); the SDL audio thread only touches the game ring under
// ringMu and the voice ring under voice.ringMu. The two ring locks are never held together.
#include "stream/audio.hpp"
#include "core/log.hpp"
@@ -23,6 +28,7 @@
#include <algorithm>
#include <atomic>
#include <climits>
#include <cstring>
#include <mutex>
#include <vector>
@@ -49,9 +55,133 @@ constexpr float kServoEmaAlpha = 0.02f;
constexpr float kServoGain = 30e-6f;
constexpr float kServoMaxAdj = 2000e-6f;
// Chat voice stream (second SSRC).
constexpr int kVoiceRingMs = 300;
constexpr int kVoicePrebufferMs = 40;
constexpr int kVoiceCapMs = 120; // above this, shed...
constexpr int kVoiceTrimMs = 60; // ...down to here
} // namespace
// Independent chat-voice stream: decoder + ring, no clock servo (voice is bursty and short
// lived; the cap bounds latency instead).
struct VoiceStream {
std::mutex decMu; // decoder state
OpusDecoder* dec = nullptr;
int lastFrameSamples = kSampleRate / 50;
std::vector<int16_t> pcm;
bool announced = false; // "chat voice stream started" logged since the last reset
std::mutex ringMu; // ring state; taken by the SDL thread, never while holding game ringMu
std::vector<int16_t> ring;
int ringSize = 0;
int readPos = 0, writePos = 0, count = 0;
bool primed = false;
std::atomic<uint64_t> packets{0}, lost{0}, droppedSamples{0};
std::atomic<int> bufferedSamples{0};
void resetRing() {
std::lock_guard<std::mutex> lk(ringMu);
readPos = writePos = count = 0;
primed = false;
bufferedSamples.store(0, std::memory_order_relaxed);
}
void resetDecoderLocked() {
if (dec) opus_decoder_ctl(dec, OPUS_RESET_STATE);
lastFrameSamples = kSampleRate / 50;
announced = false;
}
void pushRing(const int16_t* data, int n) {
if (n <= 0) return;
const int cap = kVoiceCapMs * kInt16PerMs;
const int trim = kVoiceTrimMs * kInt16PerMs;
std::lock_guard<std::mutex> lk(ringMu);
if (ringSize == 0) return;
if (n > ringSize) {
data += n - ringSize;
n = ringSize;
}
if (count + n > cap) {
int shed = std::min(count + n - trim, count);
if (shed > 0) {
shed -= shed % kChannels;
readPos = (readPos + shed) % ringSize;
count -= shed;
droppedSamples.fetch_add(static_cast<uint64_t>(shed), std::memory_order_relaxed);
}
}
if (count + n > ringSize) {
const int over = count + n - ringSize;
readPos = (readPos + over) % ringSize;
count -= over;
droppedSamples.fetch_add(static_cast<uint64_t>(over), std::memory_order_relaxed);
}
const int first = std::min(n, ringSize - writePos);
std::memcpy(&ring[writePos], data, static_cast<size_t>(first) * sizeof(int16_t));
if (n > first)
std::memcpy(&ring[0], data + first, static_cast<size_t>(n - first) * sizeof(int16_t));
writePos = (writePos + n) % ringSize;
count += n;
if (!primed && count >= kVoicePrebufferMs * kInt16PerMs) primed = true;
bufferedSamples.store(count, std::memory_order_relaxed);
}
// decMu held.
void decodeAndQueue(const uint8_t* data, size_t n, int frameSamples, bool fec) {
if (!dec) return;
const int maxSamples = (data && !fec) ? kMaxFrameSamples : frameSamples;
const int got = opus_decode(dec, data, static_cast<opus_int32>(data ? n : 0), pcm.data(),
maxSamples, fec ? 1 : 0);
if (got <= 0) return;
if (data && !fec) lastFrameSamples = got;
pushRing(pcm.data(), got * kChannels);
}
// decMu held.
void handleOpus(const uint8_t* data, size_t n, int lostBefore) {
if (!dec) return;
if (!announced) {
announced = true;
XC_LOGI("audio: chat voice stream started");
}
int frameSamples = lastFrameSamples;
const int ns = opus_packet_get_nb_samples(data, static_cast<opus_int32>(n), kSampleRate);
if (ns > 0 && ns <= kMaxFrameSamples) frameSamples = ns;
if (lostBefore > 0) {
lost.fetch_add(static_cast<uint64_t>(lostBefore), std::memory_order_relaxed);
const int conceal = std::min(lostBefore, kMaxConcealFrames);
for (int i = 0; i < conceal - 1; ++i) decodeAndQueue(nullptr, 0, lastFrameSamples, false);
decodeAndQueue(data, n, frameSamples, true);
}
decodeAndQueue(data, n, frameSamples, false);
packets.fetch_add(1, std::memory_order_relaxed);
}
// SDL audio thread: add up to `need` buffered voice samples onto out.
void mixInto(int16_t* out, int need) {
std::lock_guard<std::mutex> lk(ringMu);
if (primed && ringSize > 0) {
int done = 0;
while (done < need && count > 0) {
const int step = std::min({need - done, count, ringSize - readPos});
AudioPlayer::mixSaturate(out + done, &ring[readPos], step);
readPos = (readPos + step) % ringSize;
count -= step;
done += step;
}
// Ran dry: wait for a fresh prebuffer (the talker paused or packets were lost).
if (count == 0) primed = false;
}
bufferedSamples.store(count, std::memory_order_relaxed);
}
};
struct AudioPlayer::Impl {
VoiceStream voice;
// ---- decoder side (guarded by decMu) ----
std::mutex decMu;
OpusDecoder* dec = nullptr;
@@ -226,6 +356,7 @@ struct AudioPlayer::Impl {
}
if (given < need)
std::memset(out + given, 0, static_cast<size_t>(need - given) * sizeof(int16_t));
voice.mixInto(out, need);
if (muted.load(std::memory_order_relaxed))
std::memset(out, 0, static_cast<size_t>(need) * sizeof(int16_t));
}
@@ -277,6 +408,27 @@ bool AudioPlayer::init() {
d.ring.assign(static_cast<size_t>(d.ringSize), 0);
}
d.resetRing();
{
// Chat voice decoder: failure only disables the voice mix, never game audio.
std::lock_guard<std::mutex> lk(d.voice.decMu);
int err = OPUS_OK;
d.voice.dec = opus_decoder_create(kSampleRate, kChannels, &err);
if (err != OPUS_OK || !d.voice.dec) {
XC_LOGW("audio: chat voice decoder unavailable: %s", opus_strerror(err));
d.voice.dec = nullptr;
}
d.voice.pcm.assign(static_cast<size_t>(kMaxFrameSamples) * kChannels, 0);
d.voice.resetDecoderLocked();
}
{
std::lock_guard<std::mutex> lk(d.voice.ringMu);
d.voice.ringSize = kVoiceRingMs * kInt16PerMs;
d.voice.ring.assign(static_cast<size_t>(d.voice.ringSize), 0);
}
d.voice.resetRing();
d.voice.packets = 0;
d.voice.lost = 0;
d.voice.droppedSamples = 0;
d.packets = 0;
d.lost = 0;
d.late = 0;
@@ -388,6 +540,41 @@ void AudioPlayer::reset() {
d_->resetDecoderStateLocked();
}
d_->resetRing();
resetVoice();
}
void AudioPlayer::pushVoiceOpus(const uint8_t* opus, size_t n, int lostBefore) {
if (!opus || n == 0) return;
VoiceStream& v = d_->voice;
std::lock_guard<std::mutex> lk(v.decMu);
if (!v.dec) return;
v.handleOpus(opus, n, lostBefore < 0 ? 0 : lostBefore);
}
void AudioPlayer::resetVoice() {
VoiceStream& v = d_->voice;
{
std::lock_guard<std::mutex> lk(v.decMu);
v.resetDecoderLocked();
}
v.resetRing();
}
AudioPlayer::VoiceStats AudioPlayer::voiceStats() const {
const VoiceStream& v = d_->voice;
VoiceStats s;
s.packets = v.packets.load(std::memory_order_relaxed);
s.lost = v.lost.load(std::memory_order_relaxed);
s.droppedMs = v.droppedSamples.load(std::memory_order_relaxed) / kInt16PerMs;
s.bufferedMs = v.bufferedSamples.load(std::memory_order_relaxed) / kInt16PerMs;
return s;
}
void AudioPlayer::mixSaturate(int16_t* dst, const int16_t* src, int n) {
for (int i = 0; i < n; ++i) {
const int32_t s = static_cast<int32_t>(dst[i]) + static_cast<int32_t>(src[i]);
dst[i] = static_cast<int16_t>(std::clamp<int32_t>(s, INT16_MIN, INT16_MAX));
}
}
int AudioPlayer::bufferedMs() const {
@@ -435,6 +622,20 @@ void AudioPlayer::close() {
d.resetDecoderStateLocked();
}
d.resetRing();
{
std::lock_guard<std::mutex> lk(d.voice.decMu);
if (d.voice.dec) {
if (d.voice.packets.load() > 0)
XC_LOGI("audio: chat voice closed (%llu packets, %llu lost, %llu ms dropped)",
static_cast<unsigned long long>(d.voice.packets.load()),
static_cast<unsigned long long>(d.voice.lost.load()),
static_cast<unsigned long long>(d.voice.droppedSamples.load() / kInt16PerMs));
opus_decoder_destroy(d.voice.dec);
d.voice.dec = nullptr;
}
d.voice.resetDecoderLocked();
}
d.voice.resetRing();
}
} // namespace xc
+20 -1
View File
@@ -1,4 +1,5 @@
// Nubix — Opus audio: RTP -> libopus (48 kHz stereo) -> ring buffer -> SDL audio device.
// Nubix — Opus audio: RTP -> libopus (48 kHz stereo) -> ring buffer -> SDL audio device,
// plus an optional chat-voice stream mixed into the same output.
// Copyright (C) 2026 Nubix contributors
// SPDX-License-Identifier: GPL-3.0-only
#pragma once
@@ -53,6 +54,24 @@ public:
};
Stats stats() const;
// ---- chat / party voice (second SSRC) ----
// Separate chat/party voice stream (second SSRC): own Opus decoder (48 kHz stereo; mono
// payloads upmix in libopus), own ring (300 ms), prebuffer 40 ms, cap 120 ms (trim to 60 ms),
// no servo. Mixed into the game output in fill() with int32 sum + clamp to int16, before
// the mute check. lostBefore as in pushOpus(). Thread-safe.
void pushVoiceOpus(const uint8_t* opus, size_t n, int lostBefore = 0);
// Drop the voice ring and voice decoder state. Thread-safe. reset() also does this.
void resetVoice();
struct VoiceStats {
uint64_t packets = 0, lost = 0, droppedMs = 0;
int bufferedMs = 0;
};
VoiceStats voiceStats() const;
// dst[i] = clamp(dst[i] + src[i], INT16_MIN, INT16_MAX) for i in [0, n). Pure helper.
static void mixSaturate(int16_t* dst, const int16_t* src, int n);
private:
struct Impl;
std::unique_ptr<Impl> d_;
+436 -17
View File
@@ -16,6 +16,8 @@
// input — 125 Hz gamepad sender (contiguous sequence numbers)
// rtcp — NACK flush (20 ms), RR + REMB + stats (1 s)
// keepalive — gssv /keepalive (per session), worker-stall watchdog
// voice — VoiceChat capture thread (mic -> Opus -> WebRtc::sendMicOpus), per transport
// reneg — voice chat renegotiation: gssv POST + poll of the second offer, posts an event
// libdatachannel callbacks never block on anything the worker holds while
// calling into WebRtc: they only push events, feed the depacketizer and the
// audio player, or record rumble.
@@ -46,10 +48,12 @@ extern "C" {
#include "core/http.hpp"
#include "core/log.hpp"
#include "platform/audio_in.hpp" // complete type for VoiceChat's default source argument
#include "platform/platform.hpp"
#include "stream/audio.hpp"
#include "stream/rtp.hpp"
#include "stream/video.hpp"
#include "stream/voice.hpp"
#include "stream/webrtc.hpp"
namespace xc {
@@ -91,6 +95,7 @@ constexpr int kRtcpTickMs = 20; // NACK flush cadence
constexpr int kReportIntervalMs = 1000; // RR + REMB + stats
constexpr int kGatherTimeoutMs = 5000; // local ICE gathering
constexpr int kSdpAnswerTimeoutMs = 60000; // green-nx: 120 polls x 500 ms
constexpr int kSdpAnswerVoiceTimeoutMs = 30000; // voice offer: fall back without voice sooner
constexpr int kRemoteIceTimeoutMs = 15000; // green-nx gather deadline (home consoles: Teredo)
constexpr int kRemoteIceCloudTimeoutMs = 4000; // cloud: the first answer already carries usable candidates
constexpr int kRemoteIcePollMs = 300;
@@ -114,6 +119,11 @@ constexpr int kQueueCapMs = 2 * 60 * 60 * 1000; // WaitingForResources: wait as
constexpr int kStateCapMs = 210000; // any other gssv state that should move
constexpr int kResumeStateCapMs = 30000; // re-signal: give up on the old session quickly
constexpr int kMaxPollErrors = 5;
// Voice chat renegotiation (second offer, isMediaStreamsChatRenegotiation).
constexpr int kRenegTimeoutMs = 10000; // POST + answer, else abandoned (stream continues)
constexpr int kRenegPollMs = 500;
constexpr int kRenegMicWaitMs = 2000; // after the handshake, wait this long for the mic before renegotiating
constexpr int kChatLogInfoMessages = 5; // chat channel traffic: first N at INFO, then DEBUG
uint64_t nowMs() { return platform::monotonicMs(); }
@@ -215,11 +225,11 @@ struct Streamer::Impl {
std::atomic<bool> serverEnded{false};
std::string serverEndReason; // statusMu
// ---- worker events (from libdatachannel callbacks) ----
// ---- worker events (from libdatachannel callbacks and the renegotiation thread) ----
struct Event {
enum Type { ChannelOpen, Text, PcState } type;
enum Type { ChannelOpen, ChannelClose, Text, PcState, RenegResult } type;
uint32_t gen;
std::string ch, text;
std::string ch, text; // RenegResult: text = answer SDP ("" = failed), ch = failure reason
};
std::mutex evMu;
std::condition_variable evCv; // events + quit wakeups for the worker
@@ -250,6 +260,36 @@ struct Streamer::Impl {
RtpOpusReceiver audioRx; // internally locked
std::mutex audioDrainMu; // keeps pop -> pushOpus in order across threads
std::vector<uint8_t> audioScratch; // audioDrainMu
// Party / chat voice delivered as its own SSRC (WebRtc onChatAudioRtp): separate reorder
// window + RR statistics, played through AudioPlayer::pushVoiceOpus.
RtpOpusReceiver chatRx; // internally locked
std::vector<uint8_t> chatScratch; // audioDrainMu
std::atomic<uint64_t> chatRxPackets{0}; // per session
// ---- voice chat (microphone) ----
// The VoiceChat of the current transport (created at HandshakeAck, stopped before the
// PeerConnection closes). Readers (UI thread) copy the pointer under voiceMu.
mutable std::mutex voiceMu;
std::shared_ptr<VoiceChat> voice;
std::atomic<bool> micMuted{false}; // user preference, kept for the process lifetime
std::atomic<bool> voiceSessionOff{false}; // this session fell back to no voice (offer refused)
std::atomic<int> chatMsgs{0}; // chat data channel messages seen (log level)
// Per transport, worker thread only:
bool txVoice = false; // offer negotiated voice (sendrecv + chatStream)
bool txHome = false;
proto::OfferMic txMic;
std::string txInitialAnswer;
bool txMicAccepted = false; // answer audio is sendrecv/recvonly (server takes our mic)
std::vector<std::string> txLocalCandidates; // for the ICE exchange after the renegotiation
std::string txUfrag;
uint64_t renegDueAt = 0; // handshake time; mic Live/Muted wait ends kRenegMicWaitMs later
bool renegStarted = false;
bool renegNoChatLogged = false;
std::atomic<int> chatStreamVersion{-1}; // exchangeResponse.chatStream of the answer
std::atomic<bool> voiceRenegotiated{false};
// Renegotiation POST/poll thread (never touches WebRtc; reports through an event).
std::thread renegThr;
std::atomic<bool> renegAbort{false};
// ---- counters ----
std::atomic<uint64_t> videoBytes{0};
@@ -385,10 +425,11 @@ struct Streamer::Impl {
if (video) {
std::lock_guard<std::mutex> lk(depackMu);
depack.stats().onSenderReport(ntp);
} else if (ssrc != 0 && ssrc == chatRx.ssrc()) {
chatRx.stats().onSenderReport(ntp); // separate party/chat voice stream
} else {
audioRx.stats().onSenderReport(ntp);
}
(void)ssrc;
};
cb.onText = [this, gen](const std::string& ch, const std::string& text) {
if (gen != rtcGen.load()) return;
@@ -402,6 +443,11 @@ struct Streamer::Impl {
if (gen != rtcGen.load()) return;
pushEvent(Event{Event::ChannelOpen, gen, ch, std::string()});
};
cb.onChannelClosed = [this, gen](const std::string& ch) {
if (gen != rtcGen.load()) return;
pushEvent(Event{Event::ChannelClose, gen, ch, std::string()});
};
cb.onChatAudioRtp = [this, gen](const uint8_t* d, size_t n) { onChatAudioRtp(gen, d, n); };
cb.onState = [this, gen](const std::string& s) {
if (gen != rtcGen.load()) return;
pcConnected = (s == "connected");
@@ -483,11 +529,49 @@ struct Streamer::Impl {
int lostBefore = 0;
while (audioRx.pop(audioScratch, ts, lostBefore))
if (audioOk) audio.pushOpus(audioScratch.data(), audioScratch.size(), lostBefore);
drainChatLocked();
}
// Second PT-111 SSRC (party / chat voice as its own stream): mixed into the output.
void onChatAudioRtp(uint32_t gen, const uint8_t* d, size_t n) {
if (gen != rtcGen.load()) return;
std::lock_guard<std::mutex> lk(audioDrainMu);
if (n >= 12) {
// WebRtc replaces the chat SSRC only after the old one went quiet: start the
// reorder window and the voice decoder fresh instead of waiting for the latch.
const uint32_t ssrc = (uint32_t(d[8]) << 24) | (uint32_t(d[9]) << 16) | (uint32_t(d[10]) << 8) | d[11];
const uint32_t prev = chatRx.ssrc();
if (prev != 0 && prev != ssrc) {
chatRx.reset();
if (audioOk) audio.resetVoice();
}
}
chatRx.push(d, n);
drainChatLocked();
}
// Caller holds audioDrainMu.
void drainChatLocked() {
uint32_t ts = 0;
int lostBefore = 0;
while (chatRx.pop(chatScratch, ts, lostBefore)) {
if (!audioOk) continue;
audio.pushVoiceOpus(chatScratch.data(), chatScratch.size(), lostBefore);
chatRxPackets++;
}
}
// Server -> client input-channel reports (rumble, server metadata). Runs on a
// libdatachannel thread; only touches its own small locks.
void onBinary(const std::string& ch, const uint8_t* d, size_t n) {
if (ch == "chat") {
// Logged only (voice goes over RTP); nothing is ever sent on this channel.
if (chatMsgs.fetch_add(1) < kChatLogInfoMessages)
XC_LOGI("recv [chat] %zu binary bytes", n);
else
XC_LOGD("recv [chat] %zu binary bytes", n);
return;
}
if (ch != "input") {
XC_LOGD("recv [%s] %zu binary bytes", ch.c_str(), n);
return;
@@ -646,6 +730,8 @@ struct Streamer::Impl {
void rtcpLoop() {
uint64_t lastReport = nowMs();
uint64_t prevBytes = videoBytes.load(), prevFrames = framesDecoded.load();
uint64_t prevMicBytes = 0;
const WebRtc* prevMicRtc = nullptr;
while (!quit && !workerDone) {
std::this_thread::sleep_for(std::chrono::milliseconds(kRtcpTickMs));
std::shared_ptr<WebRtc> r = currentRtc();
@@ -677,10 +763,24 @@ struct Streamer::Impl {
vc = depack.counters();
}
RtpReceiveStats a = audioRx.stats().snapshotForReport();
RtpReceiveStats c;
if (chatRx.ssrc() != 0) c = chatRx.stats().snapshotForReport();
int micKbps = 0;
if (r) {
const WebRtc::MicTxStats mt = r->micTxStats();
// A new transport restarts its counters: no delta across PeerConnections.
if (r.get() != prevMicRtc || mt.bytes < prevMicBytes) {
prevMicRtc = r.get();
prevMicBytes = mt.bytes;
}
if (dt > 0) micKbps = static_cast<int>(static_cast<double>(mt.bytes - prevMicBytes) * 8.0 / 1000.0 / dt);
prevMicBytes = mt.bytes;
}
if (r) {
std::vector<RtpReceiveStats> blocks;
if (v.ssrc && v.packets) blocks.push_back(v);
if (a.ssrc && a.packets) blocks.push_back(a);
if (c.ssrc && c.packets) blocks.push_back(c);
if (!blocks.empty()) r->sendReceiverReport(blocks);
if (r->videoSsrc()) r->sendRemb(static_cast<uint32_t>(settings.bitrateKbps) * 1000u);
}
@@ -697,6 +797,10 @@ struct Streamer::Impl {
s.lossPercent = v.fractionLost * 100.0 / 256.0;
s.rttMs = vc.rttMs ? static_cast<int>(vc.rttMs) : -1; // NACK round-trip estimate
s.audioBufferMs = audioOk ? audio.bufferedMs() : 0;
s.voiceNegotiated = chatStreamVersion.load() >= 1;
s.voiceRenegotiated = voiceRenegotiated.load();
s.micTxKbps = micKbps;
s.chatRxPackets = chatRxPackets.load();
prevBytes = bytes;
prevFrames = frames;
{
@@ -784,8 +888,24 @@ struct Streamer::Impl {
{
std::lock_guard<std::mutex> lk(audioDrainMu);
audioRx.reset();
chatRx.reset();
}
if (audioOk) audio.reset(); // new transport: fresh RTP sequence + Opus state
if (audioOk) {
audio.reset(); // new transport: fresh RTP sequence + Opus state
audio.resetVoice();
}
txVoice = false;
txMic = proto::OfferMic();
txInitialAnswer.clear();
txMicAccepted = false;
txLocalCandidates.clear();
txUfrag.clear();
renegDueAt = 0;
renegStarted = false;
renegNoChatLogged = false;
chatStreamVersion = -1;
voiceRenegotiated = false;
chatMsgs = 0;
{
std::lock_guard<std::mutex> lk(evMu);
events.clear();
@@ -801,9 +921,29 @@ struct Streamer::Impl {
padDirty = true;
}
// Stop the microphone and the renegotiation thread of the current transport. Runs before
// every PeerConnection close (closePeer) so neither sends into a closing WebRtc. Called on
// the worker or (after the worker joined) the lifecycle thread, never on the capture or
// renegotiation thread itself (it joins them).
void stopVoice() {
std::shared_ptr<VoiceChat> v;
{
std::lock_guard<std::mutex> lk(voiceMu);
v = std::move(voice);
voice.reset();
}
if (v) {
v->stop();
XC_LOGI("voice: microphone stopped");
}
renegAbort = true;
if (renegThr.joinable()) renegThr.join();
}
// Detach and close the current PeerConnection. Callbacks of the old generation are
// ignored from here on; close() is called without holding any of our locks.
void closePeer() {
stopVoice();
std::shared_ptr<WebRtc> old;
{
std::lock_guard<std::mutex> lk(rtcMu);
@@ -840,7 +980,32 @@ struct Streamer::Impl {
rx && now >= rx ? (std::to_string((now - rx) / 1000) + "s ago").c_str() : "never");
}
void afterHandshakeAck(WebRtc& r) {
// Start the microphone for transport gen (voice negotiated in the offer). The sink runs on
// the capture thread: it sends through the current WebRtc unless the transport changed.
void startVoice(uint32_t gen) {
if (currentVoice()) return; // already running for this transport
auto v = std::make_shared<VoiceChat>();
VoiceChat::FrameSink sink = [this, gen](const uint8_t* opus, size_t n, uint32_t samples48k, bool marker) {
if (gen != rtcGen.load()) return;
std::shared_ptr<WebRtc> r = currentRtc();
if (r) r->sendMicOpus(opus, n, samples48k, marker);
};
const bool muted = micMuted.load();
if (!v->start(std::move(sink), muted)) {
XC_LOGW("voice: microphone thread could not be started, streaming without voice chat");
return;
}
XC_LOGI("voice: microphone starting (%s)", muted ? "muted" : "live");
std::lock_guard<std::mutex> lk(voiceMu);
voice = std::move(v);
}
std::shared_ptr<VoiceChat> currentVoice() const {
std::lock_guard<std::mutex> lk(voiceMu);
return voice;
}
void afterHandshakeAck(WebRtc& r, uint32_t gen) {
sendText(r, "control", proto::controlAuthorization());
sendText(r, "control", proto::controlGamepadChanged(0, true));
// Pin the resolution; without it servers pick 1440p for desktop-class clients.
@@ -860,6 +1025,14 @@ struct Streamer::Impl {
// Ask for an IDR right away (PLI + control) instead of waiting for the periodic one.
requestKeyframeInternal(true);
XC_LOGI("handshake complete, capabilities sent");
if (txVoice && txMicAccepted)
startVoice(gen);
else if (txVoice)
XC_LOGI("voice: microphone stays off (server did not accept it on this connection)");
else if (!settings.voiceChat)
XC_LOGI("voice: disabled in settings");
else
XC_LOGI("voice: not negotiated for this connection, microphone stays off");
}
// Returns true if the session must end (server disconnect).
@@ -949,7 +1122,10 @@ struct Streamer::Impl {
Impl* self;
~Closer() { self->closePeer(); }
} closer{this};
if (!r->init(makeCallbacks(gen))) {
// Voice chat: the audio m-line goes sendrecv with our mic SSRC and the POST asks for
// chatStream. Off (setting, or this session fell back): byte-identical to before.
const bool voiceWanted = settings.voiceChat && !voiceSessionOff.load();
if (!r->init(makeCallbacks(gen), voiceWanted)) {
fail("Failed to create the WebRTC connection");
return PeerOutcome::Finished;
}
@@ -970,11 +1146,32 @@ struct Streamer::Impl {
if (quit) return PeerOutcome::Finished;
XC_LOGI("gathered %zu local candidates (ufrag %s)", local.size(), ufrag.c_str());
for (const auto& c : local) XC_LOGD(" local cand: %s", c.c_str());
std::string offer = proto::buildOffer(ufrag, pwd, fingerprint, home, settings.resolution);
txHome = home;
txVoice = false;
if (voiceWanted) {
const WebRtc::MicInfo mi = r->micInfo();
if (mi.ssrc != 0) {
txMic.ssrc = mi.ssrc;
txMic.cname = mi.cname;
txMic.msid = mi.msid;
txMic.trackId = mi.trackId;
txVoice = true;
} else {
XC_LOGW("voice: no microphone send stream on the peer connection, offering without voice chat");
}
}
std::string offer =
proto::buildOffer(ufrag, pwd, fingerprint, home, settings.resolution, txVoice ? &txMic : nullptr, 2);
XC_LOGD("offer sdp:\n%s", offer.c_str());
// ---- SDP exchange ----
if (!gssv.sendSdpOffer(s, offer, err)) {
if (!gssv.sendSdpOffer(s, offer, err, txVoice)) {
if (txVoice && !quit) {
// Never let voice chat cost the stream: renegotiate this session without it.
XC_LOGW("voice: offer with voice chat refused (%s), retrying without voice chat", err.c_str());
voiceSessionOff = true;
return resuming ? PeerOutcome::RetryFresh : PeerOutcome::Resignal;
}
if (resuming) {
XC_LOGW("re-signal refused (sdp): %s", err.c_str());
return PeerOutcome::RetryFresh;
@@ -984,12 +1181,24 @@ struct Streamer::Impl {
}
std::string answer;
{
uint64_t deadline = nowMs() + kSdpAnswerTimeoutMs;
// With voice chat offered, give up on the answer sooner: the fallback without voice
// then still has the rest of the user's patience.
uint64_t deadline = nowMs() + (txVoice ? kSdpAnswerVoiceTimeoutMs : kSdpAnswerTimeoutMs);
bool ready = false;
bool rejected = false;
int errors = 0;
SdpAnswerInfo info;
err.clear();
while (!quit && !ready) {
if (!gssv.pollSdpAnswer(s, answer, ready, err)) {
if (gssv.pollSdpAnswer(s, info, ready, err)) {
answer = info.sdp;
} else {
XC_LOGW("sdp poll failed: %s", err.c_str());
// The server refused the exchange itself: polling again will not change it.
if (info.rejected) {
rejected = true;
break;
}
if (++errors >= kMaxPollErrors) break;
}
if (ready) break;
@@ -998,11 +1207,24 @@ struct Streamer::Impl {
}
if (quit) return PeerOutcome::Finished;
if (!ready || answer.empty()) {
// Never let voice chat cost the stream: a refused exchange or no answer to the
// voice offer is retried once without voice chat (as for a refused POST above).
if (txVoice) {
XC_LOGW("voice: no usable SDP answer to the voice chat offer (%s%s), retrying without voice chat",
rejected ? "refused: " : "", err.empty() ? "timeout" : err.c_str());
voiceSessionOff = true;
return resuming ? PeerOutcome::RetryFresh : PeerOutcome::Resignal;
}
if (resuming) return PeerOutcome::RetryFresh;
fail(err.empty() ? "Timed out waiting for the SDP answer" : "SDP exchange failed: " + err);
return PeerOutcome::Finished;
}
chatStreamVersion = info.chatStream;
XC_LOGI("voice: answer chat=%d chatStream=%d", info.chat, info.chatStream);
}
txInitialAnswer = answer;
txLocalCandidates = local;
txUfrag = ufrag;
XC_LOGI("answer received (%zu bytes)", answer.size());
XC_LOGD("answer sdp:\n%s", answer.c_str());
@@ -1052,14 +1274,168 @@ struct Streamer::Impl {
if (expanded.size() != remote.size())
for (const auto& c : expanded) XC_LOGD(" expanded cand: %s", c.c_str());
if (!r->setRemote(answer, expanded)) {
if (txVoice && !quit) {
XC_LOGW("voice: answer to the voice chat offer could not be applied, retrying without voice chat");
voiceSessionOff = true;
return resuming ? PeerOutcome::RetryFresh : PeerOutcome::Resignal;
}
fail("The server's SDP answer could not be applied");
return PeerOutcome::Finished;
}
if (txVoice) {
// The answer's direction is the server's view: sendonly/inactive = it will not take our mic.
const std::string dir = r->answerAudioDirection();
txMicAccepted = dir == "sendrecv" || dir == "recvonly";
XC_LOGI("voice: negotiated=%s audio=%s mic=%s", chatStreamVersion.load() >= 1 ? "yes" : "no",
dir.empty() ? "?" : dir.c_str(), txMicAccepted ? "accepted" : "refused");
if (!txMicAccepted)
XC_LOGW("voice: server answered audio %s, microphone not accepted", dir.empty() ? "(none)" : dir.c_str());
}
XC_LOGI("remote description set, checking connectivity");
return eventLoop(*r, gen);
return eventLoop(gssv, s, *r, gen);
}
PeerOutcome eventLoop(WebRtc& r, uint32_t gen) {
// ===================== voice chat renegotiation =====================
// Worker: the chat renegotiation is due once the server offered chatStream, the handshake
// is done and the microphone is open (official ChatStreamManager.enable()). Once per
// transport; any failure leaves the mic on the initial sendrecv audio line.
void maybeStartRenegotiation(GssvClient& gssv, const SessionInfo& s, WebRtc& r, uint32_t gen) {
if (!txVoice || !txMicAccepted || !handshakeDone || renegStarted) return;
// Party voice is received over the chat stream whether or not our mic works, so a mic
// failure only delays the renegotiation (at most kRenegMicWaitMs), it never blocks it.
const uint64_t now = nowMs();
if (!renegDueAt) renegDueAt = now;
std::shared_ptr<VoiceChat> v = currentVoice();
const VoiceChat::State vs = v ? v->state() : VoiceChat::State::Unavailable;
const bool micReady = vs == VoiceChat::State::Live || vs == VoiceChat::State::Muted;
if (!micReady && now - renegDueAt < static_cast<uint64_t>(kRenegMicWaitMs)) return;
const int cs = chatStreamVersion.load();
if (cs < 1) {
if (!renegNoChatLogged) {
renegNoChatLogged = true;
XC_LOGI("voice: server did not negotiate chatStream, sending mic on initial audio line only");
}
return;
}
renegStarted = true;
XC_LOGI("voice: renegotiation start (chatStream=%d, mic %s)", cs, v ? voiceStateName(vs) : "none");
std::string u, p, fp;
if (!r.beginRenegotiation(u, p, fp)) {
r.abortRenegotiation(); // roll back a local offer that was set but is unusable (no-op otherwise)
XC_LOGW("voice: renegotiation failed: no local offer");
return;
}
const std::string offer2 = proto::buildOffer(u, p, fp, txHome, settings.resolution, &txMic, 3);
XC_LOGD("renegotiation offer sdp:\n%s", offer2.c_str());
if (renegThr.joinable()) renegThr.join(); // a finished thread of an earlier transport
renegAbort = quit.load();
const std::string initial = txInitialAnswer;
const std::vector<std::string> cands = txLocalCandidates;
const std::string iceUfrag = u.empty() ? txUfrag : u;
try {
renegThr = std::thread([this, &gssv, s, offer2, initial, cands, iceUfrag, gen] {
try {
renegotiate(gssv, s, offer2, initial, cands, iceUfrag, gen);
} catch (const std::exception& e) {
XC_LOGW("voice: renegotiation thread: %s", e.what());
pushEvent(Event{Event::RenegResult, gen, std::string("failed: ") + e.what(), std::string()});
}
});
} catch (const std::exception& e) {
XC_LOGW("voice: renegotiation failed: cannot start thread (%s)", e.what());
r.abortRenegotiation();
}
}
// Renegotiation thread: POST the second offer, poll for its answer (bounded), report.
// Only gssv calls (GssvClient is safe to share across threads) and the event queue.
void renegotiate(GssvClient& gssv, const SessionInfo& s, const std::string& offer, const std::string& initial,
const std::vector<std::string>& cands, const std::string& iceUfrag, uint32_t gen) {
Http::AbortScope abortOn(&renegAbort);
const uint64_t start = nowMs();
auto stopped = [this] { return quit.load() || renegAbort.load(); };
auto report = [&](const std::string& why, const std::string& answer) {
if (!stopped()) pushEvent(Event{Event::RenegResult, gen, why, answer});
};
std::string err;
if (!gssv.sendSdpRenegotiation(s, offer, err)) {
report("failed: " + (err.empty() ? std::string("offer refused") : err), std::string());
return;
}
const uint64_t posted = nowMs();
XC_LOGI("voice: renegotiation offer posted");
int errors = 0;
bool sawInitial = false;
while (!stopped()) {
// sleep in short steps so a teardown never waits for a whole poll interval
for (int waited = 0; waited < kRenegPollMs && !stopped(); waited += 50)
std::this_thread::sleep_for(std::chrono::milliseconds(50));
if (stopped()) return;
SdpAnswerInfo info;
bool ready = false;
if (!gssv.pollSdpAnswer(s, info, ready, err)) {
XC_LOGD("voice: renegotiation poll failed: %s", err.c_str());
if (++errors >= kMaxPollErrors) {
report("failed: " + err, std::string());
return;
}
} else if (ready && !info.sdp.empty()) {
// GET /sdp keeps returning the initial answer until the second one exists; that
// is never the reply to this offer (xbox.com keeps polling too).
if (info.sdp == initial) {
sawInitial = true;
} else {
XC_LOGI("voice: renegotiation answer received (%zu bytes, %llu ms)", info.sdp.size(),
static_cast<unsigned long long>(nowMs() - posted));
// xbox.com re-runs the ICE exchange after every answer, the chat renegotiation
// included (same candidates and ufrag). Its outcome never decides the result.
if (!cands.empty() && !stopped()) {
std::string iceErr;
bool iceOk = gssv.sendIce(s, cands, iceUfrag, iceErr);
if (iceOk && !stopped()) {
std::vector<std::string> got;
bool iceReady = false;
iceOk = gssv.pollIce(s, got, iceReady, iceErr);
if (iceOk) XC_LOGD("voice: renegotiation ice poll: %zu candidates", got.size());
}
if (iceOk)
XC_LOGI("voice: renegotiation ice exchange ok");
else
XC_LOGW("voice: renegotiation ice exchange failed: %s", iceErr.c_str());
}
report(std::string(), info.sdp);
return;
}
}
if (nowMs() - start >= static_cast<uint64_t>(kRenegTimeoutMs)) {
report(sawInitial ? "timed out (server returned only the initial answer)" : "timed out", std::string());
return;
}
}
}
// Worker: apply (or abandon) the renegotiation answer. Never ends the stream.
void finishRenegotiation(WebRtc& r, const Event& ev) {
if (renegThr.joinable()) renegThr.join(); // it posted this event as its last action
if (!ev.text.empty()) {
if (r.finishRenegotiation(ev.text)) {
voiceRenegotiated = true;
XC_LOGI("voice: renegotiation ok");
return;
}
r.abortRenegotiation();
XC_LOGW("voice: renegotiation failed: answer could not be applied");
return;
}
r.abortRenegotiation();
if (ev.ch.rfind("timed out", 0) == 0)
XC_LOGW("voice: renegotiation %s", ev.ch.c_str());
else
XC_LOGW("voice: renegotiation %s", ev.ch.c_str());
}
PeerOutcome eventLoop(GssvClient& gssv, const SessionInfo& sess, WebRtc& r, uint32_t gen) {
const uint64_t negotiationStart = nowMs();
uint64_t connectedAt = 0, disconnectedAt = 0, handshakeAt = 0, ackAt = 0;
uint64_t lastSecond = nowMs(), lastLoop = nowMs();
@@ -1106,6 +1482,12 @@ struct Streamer::Impl {
case Event::ChannelOpen:
XC_LOGI("data channel open: %s", ev.ch.c_str());
break;
case Event::ChannelClose:
XC_LOGI("data channel closed: %s", ev.ch.c_str());
break;
case Event::RenegResult:
finishRenegotiation(r, ev);
break;
case Event::Text:
if (ev.ch == "message") {
XC_LOGD("recv [message] %s", preview(ev.text).c_str());
@@ -1117,7 +1499,11 @@ struct Streamer::Impl {
setStatus("Starting the stream...");
}
} else if (ev.ch == "chat") {
// chat audio is not supported
// Voice runs over RTP; the chat channel is only logged, never written.
if (chatMsgs.fetch_add(1) < kChatLogInfoMessages)
XC_LOGI("recv [chat] %s", preview(ev.text).c_str());
else
XC_LOGD("recv [chat] %s", preview(ev.text).c_str());
} else {
XC_LOGD("recv [%s] %s", ev.ch.c_str(), preview(ev.text).c_str());
}
@@ -1181,7 +1567,7 @@ struct Streamer::Impl {
if (ready || since(ackAt) > kChannelsReadyGraceMs) {
if (!ready) XC_LOGW("control/input channel not open %d ms after HandshakeAck", kChannelsReadyGraceMs);
ackPending = false;
afterHandshakeAck(r);
afterHandshakeAck(r, gen);
handshakeAt = now;
if (!resuming) setStatus("Waiting for video...");
}
@@ -1198,6 +1584,9 @@ struct Streamer::Impl {
return PeerOutcome::Finished;
}
// ---- voice chat renegotiation (mic open, or at most kRenegMicWaitMs after the handshake) ----
maybeStartRenegotiation(gssv, sess, r, gen);
// ---- first frame / streaming ----
if (gotFrame) {
if (state != StreamState::Streaming) setState(StreamState::Streaming);
@@ -1631,6 +2020,7 @@ struct Streamer::Impl {
std::lock_guard<std::mutex> lk(evMu);
quit = true;
}
renegAbort = true; // in-flight renegotiation requests abort too
evCv.notify_all();
{
std::lock_guard<std::mutex> lk(frameMu);
@@ -1763,6 +2153,8 @@ struct Streamer::Impl {
videoLossPct = 0;
frameW = 0;
frameH = 0;
voiceSessionOff = false;
chatRxPackets = 0;
{
std::lock_guard<std::mutex> lk(inputMu);
serializer = proto::InputSerializer();
@@ -1787,8 +2179,9 @@ struct Streamer::Impl {
setState(StreamState::Starting);
setStatus("Starting...");
XC_LOGI("stream start: %s %s (res %s, %d kbps)", kind == SessionKind::Home ? "home" : "cloud", target.c_str(),
settings.resolution.c_str(), settings.bitrateKbps);
XC_LOGI("stream start: %s %s (res %s, %d kbps, voice chat %s%s)", kind == SessionKind::Home ? "home" : "cloud",
target.c_str(), settings.resolution.c_str(), settings.bitrateKbps, settings.voiceChat ? "on" : "off",
settings.voiceChat && micMuted.load() ? ", mic muted" : "");
if (!decoder.init(4)) {
fail("The video decoder could not be initialised");
@@ -1922,6 +2315,32 @@ StreamStats Streamer::stats() const {
void Streamer::requestKeyframe() { d_->requestKeyframeInternal(false); }
void Streamer::setMicMuted(bool muted) {
Impl& d = *d_;
d.micMuted = muted;
if (std::shared_ptr<VoiceChat> v = d.currentVoice()) v->setMuted(muted);
}
bool Streamer::micMuted() const { return d_->micMuted.load(); }
MicState Streamer::micState() const {
std::shared_ptr<VoiceChat> v = d_->currentVoice();
if (!v) return MicState::Off;
switch (v->state()) {
case VoiceChat::State::Unavailable: return MicState::Unavailable;
case VoiceChat::State::Muted: return MicState::Muted;
case VoiceChat::State::Live: return MicState::Live;
case VoiceChat::State::Off:
case VoiceChat::State::Starting: return MicState::Off;
}
return MicState::Off;
}
float Streamer::micLevel() const {
std::shared_ptr<VoiceChat> v = d_->currentVoice();
return v ? v->level() : 0.0f;
}
void Streamer::stop() {
Impl& d = *d_;
const uint64_t gen = ++d.startGen;
+18
View File
@@ -23,6 +23,10 @@ enum class StreamState { Idle, Starting, Queued, Provisioning, Connecting, Strea
// Human readable name for a state (logs / UI).
const char* streamStateName(StreamState s);
// Microphone / voice chat state of the running stream, for the stream menu and indicator.
// Off = voice chat disabled in Settings, not streaming, or the mic is still being opened.
enum class MicState { Off, Unavailable, Muted, Live };
// Live statistics for the stream overlay.
struct StreamStats {
int width = 0, height = 0;
@@ -32,6 +36,10 @@ struct StreamStats {
double lossPercent = 0; // video packet loss over the last second
int rttMs = -1; // unknown = -1
int audioBufferMs = 0;
bool voiceNegotiated = false; // the SDP answer had chatStream >= 1
bool voiceRenegotiated = false; // the chat renegotiation (second offer) was applied
int micTxKbps = 0; // microphone RTP sent, kbps over the last second
uint64_t chatRxPackets = 0; // separate chat-SSRC voice packets played (this session)
};
class Streamer {
@@ -73,6 +81,16 @@ public:
// Send a keyframe request (PLI + control message), rate-limited.
void requestKeyframe();
// Microphone mute (voice chat). Thread-safe. The flag survives reconnects and later
// sessions of this process (not saved to disk); a muted mic keeps sending silence.
void setMicMuted(bool muted);
bool micMuted() const;
// Current microphone state (Off while voice chat is off, the mic is still opening, or no
// stream runs). Thread-safe, cheap enough to call every frame.
MicState micState() const;
// Peak input level of the last ~100 ms, 0..1 (0 when muted / unavailable / off).
float micLevel() const;
// Request the session to stop without blocking: signals the worker (in-flight requests
// abort within ~1 s) and queues the teardown (DELETE on gssv, close PC, release
// decoder/audio) on the lifecycle thread. A later start() runs after it. Idempotent.
+411
View File
@@ -0,0 +1,411 @@
// Nubix — voice chat capture: microphone -> Opus 20 ms frames for the in-stream chat RTP.
// Copyright (C) 2026 Nubix contributors
// SPDX-License-Identifier: GPL-3.0-only
//
// One capture thread per start(): open the AudioIn source, (optionally) resample to 48 kHz,
// cut 20 ms frames, encode them with libopus (mono, VOIP, 32 kbps, FEC) and hand them to the
// sink, which sends them as RTP. Muting, and a source that reports "no device", keep the frame
// cadence and send encoded digital silence, so the console's chat endpoint stays alive and no
// renegotiation is needed when the user toggles mute.
#include "stream/voice.hpp"
#include <opus.h>
#include <algorithm>
#include <atomic>
#include <chrono>
#include <cstdlib>
#include <cstring>
#include <exception>
#include <mutex>
#include <thread>
#include <vector>
#include "core/log.hpp"
#include "platform/audio_in.hpp"
namespace xc {
namespace {
using Clock = std::chrono::steady_clock;
constexpr int kRtpClock = 48000;
constexpr uint32_t kSamples48kPerFrame = kRtpClock / 1000 * VoiceChat::kFrameMs; // 960
constexpr int kMaxPacket = 400; // bytes per encoded frame (32 kbps * 20 ms = 80 typ.)
constexpr int kReadTimeoutMs = 40; // bounds stop() latency on hosts with a timeout
constexpr int kMaxGrain = 8192; // sanity bound for a source's grain
constexpr int kFatalReadErrors = 5; // consecutive read() < 0 -> give up
constexpr int kSilentPollMs = 500;
constexpr int kLevelFrames = 5; // 5 * 20 ms = level window
constexpr auto kEncoderWarnEvery = std::chrono::seconds(10);
constexpr int kPaceSlackMs = 100; // a source may run this far ahead of real time
constexpr auto kHealthFirst = std::chrono::seconds(5);
constexpr auto kHealthEvery = std::chrono::seconds(30);
bool opusRate(int r) { return r == 8000 || r == 12000 || r == 16000 || r == 24000 || r == 48000; }
// Streaming linear-interpolation resampler (mono s16), phase-continuous across blocks.
class LinearResampler {
public:
void setup(int inRate, int outRate) {
step_ = static_cast<double>(inRate) / static_cast<double>(outRate);
pos_ = 0.0;
prev_ = 0;
}
// Appends resampled samples to out.
void process(const int16_t* in, int n, std::vector<int16_t>& out) {
if (n <= 0) return;
// Index -1 is the previous block's last sample, so output lags input by one sample.
while (true) {
const int i = static_cast<int>(pos_);
if (i >= n) break;
const double frac = pos_ - i;
const int s0 = (i == 0) ? prev_ : in[i - 1];
const int s1 = in[i];
out.push_back(static_cast<int16_t>(s0 + (s1 - s0) * frac));
pos_ += step_;
}
pos_ -= n;
prev_ = in[n - 1];
}
private:
double step_ = 1.0;
double pos_ = 0.0;
int prev_ = 0;
};
} // namespace
const char* voiceStateName(VoiceChat::State s) {
switch (s) {
case VoiceChat::State::Off: return "off";
case VoiceChat::State::Starting: return "starting";
case VoiceChat::State::Unavailable: return "unavailable";
case VoiceChat::State::Muted: return "muted";
case VoiceChat::State::Live: return "live";
}
return "?";
}
struct VoiceChat::Impl {
std::unique_ptr<platform::AudioIn> src;
std::thread th;
std::mutex ctlMu; // start()/stop() serialisation
std::atomic<bool> stopReq{false};
std::atomic<bool> running{false}; // capture thread alive (cleared as it exits)
std::atomic<bool> muted{false};
std::atomic<int> state{static_cast<int>(State::Off)};
std::atomic<float> level{0.0f};
FrameSink sink;
mutable std::mutex statsMu;
Stats stats;
void setState(State s) {
const State old = static_cast<State>(state.exchange(static_cast<int>(s)));
if (old != s) XC_LOGI("voice: state %s -> %s", voiceStateName(old), voiceStateName(s));
}
void run();
void runCapture();
};
void VoiceChat::Impl::run() {
runCapture();
running.store(false);
}
void VoiceChat::Impl::runCapture() {
int rate = 0, grain = 0;
const char* backend = src->backend();
{
std::lock_guard<std::mutex> lk(statsMu);
stats = Stats{};
stats.backend = backend;
}
if (!src->open(rate, grain)) {
XC_LOGI("voice: no microphone (%s), voice chat disabled", backend);
src->close();
level.store(0.0f);
setState(State::Unavailable);
return;
}
if (rate <= 0 || grain <= 0 || grain > kMaxGrain) {
XC_LOGW("voice: no microphone (%s: bad format rate=%d grain=%d), voice chat disabled", backend, rate, grain);
src->close();
setState(State::Unavailable);
return;
}
const bool resample = !opusRate(rate);
const int encRate = resample ? kRtpClock : rate;
LinearResampler rs;
if (resample) {
rs.setup(rate, encRate);
XC_LOGI("voice: capture rate %d Hz is not an Opus rate, resampling to %d Hz", rate, encRate);
}
int err = OPUS_OK;
OpusEncoder* enc = opus_encoder_create(encRate, 1, OPUS_APPLICATION_VOIP, &err);
if (err != OPUS_OK || !enc) {
XC_LOGW("voice: opus_encoder_create(%d) failed: %s, voice chat disabled", encRate, opus_strerror(err));
src->close();
setState(State::Unavailable);
return;
}
opus_encoder_ctl(enc, OPUS_SET_BITRATE(kBitrate));
opus_encoder_ctl(enc, OPUS_SET_INBAND_FEC(1));
opus_encoder_ctl(enc, OPUS_SET_PACKET_LOSS_PERC(kLossPerc));
opus_encoder_ctl(enc, OPUS_SET_COMPLEXITY(kComplexity));
opus_encoder_ctl(enc, OPUS_SET_DTX(0));
opus_encoder_ctl(enc, OPUS_SET_SIGNAL(OPUS_SIGNAL_VOICE));
{
std::lock_guard<std::mutex> lk(statsMu);
stats.rate = encRate;
}
XC_LOGI("voice: mic started backend=%s rate=%d grain=%d", backend, rate, grain);
const int frameN = encRate * kFrameMs / 1000;
// Headroom: a backend may write a whole hardware grain larger than the one it reported
// (PS5 HqOpen is unverified); only min(n, grain) samples are consumed.
std::vector<int16_t> grainBuf(static_cast<size_t>(std::max(grain, kMaxGrain)), 0);
std::vector<int16_t> acc;
acc.reserve(static_cast<size_t>(frameN + grain * 2 + 64));
std::vector<int16_t> zeros(static_cast<size_t>(frameN), 0);
uint8_t packet[kMaxPacket];
bool marker = true;
bool noDevice = (src->silentState() & platform::kAudioInSilentNoDevice) != 0;
auto nextSilentPoll = Clock::now() + std::chrono::milliseconds(kSilentPollMs);
int consecutiveErrors = 0;
int peaks[kLevelFrames] = {0};
int peakIdx = 0;
uint64_t windowBytes = 0;
auto windowStart = Clock::now();
Clock::time_point lastEncWarn{};
bool fatal = false;
bool reopened = false;
// Real-time pacing + health log: bounds a source whose read() does not block.
auto paceStart = Clock::now();
uint64_t samplesIn = 0;
auto nextHealth = paceStart + kHealthFirst;
auto healthStart = paceStart;
uint64_t healthGrains = 0;
int healthPeak = 0;
uint32_t lastSilentBits = src->silentState();
auto updateState = [&] {
if (noDevice)
setState(State::Unavailable);
else
setState(muted.load() ? State::Muted : State::Live);
};
updateState();
while (!stopReq.load()) {
const int n = src->read(grainBuf.data(), kReadTimeoutMs);
if (n < 0) {
{
std::lock_guard<std::mutex> lk(statsMu);
++stats.readErrors;
}
const int32_t code = src->lastError();
{
std::lock_guard<std::mutex> lk(statsMu);
stats.lastError = code;
}
if (++consecutiveErrors >= kFatalReadErrors) {
// One close + reopen (headset re-plugged, controller woke up) before giving up.
int r2 = 0, g2 = 0;
if (!reopened && !stopReq.load()) {
reopened = true;
src->close();
if (src->open(r2, g2) && r2 == rate && g2 == grain) {
XC_LOGW("voice: capture error (%s 0x%08x), microphone reopened", backend,
static_cast<unsigned>(code));
consecutiveErrors = 0;
paceStart = Clock::now();
samplesIn = 0;
continue;
}
}
XC_LOGW("voice: capture error (%s 0x%08x), stopping", backend, static_cast<unsigned>(code));
fatal = true;
break;
}
std::this_thread::sleep_for(std::chrono::milliseconds(10));
continue;
}
consecutiveErrors = 0;
if (n == 0) continue;
const int got = std::min(n, grain);
auto now = Clock::now();
samplesIn += static_cast<uint64_t>(got);
{
const int64_t aheadMs = static_cast<int64_t>(samplesIn * 1000 / static_cast<uint64_t>(rate)) -
std::chrono::duration_cast<std::chrono::milliseconds>(now - paceStart).count();
if (aheadMs > kPaceSlackMs) {
std::this_thread::sleep_for(std::chrono::milliseconds(aheadMs - kPaceSlackMs));
now = Clock::now();
}
}
if (now >= nextSilentPoll) {
nextSilentPoll = now + std::chrono::milliseconds(kSilentPollMs);
lastSilentBits = src->silentState();
noDevice = (lastSilentBits & platform::kAudioInSilentNoDevice) != 0;
}
++healthGrains;
for (int i = 0; i < got; ++i) healthPeak = std::max(healthPeak, std::abs(static_cast<int>(grainBuf[i])));
if (now >= nextHealth) {
const double secs = std::chrono::duration<double>(now - healthStart).count();
uint64_t framesNow;
{
std::lock_guard<std::mutex> lk(statsMu);
framesNow = stats.frames;
}
XC_LOGI("voice: capture %d grains/s (expected %d), frames %llu, peak %.2f, silent 0x%x",
secs > 0 ? static_cast<int>(healthGrains / secs + 0.5) : 0, (rate + grain / 2) / grain,
static_cast<unsigned long long>(framesNow), healthPeak / 32767.0, lastSilentBits);
nextHealth = now + kHealthEvery;
healthStart = now;
healthGrains = 0;
healthPeak = 0;
}
updateState();
if (resample)
rs.process(grainBuf.data(), got, acc);
else
acc.insert(acc.end(), grainBuf.begin(), grainBuf.begin() + got);
size_t off = 0;
while (acc.size() - off >= static_cast<size_t>(frameN)) {
const int16_t* frame = acc.data() + off;
off += static_cast<size_t>(frameN);
const bool silent = muted.load() || noDevice;
int peak = 0;
if (silent) {
frame = zeros.data();
} else {
for (int i = 0; i < frameN; ++i) peak = std::max(peak, std::abs(static_cast<int>(frame[i])));
}
peaks[peakIdx] = peak;
peakIdx = (peakIdx + 1) % kLevelFrames;
if (silent) {
level.store(0.0f);
} else {
int m = 0;
for (int p : peaks) m = std::max(m, p);
level.store(std::min(1.0f, static_cast<float>(m) / 32767.0f));
}
const opus_int32 len = opus_encode(enc, frame, frameN, packet, kMaxPacket);
if (len < 0) {
if (lastEncWarn == Clock::time_point{} || now - lastEncWarn >= kEncoderWarnEvery) {
lastEncWarn = now;
XC_LOGW("voice: encoder error %d", static_cast<int>(len));
}
continue;
}
if (sink) sink(packet, static_cast<size_t>(len), kSamples48kPerFrame, marker);
marker = false;
windowBytes += static_cast<uint64_t>(len);
{
std::lock_guard<std::mutex> lk(statsMu);
++stats.frames;
}
}
if (off) acc.erase(acc.begin(), acc.begin() + static_cast<std::ptrdiff_t>(off));
const auto el = std::chrono::duration_cast<std::chrono::milliseconds>(now - windowStart).count();
if (el >= 1000) {
std::lock_guard<std::mutex> lk(statsMu);
stats.kbps = static_cast<int>(windowBytes * 8 / static_cast<uint64_t>(el));
windowBytes = 0;
windowStart = now;
}
}
opus_encoder_destroy(enc);
src->close();
level.store(0.0f);
{
std::lock_guard<std::mutex> lk(statsMu);
stats.kbps = 0;
XC_LOGI("voice: mic stopped (%llu frames, %llu read errors)",
static_cast<unsigned long long>(stats.frames), static_cast<unsigned long long>(stats.readErrors));
}
if (fatal) setState(State::Unavailable);
}
VoiceChat::VoiceChat(std::unique_ptr<platform::AudioIn> source) : d_(new Impl) {
d_->src = source ? std::move(source) : platform::createAudioIn();
}
VoiceChat::~VoiceChat() { stop(); }
bool VoiceChat::start(FrameSink sink, bool startMuted) {
std::lock_guard<std::mutex> lk(d_->ctlMu);
if (d_->th.joinable()) {
// A thread that already ended (no device / capture error) may be restarted.
if (d_->running.load()) return false;
d_->th.join();
}
if (!d_->src) {
d_->setState(State::Unavailable);
return false;
}
d_->stopReq.store(false);
d_->muted.store(startMuted);
d_->level.store(0.0f);
d_->sink = std::move(sink);
d_->setState(State::Starting);
d_->running.store(true);
try {
d_->th = std::thread([this] { d_->run(); });
} catch (const std::exception& e) {
XC_LOGW("voice: cannot start capture thread: %s", e.what());
d_->running.store(false);
d_->setState(State::Unavailable);
return false;
}
return true;
}
void VoiceChat::stop() {
if (!d_) return;
std::lock_guard<std::mutex> lk(d_->ctlMu);
d_->stopReq.store(true);
if (d_->th.joinable()) d_->th.join();
d_->sink = nullptr;
d_->level.store(0.0f);
d_->setState(State::Off);
}
void VoiceChat::setMuted(bool m) {
if (d_->muted.exchange(m) == m) return;
XC_LOGI("voice: mic %s", m ? "muted" : "unmuted");
// The capture thread applies it to the next frame and to state() (within one grain).
if (m) d_->level.store(0.0f);
}
bool VoiceChat::muted() const { return d_->muted.load(); }
VoiceChat::State VoiceChat::state() const { return static_cast<State>(d_->state.load()); }
float VoiceChat::level() const {
const State s = state();
if (s != State::Live) return 0.0f;
return d_->level.load();
}
VoiceChat::Stats VoiceChat::stats() const {
std::lock_guard<std::mutex> lk(d_->statsMu);
return d_->stats;
}
} // namespace xc
+63
View File
@@ -0,0 +1,63 @@
// Nubix — voice chat capture: microphone -> Opus 20 ms frames for the in-stream chat RTP.
// Copyright (C) 2026 Nubix contributors
// SPDX-License-Identifier: GPL-3.0-only
#pragma once
#include <cstddef>
#include <cstdint>
#include <functional>
#include <memory>
#include <string>
namespace xc {
namespace platform {
class AudioIn;
}
// Microphone -> Opus 20 ms frames, on its own capture thread.
class VoiceChat {
public:
// opus frame, samples in 48 kHz units (always 960 per 20 ms frame), marker = first frame
// after start(). Called on the capture thread; must not block for long (just sends RTP).
using FrameSink = std::function<void(const uint8_t* opus, size_t n, uint32_t samples48k, bool marker)>;
enum class State { Off, Starting, Unavailable, Muted, Live };
// source == nullptr -> platform::createAudioIn(). Tests inject a fake source.
explicit VoiceChat(std::unique_ptr<platform::AudioIn> source = nullptr);
~VoiceChat(); // stop()
VoiceChat(const VoiceChat&) = delete;
VoiceChat& operator=(const VoiceChat&) = delete;
// Spawn the capture thread (non-blocking; device open happens on it -> State Starting).
// False if already running. With no usable device the thread ends with State Unavailable.
bool start(FrameSink sink, bool startMuted);
// Stop and join (bounded by one grain, <= ~50 ms). Idempotent. State -> Off.
void stop();
void setMuted(bool muted); // thread-safe; muted = encode+send digital silence
bool muted() const;
State state() const; // thread-safe
float level() const; // 0..1 peak of the last ~100 ms, 0 when muted/unavailable
struct Stats {
std::string backend; // AudioIn::backend()
int rate = 0; // capture/encoder rate
uint64_t frames = 0; // frames handed to the sink
uint64_t readErrors = 0;
int32_t lastError = 0; // AudioIn::lastError() of the last failed read
int kbps = 0; // encoded payload kbps over the last second
};
Stats stats() const;
// Opus encoder settings applied by the capture thread.
static constexpr int kBitrate = 32000, kComplexity = 5, kLossPerc = 10, kFrameMs = 20;
private:
struct Impl;
std::unique_ptr<Impl> d_;
};
const char* voiceStateName(VoiceChat::State s); // "off","starting","unavailable","muted","live"
} // namespace xc
+498 -22
View File
@@ -6,6 +6,13 @@
// credentials and DTLS fingerprint are taken from libdatachannel's own local description, and
// the server answer is applied verbatim with remote candidates on mid "0".
//
// Voice chat (optional): the audio track becomes sendrecv and announces a microphone SSRC; Opus
// frames go out as raw RTP on that track (no media handler), with an RTCP SR + SDES for the mic
// SSRC once per second. Incoming PT-111 audio is split into the game stream (answer a=ssrc, else
// the first packet; never overwritten except after 2 s of silence) and one chat/party voice stream
// (any other SSRC), delivered through onAudioRtp / onChatAudioRtp. A second offer/answer round
// (chat media-stream renegotiation, xbox.com ChatStreamManager) reuses the PeerConnection.
//
// RTCP feedback (PLI with sender SSRC 0, generic NACK, compound RR+SDES with LSR/DLSR, REMB) is
// built by hand following green-nx deps/patches/libpeer-switch.patch (GPL-3.0) and sent through
// the video track: a track without media handler passes RTCP straight to the SRTP transport
@@ -35,6 +42,7 @@
#include <sstream>
#include "core/log.hpp"
#include "core/protocol.hpp"
namespace xc {
@@ -43,6 +51,18 @@ namespace {
constexpr uint8_t kDefaultVideoPt = 102;
constexpr uint8_t kDefaultAudioPt = 111;
const char* const kStunServer = "stun:stun.l.google.com:19302";
// Game audio SSRC silent this long while another PT-111 SSRC flows -> the other one becomes game.
constexpr int64_t kGameTakeoverMs = 2000;
// A different chat SSRC replaces the latched one only after it was silent this long (one voice
// stream at a time; packets of a concurrent extra stream are dropped instead of interleaved).
constexpr int64_t kChatSwitchMs = 500;
constexpr int64_t kMicSrIntervalMs = 1000;
constexpr int64_t kMicLogIntervalMs = 10000;
int64_t nowMs() {
return std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::steady_clock::now().time_since_epoch())
.count();
}
// Channel label / sub-protocol pairs (as in green-nx; reliable + ordered).
struct ChannelSpec {
@@ -126,10 +146,26 @@ uint32_t randomSsrc() {
return v;
}
uint32_t randomU32() {
std::random_device rd;
return (static_cast<uint32_t>(rd()) << 1) ^ static_cast<uint32_t>(rd());
}
const char* directionName(rtc::Description::Direction d) {
switch (d) {
case rtc::Description::Direction::SendOnly: return "sendonly";
case rtc::Description::Direction::RecvOnly: return "recvonly";
case rtc::Description::Direction::SendRecv: return "sendrecv";
case rtc::Description::Direction::Inactive: return "inactive";
default: return "unknown";
}
}
// What the server answer says about the media sections.
struct AnswerInfo {
std::set<uint8_t> videoPts, audioPts;
std::set<uint32_t> videoSsrcs, audioSsrcs;
uint32_t firstAudioSsrc = 0; // first a=ssrc of the audio section, in SDP order
};
AnswerInfo parseAnswer(const std::string& sdp) {
@@ -163,6 +199,7 @@ AnswerInfo parseAnswer(const std::string& sdp) {
const uint32_t ssrc = static_cast<uint32_t>(std::strtoul(line.c_str() + 7, nullptr, 10));
if (ssrc == 0) continue;
(sec == Sec::Video ? info.videoSsrcs : info.audioSsrcs).insert(ssrc);
if (sec == Sec::Audio && info.firstAudioSsrc == 0) info.firstAudioSsrc = ssrc;
}
}
return info;
@@ -175,7 +212,16 @@ struct Shared {
bool closed = false;
WebRtc::Callbacks cb;
std::atomic<uint32_t> videoSsrc{0}, audioSsrc{0};
std::atomic<uint32_t> videoSsrc{0}, audioSsrc{0}; // audioSsrc = game audio
std::atomic<uint32_t> chatSsrc{0}; // separate chat/party voice stream
// Game vs chat audio demux state (routeAudio).
std::mutex audioMu;
int64_t gameLastRxMs = 0;
uint64_t gameRxPackets = 0;
int64_t chatLastRxMs = 0;
uint64_t chatDropped = 0;
int64_t chatDropLogMs = 0;
std::mutex infoMu; // answer-derived demux info
std::set<uint8_t> videoPts{kDefaultVideoPt}, audioPts{kDefaultAudioPt};
@@ -200,12 +246,76 @@ struct Shared {
}
Kind classifySsrc(uint32_t ssrc) {
if (ssrc != 0 && ssrc == videoSsrc.load()) return Kind::Video;
if (ssrc != 0 && ssrc == audioSsrc.load()) return Kind::Audio;
const uint32_t game = audioSsrc.load();
if (ssrc != 0 && ssrc == game) return Kind::Audio;
// The chat/party voice SSRC: the streamer tells it apart from game audio by SSRC.
if (ssrc != 0 && ssrc == chatSsrc.load()) return Kind::Audio;
std::lock_guard<std::mutex> lk(infoMu);
if (answerVideoSsrcs.count(ssrc)) return Kind::Video;
if (answerAudioSsrcs.count(ssrc)) return Kind::Audio;
// Sender reports of a chat SSRC must not feed the game audio stats (LSR/DLSR).
if (game == 0 && answerAudioSsrcs.count(ssrc)) return Kind::Audio;
return Kind::Unknown;
}
// Game audio SSRC from the answer's first audio a=ssrc (only while none is known).
void setGameAudioFromAnswer(uint32_t ssrc) {
if (ssrc == 0) return;
std::lock_guard<std::mutex> lk(audioMu);
if (audioSsrc.load() != 0) return;
audioSsrc = ssrc;
gameLastRxMs = nowMs();
gameRxPackets = 0;
XC_LOGI("webrtc: game audio ssrc %08x (answer)", ssrc);
}
// Every PT-111 RTP packet ends up here (from the global handler or the audio track).
void routeAudio(const uint8_t* d, size_t n) {
const uint32_t ssrc = be32(d + 8);
if (ssrc == 0) return;
const int64_t now = nowMs();
bool chat = false;
{
std::lock_guard<std::mutex> lk(audioMu);
const uint32_t game = audioSsrc.load();
if (game == 0) {
audioSsrc = ssrc;
gameLastRxMs = now;
gameRxPackets = 1;
XC_LOGI("webrtc: game audio ssrc %08x (first packet)", ssrc);
} else if (ssrc == game) {
gameLastRxMs = now;
++gameRxPackets;
} else if (gameRxPackets == 0 || now - gameLastRxMs >= kGameTakeoverMs) {
// The announced game SSRC never delivered, or went silent while this one flows
// (server-side SSRC change): this stream is the game audio now.
audioSsrc = ssrc;
gameLastRxMs = now;
gameRxPackets = 1;
if (chatSsrc.load() == ssrc) chatSsrc = 0;
XC_LOGI("webrtc: game audio ssrc %08x (takeover)", ssrc);
} else {
const uint32_t cur = chatSsrc.load();
if (cur == ssrc) {
chatLastRxMs = now;
chat = true;
} else if (cur == 0 || now - chatLastRxMs >= kChatSwitchMs) {
chatSsrc = ssrc;
chatLastRxMs = now;
chat = true;
XC_LOGI("webrtc: chat audio ssrc %08x detected (game %08x)", ssrc, game);
} else {
++chatDropped;
if (now - chatDropLogMs >= kMicLogIntervalMs) {
chatDropLogMs = now;
XC_LOGD("webrtc: extra voice ssrc %08x while chat %08x active, dropped %llu packet(s)", ssrc,
cur, static_cast<unsigned long long>(chatDropped));
}
return;
}
}
}
if (chat) chatMedia(d, n);
else media(Kind::Audio, d, n);
}
void latch(Kind k, uint32_t ssrc) {
std::atomic<uint32_t>& a = k == Kind::Video ? videoSsrc : audioSsrc;
const uint32_t prev = a.exchange(ssrc);
@@ -219,6 +329,14 @@ struct Shared {
if (k == Kind::Video && cb.onVideoRtp) cb.onVideoRtp(d, n);
else if (k == Kind::Audio && cb.onAudioRtp) cb.onAudioRtp(d, n);
}
void chatMedia(const uint8_t* d, size_t n) {
std::lock_guard<std::recursive_mutex> lk(mu);
if (!closed && cb.onChatAudioRtp) cb.onChatAudioRtp(d, n);
}
void channelClosed(const std::string& ch) {
std::lock_guard<std::recursive_mutex> lk(mu);
if (!closed && cb.onChannelClosed) cb.onChannelClosed(ch);
}
void senderReport(bool video, uint32_t ssrc, uint64_t ntp) {
std::lock_guard<std::recursive_mutex> lk(mu);
if (!closed && cb.onSenderReport) cb.onSenderReport(video, ssrc, ntp);
@@ -289,12 +407,13 @@ public:
keep.push_back(std::move(m));
continue;
}
const uint32_t latched = k == Shared::Kind::Video ? s_->videoSsrc.load() : s_->audioSsrc.load();
if (latched != ssrc) s_->latch(k, ssrc);
if (k == Shared::Kind::Video && s_->videoSsrc.load() != ssrc) s_->latch(k, ssrc);
if (s_->isRoutable(ssrc)) {
keep.push_back(std::move(m)); // libdatachannel routes it to the track -> onMessage
} else if (k == Shared::Kind::Audio) {
s_->routeAudio(d, n); // unannounced SSRC: libdatachannel would drop it
} else {
s_->media(k, d, n); // unannounced SSRC: libdatachannel would drop it
s_->media(k, d, n);
}
}
messages.swap(keep);
@@ -316,6 +435,30 @@ struct WebRtc::Impl {
uint32_t localSsrc = 0;
std::string cname;
bool offered = false;
WebRtc::MicInfo mic; // ssrc 0 = voice off
std::string answerAudioDir; // direction of mid "audio" in the last applied answer
std::string initUfrag, initPwd, initFp; // credentials of the first local offer
bool renegPending = false; // beginRenegotiation() set a local offer not yet answered
// Microphone sender state (sendMicOpus), own lock: never held together with Shared::mu.
mutable std::mutex micMu;
uint16_t micSeq = 0;
uint32_t micTs = 0;
uint64_t micPackets = 0, micBytes = 0, micFailed = 0;
uint64_t micPayloadBytes = 0; // octet count for the SR (payload only, RFC 3550)
bool micFirstSent = false;
int64_t micLastSrMs = 0, micLastFailLogMs = -kMicLogIntervalMs, micLastStatsLogMs = 0;
void resetMicState() {
std::lock_guard<std::mutex> lk(micMu);
micSeq = static_cast<uint16_t>(randomU32());
micTs = randomU32();
micPackets = micBytes = micFailed = micPayloadBytes = 0;
micFirstSent = false;
micLastSrMs = 0;
micLastFailLogMs = -kMicLogIntervalMs;
micLastStatsLogMs = nowMs();
}
std::shared_ptr<rtc::DataChannel> channel(const std::string& label) const {
std::lock_guard<std::mutex> lk(mu);
@@ -343,7 +486,7 @@ struct WebRtc::Impl {
WebRtc::WebRtc() : d_(new Impl) {}
WebRtc::~WebRtc() { close(); }
bool WebRtc::init(const Callbacks& cb) {
bool WebRtc::init(const Callbacks& cb, bool voice) {
close();
initRtcLoggerOnce();
@@ -352,6 +495,15 @@ bool WebRtc::init(const Callbacks& cb) {
const uint32_t localSsrc = randomSsrc();
char cname[32];
std::snprintf(cname, sizeof(cname), "xc%08x", localSsrc);
MicInfo mic;
if (voice) {
do {
mic.ssrc = randomSsrc();
} while (mic.ssrc == localSsrc);
mic.cname = cname;
mic.msid = proto::newUuid();
mic.trackId = proto::newUuid();
}
std::shared_ptr<rtc::PeerConnection> pc;
std::shared_ptr<rtc::Track> video, audio;
@@ -396,9 +548,11 @@ bool WebRtc::init(const Callbacks& cb) {
vdesc.addH264Codec(kDefaultVideoPt,
std::string("level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f"));
video = pc->addTrack(vdesc);
rtc::Description::Audio adesc("audio", rtc::Description::Direction::RecvOnly);
rtc::Description::Audio adesc(
"audio", voice ? rtc::Description::Direction::SendRecv : rtc::Description::Direction::RecvOnly);
adesc.addOpusCodec(kDefaultAudioPt, std::string("minptime=10;useinbandfec=1"));
audio = pc->addTrack(adesc);
if (voice) adesc.addSSRC(mic.ssrc, mic.cname, mic.msid, mic.trackId);
audio = pc->addTrack(adesc); // never gets a media handler: mic RTP is sent raw
auto onTrackRtp = [wsh](Shared::Kind k) {
return [wsh, k](rtc::binary b) {
@@ -408,7 +562,13 @@ bool WebRtc::init(const Callbacks& cb) {
};
};
video->onMessage(onTrackRtp(Shared::Kind::Video), [](rtc::string) {});
audio->onMessage(onTrackRtp(Shared::Kind::Audio), [](rtc::string) {});
audio->onMessage(
[wsh](rtc::binary b) {
const auto* d = reinterpret_cast<const uint8_t*>(b.data());
if (b.size() < 12 || isRtcpPacket(d, b.size())) return;
if (auto s = wsh.lock()) s->routeAudio(d, b.size());
},
[](rtc::string) {});
// 3) data channels, in-band (DCEP), reliable + ordered, before the offer -> m-line mid "0".
for (const ChannelSpec& spec : kChannels) {
@@ -420,7 +580,10 @@ bool WebRtc::init(const Callbacks& cb) {
XC_LOGI("webrtc: channel '%s' open", label.c_str());
if (auto s = wsh.lock()) s->open(label);
});
dc->onClosed([label]() { XC_LOGI("webrtc: channel '%s' closed", label.c_str()); });
dc->onClosed([wsh, label]() {
XC_LOGI("webrtc: channel '%s' closed", label.c_str());
if (auto s = wsh.lock()) s->channelClosed(label);
});
dc->onError([label](std::string err) { XC_LOGW("webrtc: channel '%s' error: %s", label.c_str(), err.c_str()); });
dc->onMessage(
[wsh, label](rtc::binary b) {
@@ -442,16 +605,28 @@ bool WebRtc::init(const Callbacks& cb) {
return false;
}
std::lock_guard<std::mutex> lk(d_->mu);
d_->sh = sh;
d_->pc = pc;
d_->video = video;
d_->audio = audio;
d_->channels = std::move(channels);
d_->rx = rx;
d_->localSsrc = localSsrc;
d_->cname = cname;
d_->offered = false;
d_->resetMicState();
{
std::lock_guard<std::mutex> lk(d_->mu);
d_->sh = sh;
d_->pc = pc;
d_->video = video;
d_->audio = audio;
d_->channels = std::move(channels);
d_->rx = rx;
d_->localSsrc = localSsrc;
d_->cname = cname;
d_->offered = false;
d_->mic = mic;
d_->answerAudioDir.clear();
d_->initUfrag.clear();
d_->initPwd.clear();
d_->initFp.clear();
d_->renegPending = false;
}
XC_LOGI("webrtc: voice %s (audio %s, mic ssrc %08x cname %s msid %s)", voice ? "enabled" : "disabled",
voice ? "sendrecv" : "recvonly", mic.ssrc, voice ? mic.cname.c_str() : "-",
voice ? mic.msid.c_str() : "-");
return true;
}
@@ -535,6 +710,15 @@ bool WebRtc::gatherLocal(std::string& ufrag, std::string& pwd, std::string& fing
for (const auto& c : sh->trickled) add(c);
}
{
std::lock_guard<std::mutex> lk(d_->mu);
if (d_->pc == pc) {
d_->initUfrag = ufrag;
d_->initPwd = pwd;
d_->initFp = fingerprint;
}
}
XC_LOGI("webrtc: local ICE ufrag=%s, %zu candidate(s)%s", ufrag.c_str(), candidates.size(),
complete ? "" : " (gathering incomplete)");
for (const auto& c : candidates) XC_LOGD("webrtc: local %s", c.c_str());
@@ -589,7 +773,7 @@ bool WebRtc::setRemote(const std::string& answerSdp, const std::vector<std::stri
sh->routable.insert(info.audioSsrcs.begin(), info.audioSsrcs.end());
}
if (!info.videoSsrcs.empty() && sh->videoSsrc.load() == 0) sh->videoSsrc = *info.videoSsrcs.begin();
if (!info.audioSsrcs.empty() && sh->audioSsrc.load() == 0) sh->audioSsrc = *info.audioSsrcs.begin();
sh->setGameAudioFromAnswer(info.firstAudioSsrc);
XC_LOGI("webrtc: answer: video pt %d ssrcs %zu, audio pt %d ssrcs %zu",
info.videoPts.empty() ? -1 : *info.videoPts.begin(), info.videoSsrcs.size(),
info.audioPts.empty() ? -1 : *info.audioPts.begin(), info.audioSsrcs.size());
@@ -600,6 +784,12 @@ bool WebRtc::setRemote(const std::string& answerSdp, const std::vector<std::stri
XC_LOGE("webrtc: setRemoteDescription failed: %s", e.what());
return false;
}
const std::string dir = proto::sdpMediaDirection(answerSdp, "audio");
{
std::lock_guard<std::mutex> lk(d_->mu);
if (d_->pc == pc) d_->answerAudioDir = dir;
}
XC_LOGI("webrtc: answer audio direction %s", dir.empty() ? "(none)" : dir.c_str());
size_t added = 0, tried = 0;
for (const std::string& raw : remoteCandidates) {
@@ -809,6 +999,276 @@ uint32_t WebRtc::audioSsrc() const {
return d_->sh ? d_->sh->audioSsrc.load() : 0;
}
uint32_t WebRtc::chatAudioSsrc() const {
std::lock_guard<std::mutex> lk(d_->mu);
return d_->sh ? d_->sh->chatSsrc.load() : 0;
}
WebRtc::MicInfo WebRtc::micInfo() const {
std::lock_guard<std::mutex> lk(d_->mu);
return d_->mic;
}
WebRtc::MicTxStats WebRtc::micTxStats() const {
std::lock_guard<std::mutex> lk(d_->micMu);
MicTxStats st;
st.packets = d_->micPackets;
st.bytes = d_->micBytes;
st.failed = d_->micFailed;
return st;
}
std::string WebRtc::answerAudioDirection() const {
std::lock_guard<std::mutex> lk(d_->mu);
return d_->answerAudioDir;
}
bool WebRtc::sendMicOpus(const uint8_t* opus, size_t n, uint32_t samples48k, bool marker) {
if (!opus || n == 0 || n > 1200) return false;
std::shared_ptr<rtc::Track> audio;
uint32_t ssrc;
std::string cname;
{
std::lock_guard<std::mutex> lk(d_->mu);
audio = d_->audio;
ssrc = d_->mic.ssrc;
cname = d_->mic.cname;
}
if (!audio || ssrc == 0) return false;
const int64_t now = nowMs();
bool sendSr = false;
uint32_t srTs = 0, srPackets = 0, srOctets = 0;
bool ok = false;
{
std::lock_guard<std::mutex> lk(d_->micMu);
const uint16_t seq = d_->micSeq;
const uint32_t ts = d_->micTs;
d_->micTs += samples48k; // timestamps follow capture time even when a send fails
std::string why;
bool attempted = false;
try {
if (!audio->isOpen()) {
why = "track not open";
} else {
const std::vector<uint8_t> pkt = buildOpusRtp(seq, ts, ssrc, marker, opus, n);
attempted = true;
ok = audio->send(reinterpret_cast<const std::byte*>(pkt.data()), pkt.size());
if (!ok) why = "transport refused packet";
}
} catch (const std::exception& e) {
why = e.what();
}
if (attempted) ++d_->micSeq; // a refused packet is a loss the receiver may see as a gap
if (ok) {
++d_->micPackets;
d_->micBytes += n + 12;
d_->micPayloadBytes += n;
if (!d_->micFirstSent) {
d_->micFirstSent = true;
XC_LOGI("voice: first mic RTP sent ssrc=%08x seq=%u ts=%u bytes=%zu track=%s", ssrc, unsigned(seq),
unsigned(ts), n + 12, directionName(audio->direction()));
}
if (now - d_->micLastSrMs >= kMicSrIntervalMs) {
d_->micLastSrMs = now;
sendSr = true;
srTs = d_->micTs;
srPackets = static_cast<uint32_t>(d_->micPackets);
srOctets = static_cast<uint32_t>(d_->micPayloadBytes);
}
} else {
++d_->micFailed;
if (now - d_->micLastFailLogMs >= kMicLogIntervalMs) {
d_->micLastFailLogMs = now;
XC_LOGW("voice: mic RTP send failed (%s)", why.c_str());
}
}
if (now - d_->micLastStatsLogMs >= kMicLogIntervalMs) {
d_->micLastStatsLogMs = now;
XC_LOGI("voice: tx %llu pkts %llu bytes failed %llu", static_cast<unsigned long long>(d_->micPackets),
static_cast<unsigned long long>(d_->micBytes), static_cast<unsigned long long>(d_->micFailed));
}
}
if (sendSr) {
// RFC 3550 §6.4.1 SR (no report blocks) + §6.5 SDES CNAME for the mic SSRC, one compound
// packet through the existing RTCP path (video track).
const auto wall = std::chrono::system_clock::now().time_since_epoch();
const uint64_t us = static_cast<uint64_t>(std::chrono::duration_cast<std::chrono::microseconds>(wall).count());
const uint32_t ntpSec = static_cast<uint32_t>(us / 1000000u + 2208988800u);
const uint32_t ntpFrac = static_cast<uint32_t>(((us % 1000000u) << 32) / 1000000u);
std::vector<uint8_t> p;
p.reserve(64);
p.push_back(0x80);
p.push_back(200);
put16(p, 6);
put32(p, ssrc);
put32(p, ntpSec);
put32(p, ntpFrac);
put32(p, srTs);
put32(p, srPackets);
put32(p, srOctets);
const size_t sdes = p.size();
p.push_back(0x80 | 1);
p.push_back(202);
put16(p, 0); // patched below
put32(p, ssrc);
p.push_back(1); // CNAME
const size_t cl = std::min<size_t>(cname.size(), 255);
p.push_back(static_cast<uint8_t>(cl));
p.insert(p.end(), cname.begin(), cname.begin() + static_cast<std::ptrdiff_t>(cl));
p.push_back(0);
while ((p.size() - sdes) % 4) p.push_back(0);
set16(p, sdes + 2, static_cast<uint16_t>((p.size() - sdes) / 4 - 1));
d_->sendRtcp(p);
}
return ok;
}
bool WebRtc::beginRenegotiation(std::string& ufrag, std::string& pwd, std::string& fingerprint) {
ufrag.clear();
pwd.clear();
fingerprint.clear();
std::shared_ptr<rtc::PeerConnection> pc;
std::string iu, ip, ifp;
{
std::lock_guard<std::mutex> lk(d_->mu);
pc = d_->pc;
iu = d_->initUfrag;
ip = d_->initPwd;
ifp = d_->initFp;
}
if (!pc) {
XC_LOGE("voice: renegotiation without peer connection");
return false;
}
std::optional<rtc::Description> desc;
try {
if (!pc->remoteDescription()) {
XC_LOGE("voice: renegotiation before the first answer was applied");
return false;
}
if (pc->signalingState() != rtc::PeerConnection::SignalingState::Stable) {
XC_LOGE("voice: renegotiation in signaling state %d", static_cast<int>(pc->signalingState()));
return false;
}
pc->setLocalDescription(rtc::Description::Type::Offer);
{
std::lock_guard<std::mutex> lk(d_->mu);
if (d_->pc == pc) d_->renegPending = true;
}
desc = pc->localDescription();
} catch (const std::exception& e) {
XC_LOGE("voice: renegotiation setLocalDescription(offer) failed: %s", e.what());
return false;
}
if (!desc) {
XC_LOGE("voice: renegotiation produced no local description");
return false;
}
ufrag = desc->iceUfrag().value_or("");
pwd = desc->icePwd().value_or("");
if (auto fp = desc->fingerprint())
if (fp->algorithm == rtc::CertificateFingerprint::Algorithm::Sha256) fingerprint = fp->value;
const bool same = ufrag == iu && pwd == ip && fingerprint == ifp;
XC_LOGI("voice: renegotiation local offer ready (creds %s)", same ? "unchanged" : "CHANGED");
if (!same) XC_LOGW("voice: renegotiation ICE/DTLS credentials differ from the initial offer");
if (ufrag.empty() || pwd.empty() || fingerprint.empty()) {
XC_LOGE("voice: renegotiation local description lacks ICE credentials or sha-256 fingerprint");
return false;
}
return true;
}
bool WebRtc::finishRenegotiation(const std::string& answerSdp) {
std::shared_ptr<rtc::PeerConnection> pc;
std::shared_ptr<Shared> sh;
bool pending;
{
std::lock_guard<std::mutex> lk(d_->mu);
pc = d_->pc;
sh = d_->sh;
pending = d_->renegPending;
}
if (!pc || !sh) {
XC_LOGE("voice: renegotiation answer without peer connection");
return false;
}
if (!pending) {
XC_LOGE("voice: renegotiation answer without a pending renegotiation offer");
return false;
}
try {
if (pc->signalingState() != rtc::PeerConnection::SignalingState::HaveLocalOffer) {
XC_LOGE("voice: renegotiation answer in signaling state %d (no pending offer)",
static_cast<int>(pc->signalingState()));
return false;
}
pc->setRemoteDescription(rtc::Description(answerSdp, rtc::Description::Type::Answer));
} catch (const std::exception& e) {
XC_LOGE("voice: renegotiation setRemoteDescription failed: %s", e.what());
return false;
}
{
std::lock_guard<std::mutex> lk(d_->mu);
if (d_->pc == pc) d_->renegPending = false;
}
const AnswerInfo info = parseAnswer(answerSdp);
{
std::lock_guard<std::mutex> lk(sh->infoMu);
if (!info.videoPts.empty()) sh->videoPts = info.videoPts;
if (!info.audioPts.empty()) sh->audioPts = info.audioPts;
sh->answerVideoSsrcs = info.videoSsrcs;
sh->answerAudioSsrcs = info.audioSsrcs;
sh->routable.clear();
sh->routable.insert(info.videoSsrcs.begin(), info.videoSsrcs.end());
sh->routable.insert(info.audioSsrcs.begin(), info.audioSsrcs.end());
}
if (!info.videoSsrcs.empty() && sh->videoSsrc.load() == 0) sh->videoSsrc = *info.videoSsrcs.begin();
sh->setGameAudioFromAnswer(info.firstAudioSsrc); // no-op once the game SSRC is known
const std::string dir = proto::sdpMediaDirection(answerSdp, "audio");
{
std::lock_guard<std::mutex> lk(d_->mu);
if (d_->pc == pc) d_->answerAudioDir = dir;
}
XC_LOGI("webrtc: answer audio direction %s", dir.empty() ? "(none)" : dir.c_str());
XC_LOGI("voice: renegotiation answer applied (audio %s)", dir.empty() ? "(none)" : dir.c_str());
return true;
}
void WebRtc::abortRenegotiation() {
std::shared_ptr<rtc::PeerConnection> pc;
bool pending;
{
std::lock_guard<std::mutex> lk(d_->mu);
pc = d_->pc;
pending = d_->renegPending;
d_->renegPending = false;
}
if (!pc || !pending) {
XC_LOGD("voice: renegotiation rollback: no pending renegotiation offer");
return;
}
try {
const auto st = pc->signalingState();
if (st != rtc::PeerConnection::SignalingState::HaveLocalOffer) {
XC_LOGI("voice: renegotiation rollback done (nothing pending, state %d)", static_cast<int>(st));
return;
}
pc->setLocalDescription(rtc::Description::Type::Rollback);
if (pc->signalingState() == rtc::PeerConnection::SignalingState::Stable)
XC_LOGI("voice: renegotiation rollback done");
else
XC_LOGW("voice: renegotiation rollback failed: still in signaling state %d",
static_cast<int>(pc->signalingState()));
} catch (const std::exception& e) {
XC_LOGW("voice: renegotiation rollback failed: %s", e.what());
}
}
void WebRtc::close() {
if (!d_) return;
std::shared_ptr<Shared> sh;
@@ -824,6 +1284,9 @@ void WebRtc::close() {
channels.swap(d_->channels);
d_->rx.reset();
d_->offered = false;
d_->mic = MicInfo{};
d_->answerAudioDir.clear();
d_->renegPending = false;
}
if (sh) {
// Wait for a running callback to finish, then block all further ones. Must not be called
@@ -855,4 +1318,17 @@ void WebRtc::close() {
}
}
std::vector<uint8_t> buildOpusRtp(uint16_t seq, uint32_t ts, uint32_t ssrc, bool marker, const uint8_t* opus, size_t n,
uint8_t pt) {
std::vector<uint8_t> p;
p.reserve(12 + n);
p.push_back(0x80); // V=2, P=0, X=0, CC=0
p.push_back(static_cast<uint8_t>((marker ? 0x80 : 0x00) | (pt & 0x7F)));
put16(p, seq);
put32(p, ts);
put32(p, ssrc);
if (opus && n) p.insert(p.end(), opus, opus + n);
return p;
}
} // namespace xc
+53 -3
View File
@@ -1,4 +1,5 @@
// Nubix — libdatachannel wrapper: recvonly H.264 + Opus tracks (mids "video", "audio"),
// Nubix — libdatachannel wrapper: recvonly H.264 + Opus tracks (mids "video", "audio"; audio
// becomes sendrecv with an own SSRC when voice chat is on and carries the microphone as RTP),
// four reliable/ordered data channels (control/controlV1, input/1.0, message/messageV1,
// chat/chatV1, created before the offer so the data m-line gets mid "0"), raw RTP delivery
// and hand-built RTCP (PLI with sender SSRC 0, NACK, RR, REMB).
@@ -32,6 +33,11 @@ public:
std::function<void(const std::string& ch)> onChannelOpen;
// PeerConnection state changes: "new", "connecting", "connected", "disconnected", "failed", "closed".
std::function<void(const std::string& state)> onState;
// RTP (header included) of a PT-111 audio stream whose SSRC is NOT the game audio SSRC
// (party/chat voice delivered as a separate stream). libdatachannel thread.
std::function<void(const uint8_t*, size_t)> onChatAudioRtp;
// Data channel ch closed (by either side). libdatachannel thread.
std::function<void(const std::string& ch)> onChannelClosed;
};
WebRtc();
@@ -40,7 +46,46 @@ public:
WebRtc& operator=(const WebRtc&) = delete;
// Create the PeerConnection, tracks and data channels. Must be called first.
bool init(const Callbacks& cb);
// voice=false: audio track RecvOnly without SSRC (the original layout).
// voice=true: audio track SendRecv announcing a microphone SSRC (random, != the RTCP sender
// SSRC) with cname / msid / track id (see micInfo()); proto::buildOffer(..., &mic) must be
// given the same values so the hand-written offer agrees with libdatachannel.
bool init(const Callbacks& cb, bool voice = false);
// Microphone send-stream identity. ssrc 0 = voice off.
struct MicInfo {
uint32_t ssrc = 0;
std::string cname, msid, trackId;
};
MicInfo micInfo() const;
// Send one Opus frame as RTP on the audio track: V=2, PT 111, M=marker, seq++ (random start),
// ts += samples48k after this packet (random start), SSRC = mic SSRC, no extensions/CSRC.
// Raw send (no media handler on the audio track). Also sends an RTCP SR (+SDES CNAME) for the
// mic SSRC at most once per 1000 ms. Thread-safe (own mutex, never takes the receive-callback
// lock). False if voice off, track not open, or the send failed.
bool sendMicOpus(const uint8_t* opus, size_t n, uint32_t samples48k, bool marker);
struct MicTxStats {
uint64_t packets = 0, bytes = 0, failed = 0;
};
MicTxStats micTxStats() const;
// Direction of mid "audio" in the last applied remote answer ("" before setRemote).
std::string answerAudioDirection() const;
// Chat renegotiation (worker thread only, after setRemote succeeded):
// begin: setLocalDescription(Offer) again on the stable PeerConnection; returns the ICE ufrag,
// pwd and fingerprint of the new local description (logs whether they equal the initial ones).
bool beginRenegotiation(std::string& ufrag, std::string& pwd, std::string& fingerprint);
// finish: setRemoteDescription(answer) (no new candidates). Updates routable PTs, answer
// SSRCs and answerAudioDirection(). False on failure (logged); call abortRenegotiation() then.
bool finishRenegotiation(const std::string& answerSdp);
// abort: roll back the pending local offer (no-op when none is pending).
void abortRenegotiation();
// SSRC currently treated as chat/party voice (0 = none seen).
uint32_t chatAudioSsrc() const;
// Create the local offer, wait for ICE gathering (timeoutMs) and return the ICE credentials,
// sha-256 fingerprint and gathered candidate lines ("candidate:..."). The real local
@@ -72,7 +117,8 @@ public:
// Send RTCP REMB with the given max bitrate for the video SSRC.
void sendRemb(uint32_t bps);
// Media SSRCs latched from the first received RTP packet (0 if none yet).
// Media SSRCs latched from the first received RTP packet (0 if none yet). audioSsrc() is the
// game audio SSRC (answer a=ssrc, else first PT-111 packet; see chatAudioSsrc()).
uint32_t videoSsrc() const;
uint32_t audioSsrc() const;
@@ -86,4 +132,8 @@ private:
std::unique_ptr<Impl> d_;
};
// Pure helper (unit-tested): 12-byte RTP header (V=2, no padding/extension/CSRC) + payload.
std::vector<uint8_t> buildOpusRtp(uint16_t seq, uint32_t ts, uint32_t ssrc, bool marker, const uint8_t* opus, size_t n,
uint8_t pt = 111);
} // namespace xc
+87 -9
View File
@@ -280,6 +280,14 @@ public:
while (s.takeVibration(v))
if (!menuOpen_) pad.rumble(v);
// microphone: first Live of this stream -> tell the user it is on and how to mute it
micState_ = s.micState();
micLevel_ = micState_ == MicState::Live ? s.micLevel() : 0.0f;
if (micState_ == MicState::Live && !micToastShown_) {
micToastShown_ = true;
app.toast("Microphone on - mute it in the stream menu (Options + touchpad)");
}
// stats
++displayFrames_;
if (now - statsAt_ >= kStatsRefreshMs) {
@@ -319,6 +327,7 @@ public:
}
if (app.statsOverlay() && !ended_) renderStats(ui);
if (!ended_) renderMicIndicator(ui);
if (!ended_ && !menuOpen_ && now - enteredAt_ < kStartHintMs) {
const bool kb = app.gamepad().lastInputKeyboard();
@@ -337,7 +346,7 @@ public:
}
private:
enum MenuItem { ItemResume, ItemNexus, ItemStats, ItemKeyframe, ItemDisconnect, kMenuItems };
enum MenuItem { ItemResume, ItemNexus, ItemMic, ItemStats, ItemKeyframe, ItemDisconnect, kMenuItems };
void openMenu(App& app) {
menuOpen_ = true;
@@ -390,6 +399,18 @@ private:
closeMenu(app); // Cross is suppressed: the game gets Nexus alone
nexusUntil_ = platform::monotonicMs() + kNexusPulseMs;
break;
case ItemMic: {
// Unavailable / voice chat off: the item is shown greyed out and does nothing.
const MicState ms = app.streamer().micState();
if (ms != MicState::Live && ms != MicState::Muted) break;
const bool mute = !app.streamer().micMuted();
app.streamer().setMicMuted(mute);
micState_ = mute ? MicState::Muted : MicState::Live;
if (mute) micLevel_ = 0.0f;
XC_LOGI("stream: microphone %s by user", mute ? "muted" : "unmuted");
app.toast(mute ? "Microphone muted" : "Microphone on");
break; // the menu stays open
}
case ItemStats: app.setStatsOverlay(!app.statsOverlay()); break;
case ItemKeyframe:
app.streamer().requestKeyframe();
@@ -446,7 +467,7 @@ private:
}
void renderStats(Ui& ui) {
char l1[96], l2[96], l3[128], l4[64];
char l1[96], l2[96], l3[128], l4[64], l5[96];
const StreamStats& s = stats_;
std::snprintf(l1, sizeof(l1), "%d \xC3\x97 %d \xC2\xB7 %.1f fps", s.width ? s.width : texW_,
s.height ? s.height : texH_, s.fps);
@@ -457,19 +478,56 @@ private:
else
std::snprintf(l3, sizeof(l3), "decode %.1f ms \xC2\xB7 audio %d ms", s.decodeMs, s.audioBufferMs);
std::snprintf(l4, sizeof(l4), "display %.0f fps", displayFps_);
const char* lines[] = {l1, l2, l3, l4};
std::snprintf(l5, sizeof(l5), "mic %s %d kbps | chat rx %llu", micStateLabel(micState_), s.micTxKbps,
static_cast<unsigned long long>(s.chatRxPackets));
const char* lines[] = {l1, l2, l3, l4, l5};
constexpr int n = static_cast<int>(sizeof(lines) / sizeof(lines[0]));
int w = 0;
for (auto* l : lines) w = std::max(w, ui.textWidth(l, 22));
const int lh = ui.lineHeight(22);
ui.rect(28, 28, w + 48, lh * 4 + 32, withAlpha(colors::Black, 160), true, 14);
ui.rect(28, 44, 4, lh * 4, colors::AccentBright, true, 2);
ui.rect(28, 28, w + 48, lh * n + 32, withAlpha(colors::Black, 160), true, 14);
ui.rect(28, 44, 4, lh * n, colors::AccentBright, true, 2);
int y = 44;
for (int i = 0; i < 4; ++i) {
for (int i = 0; i < n; ++i) {
ui.text(lines[i], 52, y, 22, i == 0 ? colors::White : colors::Text, Align::Left, i == 0);
y += lh;
}
}
static const char* micStateLabel(MicState m) {
switch (m) {
case MicState::Off: return "off";
case MicState::Unavailable: return "unavailable";
case MicState::Muted: return "muted";
case MicState::Live: return "live";
}
return "?";
}
// Top-right microphone badge while voice chat runs: white capsule + green level bar when
// live, grey capsule with a red slash when muted. Hidden in every other state.
void renderMicIndicator(Ui& ui) {
if (micState_ != MicState::Live && micState_ != MicState::Muted) return;
const bool live = micState_ == MicState::Live;
constexpr int kBox = 40, kMargin = 16;
const int bx = Ui::kWidth - kMargin - kBox, by = kMargin;
ui.rect(bx, by, kBox, kBox, withAlpha(colors::Black, 150), true, 10);
const Color cap = live ? colors::White : colors::TextDim;
// capsule 12x20, centred horizontally, a little above centre to leave room for the stand
const int cw = 12, ch = 20, cx = bx + (kBox - cw) / 2, cy = by + 6;
ui.rect(cx, cy, cw, ch, cap, true, 6);
if (live) {
// level fill from the bottom of the capsule, height = level x 20
const float lv = std::max(0.0f, std::min(1.0f, micLevel_));
const int lh = static_cast<int>(std::lround(lv * ch));
if (lh > 0) ui.rect(cx + 2, cy + ch - lh, cw - 4, lh, Color{60, 200, 90, 255}, true, 4);
}
// stand: short stem + base bar
ui.rect(bx + kBox / 2 - 1, cy + ch + 1, 2, 5, cap);
ui.rect(bx + kBox / 2 - 6, cy + ch + 6, 12, 2, cap, true, 1);
if (!live) ui.line(bx + 9.0f, by + kBox - 9.0f, bx + kBox - 9.0f, by + 9.0f, 3.0f, colors::Error);
}
void renderMenu(App& app, Ui& ui) {
ui.rect(0, 0, Ui::kWidth, Ui::kHeight, withAlpha(colors::Black, 150));
const int w = 640, itemH = 76, gap = 12;
@@ -487,19 +545,36 @@ private:
int iy = y + 150;
for (int i = 0; i < kMenuItems; ++i) {
std::string label;
bool dim = false; // shown but inactive
switch (i) {
case ItemResume: label = "Resume"; break;
case ItemNexus: label = "Press Xbox button"; break;
case ItemMic:
switch (micState_) {
case MicState::Live: label = "Mute microphone"; break;
case MicState::Muted: label = "Unmute microphone"; break;
case MicState::Unavailable:
label = "Microphone unavailable";
dim = true;
break;
case MicState::Off:
label = "Voice chat off";
dim = true;
break;
}
break;
case ItemStats: label = app.statsOverlay() ? "Hide statistics" : "Show statistics"; break;
case ItemKeyframe: label = "Refresh video"; break;
case ItemDisconnect: label = "Disconnect"; break;
}
const bool f = i == menuFocus_;
if (f) {
ui.rect(x + 32, iy, w - 64, itemH, i == ItemDisconnect ? Color{150, 45, 40, 255} : colors::Accent, true, 14);
const Color bg = dim ? colors::PanelHi : (i == ItemDisconnect ? Color{150, 45, 40, 255} : colors::Accent);
ui.rect(x + 32, iy, w - 64, itemH, bg, true, 14);
}
ui.text(label, x + 64, iy + (itemH - ui.lineHeight(28, f)) / 2, 28,
f ? colors::White : (i == ItemDisconnect ? Color{255, 140, 130, 255} : colors::Text), Align::Left, f);
const Color fg = dim ? colors::TextDim
: (f ? colors::White : (i == ItemDisconnect ? Color{255, 140, 130, 255} : colors::Text));
ui.text(label, x + 64, iy + (itemH - ui.lineHeight(28, f)) / 2, 28, fg, Align::Left, f);
iy += itemH + gap;
}
ui::hintBar(app, ui, {{PadIcon::Cross, "Select"}, {PadIcon::Circle, "Resume"}});
@@ -537,6 +612,9 @@ private:
std::string endMsg_;
uint64_t enteredAt_ = 0;
MicState micState_ = MicState::Off;
float micLevel_ = 0.0f;
bool micToastShown_ = false; // "Microphone on" toast: once per stream screen
StreamStats stats_;
uint64_t statsAt_ = 0;
uint64_t videoFrames_ = 0;
+13 -2
View File
@@ -225,7 +225,7 @@ std::vector<std::string> regionChoices(const Tokens& t) {
const char* const kResolutions[] = {"720", "1080", "1080HQ"};
enum Row { RowResolution, RowBitrate, RowRegion, RowF2p, RowDeadzone, RowStats, RowSignOut, RowExit, RowCount };
enum Row { RowResolution, RowBitrate, RowRegion, RowF2p, RowDeadzone, RowVoice, RowStats, RowSignOut, RowExit, RowCount };
} // namespace
@@ -277,6 +277,7 @@ struct SettingsPanel::Impl {
case RowRegion: return "Server region";
case RowF2p: return "Free-to-play fallback";
case RowDeadzone: return "Extra stick deadzone";
case RowVoice: return "Voice chat";
case RowStats: return "Show stream statistics";
case RowSignOut: return "Sign out";
case RowExit: return "Exit Nubix";
@@ -307,6 +308,7 @@ struct SettingsPanel::Impl {
if (s.stickDeadzone <= 0) return "Off";
std::snprintf(buf, sizeof(buf), "%d %%", s.stickDeadzone);
return buf;
case RowVoice: return s.voiceChat ? "On" : "Off";
case RowStats: return app.statsOverlay() ? "On" : "Off";
default: return "";
}
@@ -319,6 +321,9 @@ struct SettingsPanel::Impl {
case RowRegion: return "Automatic uses the region Xbox assigns to your account.";
case RowF2p: return "Retry with the free-to-play offering when a title is not in your Game Pass library.";
case RowDeadzone: return "Ignore small stick movements (helps with drifting sticks).";
case RowVoice:
return "Use your headset or controller microphone for Xbox party and game chat. Applies to the next "
"stream.";
case RowStats: return "Resolution, frame rate, bitrate and decode time while streaming.";
case RowSignOut: return "Forget the stored Microsoft account on this console.";
case RowExit: return "Close the app.";
@@ -381,6 +386,11 @@ struct SettingsPanel::Impl {
save(app);
break;
}
case RowVoice:
s.voiceChat = !s.voiceChat;
commit();
save(app);
break;
case RowStats: app.setStatsOverlay(!app.statsOverlay()); break;
case RowSignOut:
if (dir != 0) break;
@@ -421,7 +431,8 @@ bool SettingsPanel::onNav(App& app, NavKey key) {
void SettingsPanel::render(App& app, Ui& ui, int top) {
d_->refresh(app);
const int x = 360, w = 1200, h = 74, gap = 8;
// Row height keeps all rows, the help line and the account line above the hint bar (y 1022).
const int x = 360, w = 1200, h = 66, gap = 8;
int y = top;
for (int row = 0; row < RowCount; ++row) {
const bool f = row == d_->focus;
+37
View File
@@ -121,3 +121,40 @@ XC_TEST(config, gssv_device_tier_and_auth_rejection) {
XC_CHECK(!gssv::isAuthRejection(400, ""));
XC_CHECK(!gssv::isAuthRejection(200, ""));
}
// Voice chat is on by default, persists when turned off, and an older config file without the
// key keeps the default (on).
XC_TEST(config, voice_chat_setting) {
XC_CHECK(Settings().voiceChat);
const std::string path = tempPath();
{
Config c(path);
XC_REQUIRE(c.load());
XC_CHECK(c.settings().voiceChat);
c.updateSettings([](Settings& s) { s.voiceChat = false; });
XC_REQUIRE(c.save());
}
{
Config d(path);
XC_REQUIRE(d.load());
XC_CHECK(!d.settings().voiceChat);
d.updateSettings([](Settings& s) { s.voiceChat = true; });
XC_REQUIRE(d.save());
}
{
Config e(path);
XC_REQUIRE(e.load());
XC_CHECK(e.settings().voiceChat);
}
{
FILE* f = std::fopen(path.c_str(), "wb");
XC_REQUIRE(f != nullptr);
std::fputs("{\"version\":1,\"settings\":{\"resolution\":\"720\",\"f2pFallback\":false}}", f);
std::fclose(f);
Config g(path);
XC_REQUIRE(g.load());
XC_CHECK_EQ(g.settings().resolution, std::string("720"));
XC_CHECK(g.settings().voiceChat);
}
std::remove(path.c_str());
}
+318
View File
@@ -0,0 +1,318 @@
// Nubix — voice chat capture/encoder and chat-voice mixing tests.
// Copyright (C) 2026 Nubix contributors
// SPDX-License-Identifier: GPL-3.0-only
#include <opus.h>
#include <algorithm>
#include <atomic>
#include <chrono>
#include <climits>
#include <cmath>
#include <cstdlib>
#include <functional>
#include <mutex>
#include <thread>
#include <vector>
#include "platform/audio_in.hpp"
#include "stream/audio.hpp"
#include "stream/voice.hpp"
#include "test.hpp"
using namespace std::chrono;
namespace {
// Real-time paced sine source (like a device that blocks for one grain).
class FakeAudioIn final : public xc::platform::AudioIn {
public:
FakeAudioIn(int rate, int grain, bool failOpen = false, int failReadsAfter = -1)
: rate_(rate), grain_(grain), failOpen_(failOpen), failReadsAfter_(failReadsAfter) {}
bool open(int& rate, int& grain) override {
if (failOpen_) return false;
rate = rate_;
grain = grain_;
next_ = steady_clock::now();
open_ = true;
return true;
}
int read(int16_t* buf, int) override {
if (!open_) return -1;
if (failReadsAfter_ >= 0 && reads_++ >= failReadsAfter_) return -1;
std::this_thread::sleep_until(next_);
next_ += microseconds(1000000LL * grain_ / rate_);
for (int i = 0; i < grain_; ++i) {
buf[i] = static_cast<int16_t>(8000.0 * std::sin(phase_));
phase_ += 2.0 * M_PI * 440.0 / rate_;
}
return grain_;
}
uint32_t silentState() override { return 0; }
void close() override { open_ = false; }
const char* backend() const override { return "fake"; }
private:
int rate_, grain_;
bool failOpen_;
int failReadsAfter_;
int reads_ = 0;
bool open_ = false;
double phase_ = 0.0;
steady_clock::time_point next_{};
};
struct Captured {
std::mutex mu;
std::vector<std::vector<uint8_t>> frames;
std::vector<uint32_t> samples;
std::vector<bool> markers;
xc::VoiceChat::FrameSink sink() {
return [this](const uint8_t* p, size_t n, uint32_t s48, bool m) {
std::lock_guard<std::mutex> lk(mu);
frames.emplace_back(p, p + n);
samples.push_back(s48);
markers.push_back(m);
};
}
size_t count() {
std::lock_guard<std::mutex> lk(mu);
return frames.size();
}
};
bool waitFor(const std::function<bool()>& pred, int ms) {
const auto end = steady_clock::now() + milliseconds(ms);
while (steady_clock::now() < end) {
if (pred()) return true;
std::this_thread::sleep_for(milliseconds(5));
}
return pred();
}
// Decode every frame (48 kHz mono) and return the peak of frames [skip, end).
int decodedPeak(const std::vector<std::vector<uint8_t>>& frames, size_t skip, bool& allOk) {
int err = 0;
OpusDecoder* dec = opus_decoder_create(48000, 1, &err);
allOk = dec != nullptr;
int peak = 0;
std::vector<int16_t> pcm(5760);
for (size_t i = 0; dec && i < frames.size(); ++i) {
const int got = opus_decode(dec, frames[i].data(), static_cast<opus_int32>(frames[i].size()), pcm.data(), 5760, 0);
if (got != 960) allOk = false;
if (i >= skip)
for (int k = 0; k < std::max(got, 0); ++k) peak = std::max(peak, std::abs(static_cast<int>(pcm[k])));
}
if (dec) opus_decoder_destroy(dec);
return peak;
}
} // namespace
XC_TEST(voice, frames_cadence_marker_and_decode) {
Captured cap;
xc::VoiceChat v(std::make_unique<FakeAudioIn>(16000, 256));
XC_REQUIRE(v.start(cap.sink(), false));
XC_CHECK(!v.start(cap.sink(), false)); // already running
XC_CHECK(waitFor([&] { return v.state() == xc::VoiceChat::State::Live; }, 500));
std::this_thread::sleep_for(milliseconds(1300)); // > one 1 s kbps window
const float lvl = v.level();
const auto st = v.stats();
v.stop();
XC_CHECK(v.state() == xc::VoiceChat::State::Off);
std::lock_guard<std::mutex> lk(cap.mu);
// ~50 frames per second over ~1.3 s.
XC_CHECK(cap.frames.size() >= 58 && cap.frames.size() <= 75);
XC_REQUIRE(!cap.frames.empty());
XC_CHECK(cap.markers.front());
XC_CHECK(std::count(cap.markers.begin(), cap.markers.end(), true) == 1);
XC_CHECK(std::all_of(cap.samples.begin(), cap.samples.end(), [](uint32_t s) { return s == 960; }));
bool ok = false;
const int peak = decodedPeak(cap.frames, 5, ok);
XC_CHECK(ok);
XC_CHECK(peak > 2000);
XC_CHECK(lvl > 0.1f && lvl <= 1.0f);
XC_CHECK_EQ(st.backend, std::string("fake"));
XC_CHECK_EQ(st.rate, 16000);
XC_CHECK(st.kbps > 10 && st.kbps < 60);
}
XC_TEST(voice, muted_sends_silence) {
Captured cap;
xc::VoiceChat v(std::make_unique<FakeAudioIn>(16000, 256));
XC_REQUIRE(v.start(cap.sink(), true));
XC_CHECK(v.muted());
XC_CHECK(waitFor([&] { return v.state() == xc::VoiceChat::State::Muted; }, 500));
std::this_thread::sleep_for(milliseconds(400));
XC_CHECK(v.level() == 0.0f);
v.stop();
std::lock_guard<std::mutex> lk(cap.mu);
XC_CHECK(cap.frames.size() >= 15);
bool ok = false;
const int peak = decodedPeak(cap.frames, 0, ok);
XC_CHECK(ok);
XC_CHECK(peak < 64);
}
XC_TEST(voice, mute_toggle_while_live) {
Captured cap;
xc::VoiceChat v(std::make_unique<FakeAudioIn>(16000, 256));
XC_REQUIRE(v.start(cap.sink(), false));
XC_CHECK(waitFor([&] { return v.state() == xc::VoiceChat::State::Live; }, 500));
v.setMuted(true);
XC_CHECK(waitFor([&] { return v.state() == xc::VoiceChat::State::Muted; }, 200));
XC_CHECK(v.level() == 0.0f);
const size_t before = cap.count();
std::this_thread::sleep_for(milliseconds(200));
XC_CHECK(cap.count() > before + 5); // cadence continues while muted
v.setMuted(false);
XC_CHECK(waitFor([&] { return v.state() == xc::VoiceChat::State::Live; }, 200));
v.stop();
}
XC_TEST(voice, open_failure_is_unavailable) {
Captured cap;
xc::VoiceChat v(std::make_unique<FakeAudioIn>(16000, 256, true));
XC_REQUIRE(v.start(cap.sink(), false));
XC_CHECK(waitFor([&] { return v.state() == xc::VoiceChat::State::Unavailable; }, 500));
std::this_thread::sleep_for(milliseconds(50));
XC_CHECK_EQ(cap.count(), static_cast<size_t>(0));
XC_CHECK(v.level() == 0.0f);
v.stop();
XC_CHECK(v.state() == xc::VoiceChat::State::Off);
}
XC_TEST(voice, read_errors_end_unavailable) {
Captured cap;
xc::VoiceChat v(std::make_unique<FakeAudioIn>(16000, 256, false, 10));
XC_REQUIRE(v.start(cap.sink(), false));
XC_CHECK(waitFor([&] { return v.state() == xc::VoiceChat::State::Unavailable; }, 1000));
XC_CHECK(v.stats().readErrors >= 5);
// The ended thread can be restarted (fake keeps failing -> Unavailable again).
XC_CHECK(v.start(cap.sink(), false));
v.stop();
}
XC_TEST(voice, stop_is_fast_and_idempotent) {
Captured cap;
xc::VoiceChat v(std::make_unique<FakeAudioIn>(16000, 256));
XC_REQUIRE(v.start(cap.sink(), false));
std::this_thread::sleep_for(milliseconds(150));
const auto t0 = steady_clock::now();
v.stop();
const auto ms = duration_cast<milliseconds>(steady_clock::now() - t0).count();
XC_CHECK(ms < 200);
v.stop();
XC_CHECK(v.state() == xc::VoiceChat::State::Off);
// Restart after stop.
XC_CHECK(v.start(cap.sink(), false));
v.stop();
}
XC_TEST(voice, non_opus_rate_is_resampled) {
Captured cap;
xc::VoiceChat v(std::make_unique<FakeAudioIn>(44100, 441));
XC_REQUIRE(v.start(cap.sink(), false));
std::this_thread::sleep_for(milliseconds(500));
const auto st = v.stats();
v.stop();
XC_CHECK_EQ(st.rate, 48000);
std::lock_guard<std::mutex> lk(cap.mu);
XC_CHECK(cap.frames.size() >= 20 && cap.frames.size() <= 32);
bool ok = false;
XC_CHECK(decodedPeak(cap.frames, 5, ok) > 2000);
XC_CHECK(ok);
}
XC_TEST(voice, state_names) {
XC_CHECK_EQ(std::string(xc::voiceStateName(xc::VoiceChat::State::Off)), std::string("off"));
XC_CHECK_EQ(std::string(xc::voiceStateName(xc::VoiceChat::State::Starting)), std::string("starting"));
XC_CHECK_EQ(std::string(xc::voiceStateName(xc::VoiceChat::State::Unavailable)), std::string("unavailable"));
XC_CHECK_EQ(std::string(xc::voiceStateName(xc::VoiceChat::State::Muted)), std::string("muted"));
XC_CHECK_EQ(std::string(xc::voiceStateName(xc::VoiceChat::State::Live)), std::string("live"));
}
XC_TEST(voice, mix_saturates) {
int16_t dst[6] = {1000, 30000, -30000, INT16_MAX, INT16_MIN, -5};
const int16_t src[6] = {-500, 5000, -5000, 1, -1, 5};
xc::AudioPlayer::mixSaturate(dst, src, 6);
XC_CHECK_EQ(dst[0], static_cast<int16_t>(500));
XC_CHECK_EQ(dst[1], static_cast<int16_t>(INT16_MAX));
XC_CHECK_EQ(dst[2], static_cast<int16_t>(INT16_MIN));
XC_CHECK_EQ(dst[3], static_cast<int16_t>(INT16_MAX));
XC_CHECK_EQ(dst[4], static_cast<int16_t>(INT16_MIN));
XC_CHECK_EQ(dst[5], static_cast<int16_t>(0));
}
XC_TEST(voice, player_voice_api_without_device) {
// Not initialised: pushes are ignored, stats stay zero, resets are safe.
xc::AudioPlayer p;
const uint8_t junk[3] = {0xf8, 0xff, 0xfe};
p.pushVoiceOpus(junk, sizeof(junk), 2);
p.resetVoice();
p.reset();
const auto vs = p.voiceStats();
XC_CHECK_EQ(vs.packets, static_cast<uint64_t>(0));
XC_CHECK_EQ(vs.bufferedMs, 0);
}
XC_TEST(voice, host_source_env) {
#ifndef XC_PS5
setenv("XC_MIC_SOURCE", "none", 1);
{
auto s = xc::platform::createAudioIn();
int r = 0, g = 0;
XC_CHECK_EQ(std::string(s->backend()), std::string("none"));
XC_CHECK(!s->open(r, g));
}
setenv("XC_MIC_SOURCE", "tone", 1);
{
auto s = xc::platform::createAudioIn();
int r = 0, g = 0;
XC_REQUIRE(s->open(r, g));
XC_CHECK_EQ(r, 48000);
XC_CHECK_EQ(g, 480);
std::vector<int16_t> buf(static_cast<size_t>(g));
XC_CHECK_EQ(s->read(buf.data(), 50), g);
int peak = 0;
for (int16_t x : buf) peak = std::max(peak, std::abs(static_cast<int>(x)));
XC_CHECK(peak > 3000 && peak < 3400); // -20 dBFS
s->close();
}
unsetenv("XC_MIC_SOURCE");
#endif
}
XC_TEST(voice, player_voice_ring_cap) {
#ifndef XC_PS5
// SDL's dummy driver drains the device in real time without touching hardware.
setenv("SDL_AUDIODRIVER", "dummy", 1);
xc::AudioPlayer p;
const bool ok = p.init();
unsetenv("SDL_AUDIODRIVER");
if (!ok) return; // no SDL audio at all in this environment
int err = 0;
OpusEncoder* enc = opus_encoder_create(48000, 1, OPUS_APPLICATION_VOIP, &err);
XC_REQUIRE(enc != nullptr);
std::vector<int16_t> pcm(960);
for (int i = 0; i < 960; ++i) pcm[static_cast<size_t>(i)] = static_cast<int16_t>(6000.0 * std::sin(i * 0.057));
uint8_t pkt[400];
const int n = opus_encode(enc, pcm.data(), 960, pkt, sizeof(pkt));
XC_REQUIRE(n > 0);
// 400 ms burst: the cap (120 ms) trims back to 60 ms, so latency stays bounded.
for (int i = 0; i < 20; ++i) p.pushVoiceOpus(pkt, static_cast<size_t>(n), i == 10 ? 1 : 0);
const auto vs = p.voiceStats();
XC_CHECK_EQ(vs.packets, static_cast<uint64_t>(20));
XC_CHECK_EQ(vs.lost, static_cast<uint64_t>(1));
XC_CHECK(vs.bufferedMs <= 120);
XC_CHECK(vs.droppedMs >= 200);
// Game stream untouched by voice pushes.
XC_CHECK_EQ(p.stats().packets, static_cast<uint64_t>(0));
p.reset();
XC_CHECK_EQ(p.voiceStats().bufferedMs, 0);
opus_encoder_destroy(enc);
p.close();
#endif
}
+239
View File
@@ -0,0 +1,239 @@
// Nubix — voice chat end-to-end loopback test (local libdatachannel "server" peer).
// Copyright (C) 2026 Nubix contributors
// SPDX-License-Identifier: GPL-3.0-only
//
// voice chat end-to-end over a local libdatachannel "server" peer: mic RTP upstream,
// game vs chat SSRC demux downstream, chat renegotiation (apply + rollback).
// Needs working local UDP (host ICE candidates); run with: xc-tests voice_loopback
#include <rtc/rtc.hpp>
#include <atomic>
#include <chrono>
#include <functional>
#include <mutex>
#include <thread>
#include <vector>
#include "core/protocol.hpp"
#include "stream/webrtc.hpp"
#include "test.hpp"
using namespace xc;
namespace {
bool waitFor(const std::function<bool()>& cond, int timeoutMs) {
const auto end = std::chrono::steady_clock::now() + std::chrono::milliseconds(timeoutMs);
while (std::chrono::steady_clock::now() < end) {
if (cond()) return true;
std::this_thread::sleep_for(std::chrono::milliseconds(10));
}
return cond();
}
uint32_t ssrcOf(const std::vector<uint8_t>& p) {
return p.size() >= 12 ? (uint32_t(p[8]) << 24) | (uint32_t(p[9]) << 16) | (uint32_t(p[10]) << 8) | p[11] : 0;
}
// Plays the console: answers our offers, sends audio RTP, records what arrives on its audio track.
struct FakeServer {
std::shared_ptr<rtc::PeerConnection> pc;
std::mutex mu;
std::shared_ptr<rtc::Track> audio;
std::vector<std::vector<uint8_t>> rx; // RTP received on the audio track
std::vector<std::shared_ptr<rtc::DataChannel>> channels;
std::atomic<bool> gathered{false};
FakeServer() {
rtc::Configuration cfg; // host candidates only, auto-negotiation on (answers offers itself)
pc = std::make_shared<rtc::PeerConnection>(cfg);
pc->onGatheringStateChange([this](rtc::PeerConnection::GatheringState st) {
if (st == rtc::PeerConnection::GatheringState::Complete) gathered = true;
});
pc->onTrack([this](std::shared_ptr<rtc::Track> t) {
if (t->mid() != "audio") return;
t->onMessage(
[this](rtc::binary b) {
const auto* d = reinterpret_cast<const uint8_t*>(b.data());
if (b.size() < 12 || ((d[1] & 0x7F) >= 64 && (d[1] & 0x7F) <= 95)) return; // RTP only
std::lock_guard<std::mutex> lk(mu);
rx.emplace_back(d, d + b.size());
},
[](rtc::string) {});
std::lock_guard<std::mutex> lk(mu);
audio = t;
});
pc->onDataChannel([this](std::shared_ptr<rtc::DataChannel> dc) {
std::lock_guard<std::mutex> lk(mu);
channels.push_back(dc);
});
}
~FakeServer() {
try {
pc->resetCallbacks();
pc->close();
} catch (...) {
}
}
std::string answer(const std::string& offerSdp) {
pc->setRemoteDescription(rtc::Description(offerSdp, rtc::Description::Type::Offer));
waitFor([this] {
return gathered.load() || pc->gatheringState() == rtc::PeerConnection::GatheringState::Complete;
}, 3000);
auto d = pc->localDescription();
return d ? std::string(*d) : std::string();
}
std::shared_ptr<rtc::DataChannel> channel(const std::string& label) {
std::lock_guard<std::mutex> lk(mu);
for (auto& c : channels)
if (c->label() == label) return c;
return nullptr;
}
std::shared_ptr<rtc::Track> track() {
std::lock_guard<std::mutex> lk(mu);
return audio;
}
size_t received() {
std::lock_guard<std::mutex> lk(mu);
return rx.size();
}
};
} // namespace
XC_TEST(voice_loopback, mic_up_chat_down_and_renegotiation) {
std::mutex mu;
std::vector<uint32_t> gameSsrcs, chatSsrcs;
std::vector<std::string> closed;
WebRtc::Callbacks cb;
cb.onAudioRtp = [&](const uint8_t* d, size_t n) {
std::lock_guard<std::mutex> lk(mu);
gameSsrcs.push_back(ssrcOf(std::vector<uint8_t>(d, d + n)));
};
cb.onChatAudioRtp = [&](const uint8_t* d, size_t n) {
std::lock_guard<std::mutex> lk(mu);
chatSsrcs.push_back(ssrcOf(std::vector<uint8_t>(d, d + n)));
};
cb.onChannelClosed = [&](const std::string& ch) {
std::lock_guard<std::mutex> lk(mu);
closed.push_back(ch);
};
WebRtc rtc;
XC_REQUIRE(rtc.init(cb, true));
const WebRtc::MicInfo mic = rtc.micInfo();
XC_REQUIRE(mic.ssrc != 0);
std::string u, p, fp;
std::vector<std::string> cands;
rtc.gatherLocal(u, p, fp, cands, 3000);
XC_REQUIRE(!u.empty() && !fp.empty());
FakeServer server;
const std::string answer = server.answer(rtc.localDescriptionSdp());
XC_REQUIRE(!answer.empty());
XC_REQUIRE(rtc.setRemote(answer, {}));
XC_CHECK_EQ(rtc.answerAudioDirection(), std::string("sendrecv"));
XC_REQUIRE(waitFor([&] { auto t = server.track(); return t && t->isOpen(); }, 10000));
// Our track opens with the same DTLS transport; give it a moment.
const uint8_t opus[] = {0xF8, 0xFF, 0xFE};
XC_REQUIRE(waitFor([&] { return rtc.sendMicOpus(opus, sizeof(opus), 960, true); }, 5000));
for (int i = 0; i < 9; ++i) XC_CHECK(rtc.sendMicOpus(opus, sizeof(opus), 960, false));
XC_REQUIRE(waitFor([&] { return server.received() >= 10; }, 5000));
{
std::lock_guard<std::mutex> lk(server.mu);
const auto& first = server.rx.front();
XC_CHECK_EQ(ssrcOf(first), mic.ssrc);
XC_CHECK_EQ(int(first[1] & 0x7F), 111);
XC_CHECK(first[1] & 0x80); // marker on the first frame
XC_CHECK(!(server.rx[1][1] & 0x80)); // not on the next
const uint16_t s0 = uint16_t(first[2] << 8 | first[3]), s1 = uint16_t(server.rx[1][2] << 8 | server.rx[1][3]);
XC_CHECK_EQ(uint16_t(s0 + 1), s1);
const uint32_t t0 = (uint32_t(first[4]) << 24) | (uint32_t(first[5]) << 16) | (uint32_t(first[6]) << 8) | first[7];
const uint32_t t1 = (uint32_t(server.rx[1][4]) << 24) | (uint32_t(server.rx[1][5]) << 16) |
(uint32_t(server.rx[1][6]) << 8) | server.rx[1][7];
XC_CHECK_EQ(uint32_t(t1 - t0), 960u);
XC_CHECK_EQ(first.size(), size_t(15));
}
XC_CHECK(rtc.micTxStats().packets >= 10);
// Downstream: game SSRC first, then a second SSRC = chat voice.
auto st = server.track();
XC_REQUIRE(st);
const uint32_t game = 0x11111111u, chat = 0x22222222u;
uint16_t gseq = 100, cseq = 500;
for (int i = 0; i < 5; ++i) {
auto pkt = buildOpusRtp(gseq++, 960u * i, game, false, opus, sizeof(opus));
st->send(reinterpret_cast<const std::byte*>(pkt.data()), pkt.size());
}
XC_REQUIRE(waitFor([&] { std::lock_guard<std::mutex> lk(mu); return gameSsrcs.size() >= 5; }, 5000));
for (int i = 0; i < 5; ++i) {
auto g = buildOpusRtp(gseq++, 960u * (5 + i), game, false, opus, sizeof(opus));
st->send(reinterpret_cast<const std::byte*>(g.data()), g.size());
auto c = buildOpusRtp(cseq++, 960u * i, chat, false, opus, sizeof(opus));
st->send(reinterpret_cast<const std::byte*>(c.data()), c.size());
}
XC_REQUIRE(waitFor([&] { std::lock_guard<std::mutex> lk(mu); return chatSsrcs.size() >= 5 && gameSsrcs.size() >= 10; }, 5000));
{
std::lock_guard<std::mutex> lk(mu);
for (uint32_t s : gameSsrcs) XC_CHECK_EQ(s, game);
for (uint32_t s : chatSsrcs) XC_CHECK_EQ(s, chat);
}
XC_CHECK_EQ(rtc.audioSsrc(), game);
XC_CHECK_EQ(rtc.chatAudioSsrc(), chat);
// Renegotiation, rolled back first ...
std::string ru, rp, rfp;
XC_REQUIRE(rtc.beginRenegotiation(ru, rp, rfp));
XC_CHECK_EQ(ru, u);
XC_CHECK_EQ(rp, p);
XC_CHECK_EQ(rfp, fp);
rtc.abortRenegotiation();
// ... then applied for real.
XC_REQUIRE(rtc.beginRenegotiation(ru, rp, rfp));
const std::string offer2 = rtc.localDescriptionSdp();
XC_CHECK_EQ(proto::sdpMediaDirection(offer2, "audio"), std::string("sendrecv"));
const std::string answer2 = server.answer(offer2);
XC_REQUIRE(!answer2.empty());
XC_REQUIRE(rtc.finishRenegotiation(answer2));
XC_CHECK_EQ(rtc.answerAudioDirection(), std::string("sendrecv"));
XC_CHECK(!rtc.finishRenegotiation(answer2)); // nothing pending any more
// Media keeps flowing both ways after the renegotiation.
const size_t before = server.received();
for (int i = 0; i < 5; ++i) XC_CHECK(rtc.sendMicOpus(opus, sizeof(opus), 960, false));
XC_CHECK(waitFor([&] { return server.received() >= before + 5; }, 5000));
const size_t gameBefore = [&] { std::lock_guard<std::mutex> lk(mu); return gameSsrcs.size(); }();
for (int i = 0; i < 3; ++i) {
auto g = buildOpusRtp(gseq++, 960u * (10 + i), game, false, opus, sizeof(opus));
st->send(reinterpret_cast<const std::byte*>(g.data()), g.size());
}
XC_CHECK(waitFor([&] { std::lock_guard<std::mutex> lk(mu); return gameSsrcs.size() >= gameBefore + 3; }, 5000));
// Game SSRC silent for > 2 s while another stream flows: the new one takes over as game.
std::this_thread::sleep_for(std::chrono::milliseconds(2100));
const uint32_t game2 = 0x33333333u;
for (int i = 0; i < 3; ++i) {
auto g = buildOpusRtp(uint16_t(900 + i), 960u * i, game2, false, opus, sizeof(opus));
st->send(reinterpret_cast<const std::byte*>(g.data()), g.size());
}
XC_CHECK(waitFor([&] {
std::lock_guard<std::mutex> lk(mu);
return !gameSsrcs.empty() && gameSsrcs.back() == game2;
}, 5000));
XC_CHECK_EQ(rtc.audioSsrc(), game2);
// The server closing a data channel (as xCloud does with "chat") reaches onChannelClosed.
XC_REQUIRE(waitFor([&] { return server.channel("chat") != nullptr; }, 5000));
server.channel("chat")->close();
XC_CHECK(waitFor([&] {
std::lock_guard<std::mutex> lk(mu);
for (const auto& c : closed)
if (c == "chat") return true;
return false;
}, 5000));
XC_CHECK(rtc.isChannelOpen("control"));
rtc.close();
XC_CHECK_EQ(rtc.micInfo().ssrc, 0u);
}
+293
View File
@@ -0,0 +1,293 @@
// Nubix — voice chat signalling tests: mic offer, chatStream bodies, exchange fields, RTP builder.
// Copyright (C) 2026 Nubix contributors
// SPDX-License-Identifier: GPL-3.0-only
#include <atomic>
#include <sstream>
#include <vector>
#include <nlohmann/json.hpp>
#include "core/protocol.hpp"
#include "stream/webrtc.hpp"
#include "test.hpp"
using namespace xc;
namespace {
const char* const kAudioRecvonly =
"m=audio 9 UDP/TLS/RTP/SAVPF 111\r\n"
"c=IN IP4 0.0.0.0\r\n"
"a=mid:audio\r\n"
"a=rtcp-mux\r\n"
"a=rtcp-rsize\r\n"
"a=rtpmap:111 opus/48000/2\r\n"
"a=fmtp:111 minptime=10;useinbandfec=1;stereo=1\r\n"
"a=recvonly\r\n";
const char* const kAudioMic =
"m=audio 9 UDP/TLS/RTP/SAVPF 111\r\n"
"c=IN IP4 0.0.0.0\r\n"
"a=mid:audio\r\n"
"a=rtcp-mux\r\n"
"a=rtcp-rsize\r\n"
"a=rtpmap:111 opus/48000/2\r\n"
"a=fmtp:111 minptime=10;useinbandfec=1;stereo=1\r\n"
"a=sendrecv\r\n"
"a=msid:stream-1 track-1\r\n"
"a=ssrc:3735928559 cname:xcdeadbeef\r\n"
"a=ssrc:3735928559 msid:stream-1 track-1\r\n";
proto::OfferMic testMic() {
proto::OfferMic m;
m.ssrc = 0xDEADBEEFu;
m.cname = "xcdeadbeef";
m.msid = "stream-1";
m.trackId = "track-1";
return m;
}
std::string replaceOnce(std::string s, const std::string& from, const std::string& to) {
const size_t at = s.find(from);
if (at != std::string::npos) s.replace(at, from.size(), to);
return s;
}
// Lines of the m-section with a=mid:<mid> (CR stripped).
std::vector<std::string> section(const std::string& sdp, const std::string& mid) {
std::vector<std::string> cur, found;
bool match = false;
std::istringstream in(sdp);
std::string line;
auto flush = [&] {
if (match && found.empty()) found = cur;
cur.clear();
match = false;
};
while (std::getline(in, line)) {
if (!line.empty() && line.back() == '\r') line.pop_back();
if (line.rfind("m=", 0) == 0) flush();
if (line == "a=mid:" + mid) match = true;
cur.push_back(line);
}
flush();
return found;
}
bool has(const std::vector<std::string>& lines, const std::string& l) {
for (const auto& x : lines)
if (x == l) return true;
return false;
}
} // namespace
XC_TEST(voice_sdp, offer_without_mic_unchanged) {
for (bool home : {false, true}) {
const std::string base = proto::buildOffer("uFrAg", "pWd", "AB:CD", home, "1080");
XC_CHECK_EQ(proto::buildOffer("uFrAg", "pWd", "AB:CD", home, "1080", nullptr, 2), base);
XC_CHECK(base.find(kAudioRecvonly) != std::string::npos);
proto::OfferMic zero; // ssrc 0 = no mic
XC_CHECK_EQ(proto::buildOffer("uFrAg", "pWd", "AB:CD", home, "1080", &zero), base);
}
}
XC_TEST(voice_sdp, offer_with_mic_exact_audio_lines) {
const proto::OfferMic mic = testMic();
const std::string base = proto::buildOffer("uFrAg", "pWd", "AB:CD", false, "1080");
const std::string withMic = proto::buildOffer("uFrAg", "pWd", "AB:CD", false, "1080", &mic);
XC_CHECK_EQ(withMic, replaceOnce(base, kAudioRecvonly, kAudioMic));
// video stays recvonly, data m-line unchanged
XC_CHECK_EQ(proto::sdpMediaDirection(withMic, "video"), std::string("recvonly"));
XC_CHECK_EQ(proto::sdpMediaDirection(withMic, "audio"), std::string("sendrecv"));
// sessionVersion 3 changes only the o= line
const std::string v3 = proto::buildOffer("uFrAg", "pWd", "AB:CD", false, "1080", &mic, 3);
XC_CHECK_EQ(v3, replaceOnce(withMic, "o=- 4611731400430051 2 IN IP4", "o=- 4611731400430051 3 IN IP4"));
XC_CHECK(v3 != withMic);
XC_CHECK(v3.find("o=- 4611731400430051 3 IN IP4 127.0.0.1\r\n") != std::string::npos);
}
XC_TEST(voice_sdp, post_body_chat_stream) {
const std::string offer = "v=0\r\n";
const std::string plain = proto::sdpPostBody(offer);
XC_CHECK_EQ(proto::sdpPostBody(offer, false), plain);
XC_CHECK(plain.find("chatStream") == std::string::npos);
auto j = nlohmann::json::parse(proto::sdpPostBody(offer, true), nullptr, false);
XC_REQUIRE(!j.is_discarded());
auto base = nlohmann::json::parse(plain);
XC_REQUIRE(j.contains("configuration"));
const auto& cfg = j["configuration"];
XC_REQUIRE(cfg.contains("chatStream"));
XC_CHECK_EQ(cfg["chatStream"].value("minVersion", 0), 1);
XC_CHECK_EQ(cfg["chatStream"].value("maxVersion", 0), 1);
XC_CHECK(cfg.contains("chatConfiguration"));
// everything else identical
j["configuration"].erase("chatStream");
XC_CHECK(j == base);
}
XC_TEST(voice_sdp, renegotiation_body_shape) {
const std::string body = proto::sdpChatRenegotiationBody("v=0\r\n");
XC_CHECK_EQ(body, std::string(R"({"configuration":{"isMediaStreamsChatRenegotiation":true},)"
R"("messageType":"offer","requestId":"2","sdp":"v=0\r\n"})"));
}
XC_TEST(voice_sdp, parse_exchange_fields) {
// string-wrapped object (what gssv returns)
nlohmann::json inner = {{"sdp", "v=0\r\na=x\r\n"}, {"sdpType", "answer"}, {"status", "success"},
{"chat", 1}, {"chatStream", 1}};
const std::string wrapped = nlohmann::json(inner.dump()).dump();
proto::SdpExchangeFields f;
XC_REQUIRE(proto::parseExchangeFields(wrapped, f));
XC_CHECK_EQ(f.sdp, std::string("v=0\r\na=x\r\n"));
XC_CHECK_EQ(f.status, std::string("success"));
XC_CHECK_EQ(f.chat, 1);
XC_CHECK_EQ(f.chatStream, 1);
XC_CHECK(f.summary.find("sdpType") != std::string::npos);
XC_CHECK(f.summary.find("v=0") == std::string::npos);
XC_CHECK(f.summary.find("\"sdp\"") == std::string::npos);
// plain object without chatStream
proto::SdpExchangeFields g;
XC_REQUIRE(proto::parseExchangeFields(R"({"sdp":"v=0","chat":0,"sdpType":"answer"})", g));
XC_CHECK_EQ(g.chat, 0);
XC_CHECK_EQ(g.chatStream, -1);
XC_CHECK(g.status.empty());
// numeric string / bool tolerance, long summary truncated
proto::SdpExchangeFields h;
nlohmann::json big = {{"sdp", "v=0"}, {"chatStream", "2"}, {"chat", true}, {"pad", std::string(1000, 'x')}};
XC_REQUIRE(proto::parseExchangeFields(big.dump(), h));
XC_CHECK_EQ(h.chatStream, 2);
XC_CHECK_EQ(h.chat, 1);
XC_CHECK(h.summary.size() <= 400);
proto::SdpExchangeFields bad;
XC_CHECK(!proto::parseExchangeFields("not json", bad));
XC_CHECK(!proto::parseExchangeFields("[1,2]", bad));
XC_CHECK(!proto::parseExchangeFields(R"("not an object")", bad));
}
XC_TEST(voice_sdp, media_direction) {
const std::string base = "v=0\r\nm=video 9 X 102\r\na=mid:video\r\na=recvonly\r\nm=audio 9 X 111\r\n"
"a=mid:audio\r\na=%s\r\nm=application 9 X\r\na=mid:0\r\n";
for (const char* d : {"sendrecv", "sendonly", "recvonly", "inactive"}) {
const std::string sdp = replaceOnce(base, "%s", d);
XC_CHECK_EQ(proto::sdpMediaDirection(sdp, "audio"), std::string(d));
XC_CHECK_EQ(proto::sdpMediaDirection(sdp, "video"), std::string("recvonly"));
}
// direction before a=mid, LF-only line endings
XC_CHECK_EQ(proto::sdpMediaDirection("v=0\nm=audio 9 X 111\na=sendonly\na=mid:audio\n", "audio"),
std::string("sendonly"));
// section without direction attribute
XC_CHECK_EQ(proto::sdpMediaDirection("v=0\r\nm=audio 9 X 111\r\na=mid:audio\r\n", "audio"),
std::string("sendrecv"));
// session-level direction is not a section's
XC_CHECK_EQ(proto::sdpMediaDirection("v=0\r\na=inactive\r\nm=audio 9 X 111\r\na=mid:audio\r\n", "audio"),
std::string("sendrecv"));
// absent mid
XC_CHECK_EQ(proto::sdpMediaDirection(replaceOnce(base, "%s", "sendrecv"), "nope"), std::string());
XC_CHECK_EQ(proto::sdpMediaDirection("", "audio"), std::string());
}
XC_TEST(voice_sdp, party_chat_active_message) {
proto::ChannelMessage m;
XC_REQUIRE(proto::parseChannelMessage(proto::messageSetPartyChatActive(true, "cv1"), m));
XC_CHECK_EQ(m.type, std::string("TransactionStart"));
XC_CHECK_EQ(m.target, std::string("/streaming/social/partyChatAudioCoordination/setPartyChatActive"));
XC_CHECK_EQ(m.content, std::string(R"({"partyChatActive":true})"));
XC_CHECK_EQ(m.cv, std::string("cv1"));
XC_CHECK_EQ(m.id.size(), size_t(36));
XC_REQUIRE(proto::parseChannelMessage(proto::messageSetPartyChatActive(false), m));
XC_CHECK_EQ(m.content, std::string(R"({"partyChatActive":false})"));
XC_CHECK_EQ(m.cv, std::string());
}
XC_TEST(voice_sdp, opus_rtp_bytes) {
const uint8_t payload[] = {0xF8, 0xFF, 0xFE};
const std::vector<uint8_t> p = buildOpusRtp(0xABCD, 0x01020304u, 0xDEADBEEFu, true, payload, sizeof(payload));
const std::vector<uint8_t> want = {0x80, 0xEF, 0xAB, 0xCD, 0x01, 0x02, 0x03, 0x04,
0xDE, 0xAD, 0xBE, 0xEF, 0xF8, 0xFF, 0xFE};
XC_CHECK(p == want);
const std::vector<uint8_t> q = buildOpusRtp(0xFFFF, 0xFFFFFC40u, 1, false, payload, 1, 96);
XC_REQUIRE(q.size() == 13);
XC_CHECK_EQ(q[0], uint8_t(0x80));
XC_CHECK_EQ(q[1], uint8_t(96)); // no marker
XC_CHECK_EQ(q[2], uint8_t(0xFF));
XC_CHECK_EQ(q[3], uint8_t(0xFF));
XC_CHECK_EQ(q[4], uint8_t(0xFF));
XC_CHECK_EQ(q[7], uint8_t(0x40));
XC_CHECK_EQ(q[11], uint8_t(1));
XC_CHECK_EQ(q[12], uint8_t(0xF8));
XC_CHECK_EQ(buildOpusRtp(1, 2, 3, false, nullptr, 0).size(), size_t(12));
}
XC_TEST(voice_sdp, webrtc_voice_off_has_no_mic) {
WebRtc rtc;
XC_REQUIRE(rtc.init(WebRtc::Callbacks{}));
XC_CHECK_EQ(rtc.micInfo().ssrc, 0u);
const uint8_t b[3] = {1, 2, 3};
XC_CHECK(!rtc.sendMicOpus(b, sizeof(b), 960, true));
XC_CHECK(rtc.answerAudioDirection().empty());
XC_CHECK_EQ(rtc.chatAudioSsrc(), 0u);
rtc.close();
}
// The hand-written offer must agree with libdatachannel's own local description for the
// audio m-line: direction, SSRC, cname, msid / track id.
XC_TEST(voice_sdp, offer_matches_local_description) {
WebRtc rtc;
XC_REQUIRE(rtc.init(WebRtc::Callbacks{}, true));
const WebRtc::MicInfo mi = rtc.micInfo();
XC_REQUIRE(mi.ssrc != 0);
XC_CHECK(!mi.cname.empty());
XC_CHECK_EQ(mi.msid.size(), size_t(36));
XC_CHECK_EQ(mi.trackId.size(), size_t(36));
XC_CHECK(mi.msid != mi.trackId);
std::string u, p, fp;
std::vector<std::string> cands;
std::atomic<bool> abort{false};
rtc.gatherLocal(u, p, fp, cands, 1500, &abort); // result irrelevant: only the description matters
const std::string local = rtc.localDescriptionSdp();
XC_REQUIRE(!local.empty());
proto::OfferMic om;
om.ssrc = mi.ssrc;
om.cname = mi.cname;
om.msid = mi.msid;
om.trackId = mi.trackId;
const std::string offer = proto::buildOffer("u", "p", "AB:CD", false, "1080", &om);
const auto mine = section(offer, "audio");
const auto theirs = section(local, "audio");
XC_REQUIRE(!mine.empty());
XC_REQUIRE(!theirs.empty());
XC_CHECK_EQ(proto::sdpMediaDirection(local, "audio"), std::string("sendrecv"));
XC_CHECK_EQ(proto::sdpMediaDirection(offer, "audio"), std::string("sendrecv"));
const std::string ssrc = std::to_string(mi.ssrc);
for (const std::string& l : {"a=ssrc:" + ssrc + " cname:" + mi.cname,
"a=ssrc:" + ssrc + " msid:" + mi.msid + " " + mi.trackId,
"a=msid:" + mi.msid + " " + mi.trackId}) {
XC_CHECK(has(mine, l));
XC_CHECK(has(theirs, l));
}
// video stays receive-only in both
XC_CHECK_EQ(proto::sdpMediaDirection(local, "video"), std::string("recvonly"));
// Not connected: sending fails cleanly and is counted.
const uint8_t b[3] = {1, 2, 3};
XC_CHECK(!rtc.sendMicOpus(b, sizeof(b), 960, true));
XC_CHECK_EQ(rtc.micTxStats().packets, uint64_t(0));
XC_CHECK_EQ(rtc.micTxStats().failed, uint64_t(1));
// No answer yet: renegotiation refuses, rollback is a no-op.
std::string ru, rp, rfp;
XC_CHECK(!rtc.beginRenegotiation(ru, rp, rfp));
XC_CHECK(!rtc.finishRenegotiation("v=0\r\n"));
rtc.abortRenegotiation();
rtc.close();
XC_CHECK_EQ(rtc.micInfo().ssrc, 0u);
}