From adfb4287962f908ea35d9fed29729a95fc03f75a Mon Sep 17 00:00:00 2001 From: David Rowe Date: Thu, 20 May 2021 09:43:18 +1200 Subject: [PATCH 01/82] Distinguish use of WebRTC features - audio versus data channel --- libraries/audio-client/src/AudioClient.cpp | 11 ++++++----- libraries/audio-client/src/AudioClient.h | 8 +++++++- libraries/shared/src/shared/WebRTC.h | 22 +++++++++++++--------- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 38635870fd5..c03f37576db 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -4,6 +4,7 @@ // // Created by Stephen Birarda on 1/22/13. // Copyright 2013 High Fidelity, Inc. +// Copyright 2021 Vircadia contributors. // // Distributed under the Apache License, Version 2.0. // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html @@ -363,7 +364,7 @@ AudioClient::AudioClient() { configureReverb(); -#if defined(WEBRTC_ENABLED) +#if defined(WEBRTC_AUDIO) configureWebrtc(); #endif @@ -1142,7 +1143,7 @@ void AudioClient::setReverbOptions(const AudioEffectOptions* options) { } } -#if defined(WEBRTC_ENABLED) +#if defined(WEBRTC_AUDIO) static void deinterleaveToFloat(const int16_t* src, float* const* dst, int numFrames, int numChannels) { for (int i = 0; i < numFrames; i++) { @@ -1261,7 +1262,7 @@ void AudioClient::processWebrtcNearEnd(int16_t* samples, int numFrames, int numC } } -#endif // WEBRTC_ENABLED +#endif // WEBRTC_AUDIO void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) { // If there is server echo, reverb will be applied to the recieved audio stream so no need to have it here. @@ -1462,7 +1463,7 @@ void AudioClient::handleMicAudioInput() { } isClipping = (_timeSinceLastClip >= 0.0f) && (_timeSinceLastClip < 2.0f); // 2 second hold time -#if defined(WEBRTC_ENABLED) +#if defined(WEBRTC_AUDIO) if (_isAECEnabled) { processWebrtcNearEnd(inputAudioSamples.get(), inputSamplesRequired / _inputFormat.channelCount(), _inputFormat.channelCount(), _inputFormat.sampleRate()); @@ -2420,7 +2421,7 @@ qint64 AudioClient::AudioOutputIODevice::readData(char * data, qint64 maxSize) { // limit the audio _audio->_audioLimiter.render(mixBuffer, scratchBuffer, framesPopped); -#if defined(WEBRTC_ENABLED) +#if defined(WEBRTC_AUDIO) if (_audio->_isAECEnabled) { _audio->processWebrtcFarEnd(scratchBuffer, framesPopped, OUTPUT_CHANNEL_COUNT, _audio->_outputFormat.sampleRate()); } diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index a5de9bd4ca8..19ccb587a44 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -4,6 +4,7 @@ // // Created by Stephen Birarda on 1/22/13. // Copyright 2013 High Fidelity, Inc. +// Copyright 2021 Vircadia contributors. // // Distributed under the Apache License, Version 2.0. // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html @@ -56,6 +57,11 @@ #include "AudioFileWav.h" #include "HifiAudioDeviceInfo.h" +#if defined(WEBRTC_AUDIO) +# include +# include "modules/audio_processing/audio_processing_impl.h" +#endif + #ifdef _WIN32 #pragma warning( push ) #pragma warning( disable : 4273 ) @@ -450,7 +456,7 @@ public slots: void updateReverbOptions(); void handleLocalEchoAndReverb(QByteArray& inputByteArray); -#if defined(WEBRTC_ENABLED) +#if defined(WEBRTC_AUDIO) static const int WEBRTC_SAMPLE_RATE_MAX = 96000; static const int WEBRTC_CHANNELS_MAX = 2; static const int WEBRTC_FRAMES_MAX = webrtc::AudioProcessing::kChunkSizeMs * WEBRTC_SAMPLE_RATE_MAX / 1000; diff --git a/libraries/shared/src/shared/WebRTC.h b/libraries/shared/src/shared/WebRTC.h index 2f0e444bff0..e99c6430456 100644 --- a/libraries/shared/src/shared/WebRTC.h +++ b/libraries/shared/src/shared/WebRTC.h @@ -3,6 +3,7 @@ // libraries/shared/src/shared/ // // Copyright 2019 High Fidelity, Inc. +// Copyright 2021 Vircadia contributors. // // Distributed under the Apache License, Version 2.0. // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html @@ -11,26 +12,29 @@ #ifndef hifi_WebRTC_h #define hifi_WebRTC_h +#ifndef QSYSTEMDETECTION_H +#include +#endif + +// WEBRTC_AUDIO: WebRTC audio features, e.g., echo canceling. +// WEBRTC_DATA_CHANNEL: WebRTC client-server connections in parallel with UDP. + #if defined(Q_OS_MAC) -# define WEBRTC_ENABLED 1 +# define WEBRTC_AUDIO 1 # define WEBRTC_POSIX 1 #elif defined(Q_OS_WIN) -# define WEBRTC_ENABLED 1 +# define WEBRTC_AUDIO 1 +# define WEBRTC_DATA_CHANNEL 1 # define WEBRTC_WIN 1 # define NOMINMAX 1 # define WIN32_LEAN_AND_MEAN 1 #elif defined(Q_OS_ANDROID) // I don't yet have a working libwebrtc for android -// # define WEBRTC_ENABLED 1 +// # define WEBRTC_AUDIO 1 // # define WEBRTC_POSIX 1 #elif defined(Q_OS_LINUX) -# define WEBRTC_ENABLED 1 +# define WEBRTC_AUDIO 1 # define WEBRTC_POSIX 1 #endif -#if defined(WEBRTC_ENABLED) -# include -# include "modules/audio_processing/audio_processing_impl.h" -#endif - #endif // hifi_WebRTC_h From 6afb8044ea02bbee35bcc74811ae8eebec354d25 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Thu, 20 May 2021 09:51:14 +1200 Subject: [PATCH 02/82] Add WebRTC signaling channel --- domain-server/src/DomainServer.cpp | 6 +- domain-server/src/DomainServer.h | 6 + libraries/networking/CMakeLists.txt | 6 +- libraries/networking/src/DomainHandler.h | 10 +- libraries/networking/src/NetworkLogging.cpp | 1 + libraries/networking/src/NetworkLogging.h | 1 + libraries/networking/src/NodeList.cpp | 1 - .../src/webrtc/WebRTCSignalingServer.cpp | 96 ++++++++++++++++ .../src/webrtc/WebRTCSignalingServer.h | 104 ++++++++++++++++++ 9 files changed, 227 insertions(+), 4 deletions(-) create mode 100644 libraries/networking/src/webrtc/WebRTCSignalingServer.cpp create mode 100644 libraries/networking/src/webrtc/WebRTCSignalingServer.h diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 3acd17f6afc..02fd810b0ef 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -165,7 +165,11 @@ bool DomainServer::forwardMetaverseAPIRequest(HTTPConnection* connection, DomainServer::DomainServer(int argc, char* argv[]) : QCoreApplication(argc, argv), _gatekeeper(this), - _httpManager(QHostAddress::AnyIPv4, DOMAIN_SERVER_HTTP_PORT, QString("%1/resources/web/").arg(QCoreApplication::applicationDirPath()), this) +#ifdef WEBRTC_DATA_CHANNEL + _webrtcSignalingServer(QHostAddress::AnyIPv4, DEFAULT_DOMAIN_SERVER_WS_PORT, this), +#endif + _httpManager(QHostAddress::AnyIPv4, DOMAIN_SERVER_HTTP_PORT, + QString("%1/resources/web/").arg(QCoreApplication::applicationDirPath()), this) { if (_parentPID != -1) { watchParentProcess(_parentPID); diff --git a/domain-server/src/DomainServer.h b/domain-server/src/DomainServer.h index 11d04cb2554..29142505a89 100644 --- a/domain-server/src/DomainServer.h +++ b/domain-server/src/DomainServer.h @@ -27,6 +27,8 @@ #include #include #include +#include +#include #include "AssetsBackupHandler.h" #include "DomainGatekeeper.h" @@ -311,6 +313,10 @@ private slots: std::unordered_map> _pendingContentFiles; QThread _assetClientThread; + +#ifdef WEBRTC_DATA_CHANNEL + WebRTCSignalingServer _webrtcSignalingServer; +#endif }; diff --git a/libraries/networking/CMakeLists.txt b/libraries/networking/CMakeLists.txt index 9f63f2cb00f..50382cda991 100644 --- a/libraries/networking/CMakeLists.txt +++ b/libraries/networking/CMakeLists.txt @@ -1,9 +1,13 @@ set(TARGET_NAME networking) -setup_hifi_library(Network) +setup_hifi_library(Network WebSockets) link_hifi_libraries(shared platform) target_openssl() target_tbb() +if (WIN32) + # WEBRTC TODO: Add UNIX. + target_webrtc() +endif () if (WIN32) # we need ws2_32.lib on windows, but it's static so we don't bubble it up diff --git a/libraries/networking/src/DomainHandler.h b/libraries/networking/src/DomainHandler.h index a56d3162bd3..a8c316572a0 100644 --- a/libraries/networking/src/DomainHandler.h +++ b/libraries/networking/src/DomainHandler.h @@ -40,7 +40,15 @@ const unsigned short DEFAULT_DOMAIN_SERVER_PORT = ? QProcessEnvironment::systemEnvironment() .value("HIFI_DOMAIN_SERVER_PORT") .toUShort() - : 40102; + : 40102; // UDP + +const unsigned short DEFAULT_DOMAIN_SERVER_WS_PORT = + QProcessEnvironment::systemEnvironment() + .contains("HIFI_DOMAIN_SERVER_WS_PORT") + ? QProcessEnvironment::systemEnvironment() + .value("HIFI_DOMAIN_SERVER_WS_PORT") + .toUShort() + : 40102; // TCP const unsigned short DEFAULT_DOMAIN_SERVER_DTLS_PORT = QProcessEnvironment::systemEnvironment() diff --git a/libraries/networking/src/NetworkLogging.cpp b/libraries/networking/src/NetworkLogging.cpp index 3d7c2fc5d53..daa8c8c4161 100644 --- a/libraries/networking/src/NetworkLogging.cpp +++ b/libraries/networking/src/NetworkLogging.cpp @@ -16,3 +16,4 @@ Q_LOGGING_CATEGORY(networking_ice, "hifi.networking.ice") Q_LOGGING_CATEGORY(resourceLog, "hifi.networking.resource") Q_LOGGING_CATEGORY(asset_client, "hifi.networking.asset_client") Q_LOGGING_CATEGORY(messages_client, "hifi.networking.messages_client") +Q_LOGGING_CATEGORY(networking_webrtc, "hifi.networking.webrtc") diff --git a/libraries/networking/src/NetworkLogging.h b/libraries/networking/src/NetworkLogging.h index 8247c600962..906947b7c44 100644 --- a/libraries/networking/src/NetworkLogging.h +++ b/libraries/networking/src/NetworkLogging.h @@ -19,5 +19,6 @@ Q_DECLARE_LOGGING_CATEGORY(networking) Q_DECLARE_LOGGING_CATEGORY(networking_ice) Q_DECLARE_LOGGING_CATEGORY(asset_client) Q_DECLARE_LOGGING_CATEGORY(messages_client) +Q_DECLARE_LOGGING_CATEGORY(networking_webrtc) #endif // hifi_NetworkLogging_h diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index a9753026993..31534f34f4f 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -409,7 +409,6 @@ void NodeList::sendDomainServerCheckIn() { if (domainPacketType == PacketType::DomainConnectRequest) { #if (PR_BUILD || DEV_BUILD) - // ####### if (_shouldSendNewerVersion) { domainPacket->setVersion(versionForPacketType(domainPacketType) + 1); } diff --git a/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp b/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp new file mode 100644 index 00000000000..66b1d6616c0 --- /dev/null +++ b/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp @@ -0,0 +1,96 @@ +// +// WebRTCSignalingServer.cpp +// libraries/networking/src/webrtc +// +// Created by David Rowe on 16 May 2021. +// Copyright 2021 Vircadia contributors. +// + +#include "WebRTCSignalingServer.h" + +#if defined(WEBRTC_DATA_CHANNEL) + +#include +#include + +#include "../NetworkLogging.h" +#include "../NodeType.h" + + +const int WEBRTC_SOCKET_CHECK_INTERVAL_IN_MS = 30000; + +WebRTCSignalingServer::WebRTCSignalingServer(const QHostAddress& address, quint16 port, QObject* parent) : + QObject(parent), + _address(address), + _port(port), + _webSocketServer(new QWebSocketServer(QStringLiteral("WebRTC Signaling Server"), QWebSocketServer::NonSecureMode, this)) +{ + connect(_webSocketServer, &QWebSocketServer::newConnection, this, &WebRTCSignalingServer::newWebSocketConnection); + + bindSocket(); + + // Automatically recover from network interruptions. + _isWebSocketServerListeningTimer = new QTimer(this); + connect(_isWebSocketServerListeningTimer, &QTimer::timeout, this, &WebRTCSignalingServer::checkWebSocketServerIsListening); + _isWebSocketServerListeningTimer->start(WEBRTC_SOCKET_CHECK_INTERVAL_IN_MS); +} + +void WebRTCSignalingServer::checkWebSocketServerIsListening() { + if (!_webSocketServer->isListening()) { + qCWarning(networking_webrtc) << "WebSocket on port " << QString::number(_port) << " is no longer listening"; + _webSockets.clear(); + bindSocket(); + } +} + +void WebRTCSignalingServer::bindSocket() { + if (!_webSocketServer->listen(_address, _port)) { + qCWarning(networking_webrtc) << "Failed to open WebSocket for WebRTC signaling."; + } +} + +void WebRTCSignalingServer::webSocketTextMessageReceived(const QString& message) { + auto source = qobject_cast(sender()); + if (source) { + QJsonObject json = QJsonDocument::fromJson(message.toUtf8()).object(); + // WEBRTC TODO: Move domain server echoing into domain server. + if (json.keys().contains("echo") && json.value("to").toString() == QString(QChar(NodeType::DomainServer))) { + // Domain server echo request - echo message back to sender. + json.remove("to"); + json.insert("from", QString(QChar(NodeType::DomainServer))); + QString echo = QJsonDocument(json).toJson(); + source->sendTextMessage(echo); + } else { + // WebRTC message or assignment client echo request. (Send both to target.) + json.insert("from", source->peerPort()); + emit messageReceived(json); + } + } else { + qCWarning(networking_webrtc) << "Failed to find WebSocket for incoming WebRTC signaling message."; + } +} + +void WebRTCSignalingServer::sendMessage(const QJsonObject& message) { + quint16 destinationPort = message.value("to").toInt(); + if (_webSockets.contains(destinationPort)) { + _webSockets.value(destinationPort)->sendTextMessage(QString(QJsonDocument(message).toJson())); + } else { + qCWarning(networking_webrtc) << "Failed to find WebSocket for outgoing WebRTC signaling message."; + } +} + +void WebRTCSignalingServer::webSocketDisconnected() { + auto source = qobject_cast(sender()); + if (source) { + _webSockets.remove(source->peerPort()); + } +} + +void WebRTCSignalingServer::newWebSocketConnection() { + auto webSocket = _webSocketServer->nextPendingConnection(); + connect(webSocket, &QWebSocket::textMessageReceived, this, &WebRTCSignalingServer::webSocketTextMessageReceived); + connect(webSocket, &QWebSocket::disconnected, this, &WebRTCSignalingServer::webSocketDisconnected); + _webSockets.insert(webSocket->peerPort(), webSocket); +} + +#endif // WEBRTC_DATA_CHANNEL diff --git a/libraries/networking/src/webrtc/WebRTCSignalingServer.h b/libraries/networking/src/webrtc/WebRTCSignalingServer.h new file mode 100644 index 00000000000..41d79dbd571 --- /dev/null +++ b/libraries/networking/src/webrtc/WebRTCSignalingServer.h @@ -0,0 +1,104 @@ +// +// WebRTCSignalingServer.h +// libraries/networking/src/webrtc +// +// Provides a signaling channel for setting up WebRTC connections between the Web app and the domain servers and mixers. +// +// Created by David Rowe on 16 May 2021. +// Copyright 2021 Vircadia contributors. +// + +#ifndef vircadia_SignalingServer_h +#define vircadia_SignalingServer_h + +#include + +#if defined(WEBRTC_DATA_CHANNEL) + +#include +#include +#include + +#include "../HifiSockAddr.h" + +/// @brief WebRTC signaling server that Interface clients can use to initiate WebRTC connections to the domain server and +/// assignment clients. +/// +/// @details The signaling server is expected to be hosted in the domain server. It provides a WebSocket for Interface clients +/// to use in the WebRTC signaling handshake process to establish WebRTC data channel connections to each of the domain server +/// and the assignment clients (i.e., separate WebRTC data channels for each but only a single signaling WebSocket). The +/// assignment client signaling messages are expected to be relayed - by the domain server - via Vircadia protocol messages on +/// the UDP connections between the domain server and assignment clients. +/// +/// Additionally, for debugging purposes, instead of containing a WebRTC payload a signaling message may be an echo request. +/// This is bounced back to the client from the WebRTCSignalingServer if the domain server was the target, otherwise it is +/// expected to be bounced back upon receipt by the relevant assignment client. +/// +/// The signaling messages are sent and received as JSON objects, with `to` and `from` fields in addition to either the WebRTC +/// signaling `data` payload or an `echo` request: +/// +/// | Interface -> Server || +/// | -------- | -----------------------| +/// | `to` | NodeType | +/// | `from` | WebSocket port number* | +/// | [`data`] | WebRTC payload | +/// | [`echo`] | Echo request | +/// * The `from` field is filled in by the WebRTCSignalingServer. +/// +/// | Server -> Interface || +/// | -------- | --------------------- | +/// | `to` | WebSocket port number | +/// | `from` | NodeType | +/// | [`data`] | WebRTC payload | +/// | [`echo`] | Echo request | +/// +class WebRTCSignalingServer : public QObject { + Q_OBJECT + +public: + + /// @brief Constructs a new WebRTCSignalingServer. + /// @param address The IP address to use for the WebSocket. + /// @param port The port to use for the WebSocket. + /// @param parent Qt parent object. + WebRTCSignalingServer(const QHostAddress& address, quint16 port, QObject* parent = nullptr); + +public slots: + + /// @brief Send a WebRTC signaling channel message to an Interface client. + /// @param message The message to send to the Interface client. Includes details of the sender and the destination in + /// addition to the WebRTC signaling channel payload. + void sendMessage(const QJsonObject& message); + +signals: + + /// @brief A WebRTC signaling channel message was received from an Interface client. + /// @param message The message received from the Interface client. Includes details of the sender and the destination in + /// addition to the WebRTC signaling channel payload.\n + /// Not emitted if the message was an echo request for the domain server. + void messageReceived(const QJsonObject& message); + +private slots: + + void newWebSocketConnection(); + void webSocketTextMessageReceived(const QString& message); + void webSocketDisconnected(); + +private: + + void checkWebSocketServerIsListening(); + void bindSocket(); + + QWebSocketServer* _webSocketServer; + QHostAddress _address; + const quint16 _port; + + QHash _webSockets; // client WebSocket port, client WebSocket object + + QTimer* _isWebSocketServerListeningTimer; +}; + + +#endif // WEBRTC_DATA_CHANNEL + +#endif // vircadia_SignalingServer_h From 6e3c68048755e7b88511cf39ed46d5cf0ce4992a Mon Sep 17 00:00:00 2001 From: David Rowe Date: Thu, 20 May 2021 09:51:51 +1200 Subject: [PATCH 03/82] Add Doxygen comments information --- CODING_STANDARD.md | 10 +++++++++- tools/doxygen/README.md | 5 +++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CODING_STANDARD.md b/CODING_STANDARD.md index 1c2f3cb27e1..3d4516f99fd 100644 --- a/CODING_STANDARD.md +++ b/CODING_STANDARD.md @@ -948,7 +948,7 @@ In an international environment English is the preferred language. #### [4.3.2] Use // for all comments, including multi-line comments. -An exception to this rule applies for jsdoc or Doxygen comments. +An exception to this rule applies to JSDoc and Doxygen comments. ```cpp // Comment spanning @@ -1008,3 +1008,11 @@ These types of comments are explicitly not allowed. If you need to break up sect //-------------------------------------------------------------------------------- ``` +#### [4.3.6] Doxygen comments should use "///" + +Use the `///` style of [Doxygen](https://www.doxygen.nl/index.html) comments when documenting public interfaces. + +Some editors can automatically create a Doxygen documentation stub if you type `///` in the line above the item to be +documented. + +**Visual Studio:** To configure Visual Studio's Doxygen commenting behavior, search for "Doxygen" in Tools > Options. diff --git a/tools/doxygen/README.md b/tools/doxygen/README.md index 34d12c1d8af..40f444ca071 100644 --- a/tools/doxygen/README.md +++ b/tools/doxygen/README.md @@ -12,6 +12,11 @@ Make a `/build/doxygen/` directory. If you want to run Doxygen from a command prompt, add the Doxygen install's `/bin` directory to your system PATH. +## Documenting + +See section 4.3.6 of the [*Coding Standard*](../../CODING_STANDARD.md). + + ## Running ### Using the Doxywizard GUI From ce31b70a1d45a7de08b10f8246f688c9222de7fb Mon Sep 17 00:00:00 2001 From: David Rowe Date: Fri, 21 May 2021 07:58:10 +1200 Subject: [PATCH 04/82] Update Windows WebRTC from version M81 to M84 --- cmake/ports/webrtc/CONTROL | 2 +- cmake/ports/webrtc/README.md | 215 +++++++++++++++++++++ cmake/ports/webrtc/copy-VCPKG-file-win.cmd | 36 ++++ cmake/ports/webrtc/portfile.cmake | 8 +- libraries/audio-client/src/AudioClient.cpp | 2 + libraries/audio-client/src/AudioClient.h | 1 + libraries/shared/src/shared/WebRTC.h | 3 + 7 files changed, 262 insertions(+), 5 deletions(-) create mode 100644 cmake/ports/webrtc/README.md create mode 100644 cmake/ports/webrtc/copy-VCPKG-file-win.cmd diff --git a/cmake/ports/webrtc/CONTROL b/cmake/ports/webrtc/CONTROL index 12a76920b9d..b705809c1a6 100644 --- a/cmake/ports/webrtc/CONTROL +++ b/cmake/ports/webrtc/CONTROL @@ -1,3 +1,3 @@ Source: webrtc -Version: 20190626 +Version: 20210105 Description: WebRTC diff --git a/cmake/ports/webrtc/README.md b/cmake/ports/webrtc/README.md new file mode 100644 index 00000000000..3db73dce96f --- /dev/null +++ b/cmake/ports/webrtc/README.md @@ -0,0 +1,215 @@ +# WebRTC + +WebRTC Information: +- https://webrtc.org/ +- https://webrtc.googlesource.com/src +- https://webrtc.googlesource.com/src/+/refs/heads/master/docs/native-code/index.md + - https://webrtc.googlesource.com/src/+/refs/heads/master/docs/native-code/development/prerequisite-sw/index.md + - https://webrtc.googlesource.com/src/+/refs/heads/master/docs/native-code/development/index.md +- https://www.chromium.org/developers/calendar +- https://github.com/microsoft/winrtc +- https://docs.microsoft.com/en-us/winrtc/getting-started +- https://groups.google.com/g/discuss-webrtc \ + See "PSA" posts for release information. +- https://bugs.chromium.org/p/webrtc/issues/list +- https://stackoverflow.com/questions/27809193/webrtc-not-building-for-windows +- https://github.com/aisouard/libwebrtc/issues/57 + +## Windows - M84 + +WebRTC's M84 release is currently used because it corresponded to Microsoft's latest WinRTC release at the time of develeopment, +and WinRTC is a source of potentially useful patches. + +The following notes document how the M84-based Windows VCPKG was created, using Visual Studio 2019. + +### Set Up depot_tools + +Install Google's depot_tools. +- Download depot_tools.zip. +- Extract somewhere. +- Add the extracted directory to the start of the system `PATH` environment variable. + +Configure depot_tools. +- Set an environment variable `DEPOT_TOOLS_WIN_TOOLCHAIN=0` +- Set an environment variable `GYP_MSVS_VERSION=2019` + +Initialize depot_tools. +- VS2019 developer command prompt in the directory where the source tree will be created. +- `gclient` + +### Get the Code + +Fetch the code into a *\src* subdirectory. This may take some time! +- `fetch --nohooks webrtc` + +Switch to the M84 branch. +- `cd src` +- `git checkout branch-heads/4147` + +Fetch all the subrepositories. +- `gclient sync -D -r branch-heads/4147` + +### Patch the Code + +#### Modify compiler switches +- Edit *build\config\win\BUILD.gn*: + - Change all `/MT` to `/MD`, and `/MTd` to `/MDd`. + - Change all `cflags = [ "/MDd" ]` to `[ "/MDd", "-D_ITERATOR_DEBUG_LEVEL=2", "-D_HAS_ITERATOR_DEBUGGING=1" ]`. +- Edit *build\config\compiler\BUILD.gn*:\ + Change: + ``` + if (is_win) { + if (is_clang) { + cflags = [ "/Z7" ] # Debug information in the .obj files. + } else { + cflags = [ "/Zi" ] # Produce PDB file, no edit and continue. + } + ``` + to: + ``` + if (is_win) { + if (is_clang) { + cflags = [ "/Z7", "/std:c++17", "/Zc:__cplusplus" ] # Debug information in the .obj files. + } else { + cflags = [ "/Zi", "/std:c++17", "/Zc:__cplusplus" ] # Produce PDB file, no edit and continue. + } + ``` + +#### H265 Codec Fixes +https://bugs.webrtc.org/9213#c13 +- Edit the following files: + - *modules\video_coding\codecs\h264\h264_color_space.h* + - *modules\video_coding\codecs\h264\h264_decoder_impl.h* + - *modules\video_coding\codecs\h264\h264_encoder_impl.h* + In each, comment out the following lines: + ``` + #if defined(WEBRTC_WIN) && !defined(__clang__) + #error "See: bugs.webrtc.org/9213#c13." + #endif + ``` +- Edit *third_party\ffmpeg\libavcodec\fft_template.c*:\ + Comment out all of `ff_fft_init` except the fail clause at the end. +- Edit *third_party\ffmpeg\libavcodec\pcm.c*:\ + Comment out last line, containing `PCM Archimedes VIDC`. +- Edit *third_party\ffmpeg\libavutil\x86\imgutils_init.c*:\ + Add the following method to the end of the file: + ``` + void avpriv_emms_asm(void) {} // Fix missing symbol in FFMPEG. + ``` + +#### Exclude BoringSSL +A separate OpenSSL VCPKG is used for building Vircadia. +The following patches are needed even though SSL is excluded in the `gn gen` build commands. +- Rename *third_party\boringssl* to *third_party\boringssl-NO*. +- Edit *third_party\libsrtp\BUILD.gn:\ + Change: + ``` + public_deps = [ + "//third_party/boringssl:boringssl", + ] + ``` + To: + ``` + public_deps = [ + # "//third_party/boringssl:boringssl", + ] + ``` + +- Edit *third_party\usrsctp\BUILD.gn*:\ + Change: + ``` + deps = [ "//third_party/boringssl" ] + ``` + To: + ``` + deps = [ + # "//third_party/boringssl" + ] + ``` +- Edit *base\BUILD.gn*:\ + In the code under: + ``` + # Use the base implementation of hash functions when building for + # NaCl. Otherwise, use boringssl. + ``` + Change: + ``` + if (is_nacl) { + ``` + To: + ``` + # if (is_nacl) { + if (true) { + ``` +- Edit *rtc_base\BUILD.gn*:\ + Change: + ``` + if (rtc_build_ssl) { + deps += [ "//third_party/boringssl" ] + } else { + ``` + To: + ``` + if (rtc_build_ssl) { + # deps += [ "//third_party/boringssl" ] + } else { + ``` + +### Set Up OpenSSL + +Do one of the following to provide OpenSSL for building against: +a. If you have built Vircadia, find the **HIFI_VCPKG_BASE** subdirectory used in your build and make note of the path to and +including the *installed\x64-windows\include* directory (which includes an *openssl* directory). +a. Follow https://github.com/vircadia/vcpkg to install *vcpkg* and then *openssl*. Make note of the path to and including the +*packages\openssl-windows_x64-windows\include* directory (which includes an *openssl* directory). + +Copy the *\\openssl* directory to the following locations (i.e., add as *openssl* subdirectories): +- *third_party\libsrtp\crypto\include* +- *third_party\usrsctp\usrsctplib\usrsctplib* + +Also use the path in the `gn gen` commands, below, making sure to escape `\`s as `\\`s. + +### Build and Package + +Use a VS2019 developer command prompt in the *src* directory. + +Create a release build of the WebRTC library: +- `gn gen --ide=vs2019 out\Release --filters=//:webrtc "--args=is_debug=false is_clang=false use_custom_libcxx=false libcxx_is_shared=true symbol_level=2 use_lld=false rtc_include_tests=false rtc_build_tools=false rtc_build_examples=false proprietary_codecs=true rtc_use_h264=true enable_libaom=false rtc_enable_protobuf=false rtc_build_ssl=false rtc_ssl_root=\"\""` +- `ninja -C out\Release` + +Create a debug build of the WebRTC library: +- `gn gen --ide=vs2019 out\Debug --filters=//:webrtc "--args=is_debug=true is_clang=false use_custom_libcxx=false libcxx_is_shared=true enable_iterator_debugging=true use_lld=false rtc_include_tests=false rtc_build_tools=false rtc_build_examples=false proprietary_codecs=true rtc_use_h264=true enable_libaom=false rtc_enable_protobuf=false rtc_build_ssl=false rtc_ssl_root=\"\""` +- `ninja -C out\Debug` + +Create VCPKG file: +- Assemble files in VCPKG directory structure: Use the *copy-VCPKG-files-win.cmd* batch file per instructions in it. +`cd ..`\ +`copy-VCPKG-files-win` +- Zip up the VCPKG *webrtc* directory created by the batch file. +`cd vcpkg`\ +`7z a -tzip webrtc-m84-yyyymmdd-windows.zip webrtc` +- Calculate the SHA512 of the zip file. E.g., using a Windows PowerShell command window:\ + `Get-FileHash -Algorithm SHA512 | Format-List` +- Convert the SHA512 to lower case. E.g., using Microsoft Word, select the SHA512 text and use Shift-F3 to change the case. +- Host the zip file on the Web. +- Update *CONTROL* and *portfile.cmake* with version number (revision date), zip file Web URL, and SHA512. + +### Tidying up + +Disable the depot_tools: +- Rename the *depot_tools* directory so that these tools don't interfere with the rest of your development environment for + other work. + + +## Linux - M81 + +The original, High Fidelity-provided WebRTC VCPKG library is used for AEC (audio echo cancellation) only. + +**TODO:** Update to M84 and include WebRTC components per Windows WebRTC. + + +## MacOS - M78 + +The original, High Fidelity-provided WebRTC VCPKG library is used for AEC (audio echo cancellation) only. + +**TODO:** Update to M84 and include WebRTC components per Windows WebRTC. diff --git a/cmake/ports/webrtc/copy-VCPKG-file-win.cmd b/cmake/ports/webrtc/copy-VCPKG-file-win.cmd new file mode 100644 index 00000000000..45ec95c1281 --- /dev/null +++ b/cmake/ports/webrtc/copy-VCPKG-file-win.cmd @@ -0,0 +1,36 @@ +rem Copy this file to a directory above the WebRTC \src directory and run it from there in a command window. +set WEBRTC_SRC_DIR=src +set RELEASE_LIB_DIR=%WEBRTC_SRC_DIR%\out\Release\obj +set DEBUG_LIB_DIR=%WEBRTC_SRC_DIR%\out\Debug\obj +set VCPKG_TGT_DIR=vcpkg + +if exist %VCPKG_TGT_DIR% rd /s /q %VCPKG_TGT_DIR% +mkdir %VCPKG_TGT_DIR% + +rem License and .lib files +mkdir %VCPKG_TGT_DIR%\webrtc\share\webrtc\ +copy %WEBRTC_SRC_DIR%\LICENSE %VCPKG_TGT_DIR%\webrtc\share\webrtc\copyright +xcopy /v %RELEASE_LIB_DIR%\webrtc.lib %VCPKG_TGT_DIR%\webrtc\lib\ +xcopy /v %DEBUG_LIB_DIR%\webrtc.lib %VCPKG_TGT_DIR%\webrtc\debug\lib\ + +rem Header files +mkdir %VCPKG_TGT_DIR%\webrtc\include\webrtc\ +copy %WEBRTC_SRC_DIR%\common_types.h %VCPKG_TGT_DIR%\webrtc\include\webrtc +xcopy /v /s /i %WEBRTC_SRC_DIR%\api\*.h %VCPKG_TGT_DIR%\webrtc\include\webrtc\api +xcopy /v /s /i %WEBRTC_SRC_DIR%\audio\*.h %VCPKG_TGT_DIR%\webrtc\include\webrtc\audio +xcopy /v /s /i %WEBRTC_SRC_DIR%\base\*.h %VCPKG_TGT_DIR%\webrtc\include\webrtc\base +xcopy /v /s /i %WEBRTC_SRC_DIR%\call\*.h %VCPKG_TGT_DIR%\webrtc\include\webrtc\call +xcopy /v /s /i %WEBRTC_SRC_DIR%\common_audio\*.h %VCPKG_TGT_DIR%\webrtc\include\webrtc\common_audio +xcopy /v /s /i %WEBRTC_SRC_DIR%\common_video\*.h %VCPKG_TGT_DIR%\webrtc\include\webrtc\common_video +xcopy /v /s /i %WEBRTC_SRC_DIR%\logging\*.h %VCPKG_TGT_DIR%\webrtc\include\webrtc\logging +xcopy /v /s /i %WEBRTC_SRC_DIR%\media\*.h %VCPKG_TGT_DIR%\webrtc\include\webrtc\media +xcopy /v /s /i %WEBRTC_SRC_DIR%\modules\*.h %VCPKG_TGT_DIR%\webrtc\include\webrtc\modules +xcopy /v /s /i %WEBRTC_SRC_DIR%\p2p\*.h %VCPKG_TGT_DIR%\webrtc\include\webrtc\p2p +xcopy /v /s /i %WEBRTC_SRC_DIR%\pc\*.h %VCPKG_TGT_DIR%\webrtc\include\webrtc\pc +xcopy /v /s /i %WEBRTC_SRC_DIR%\rtc_base\*.h %VCPKG_TGT_DIR%\webrtc\include\webrtc\rtc_base +xcopy /v /s /i %WEBRTC_SRC_DIR%\rtc_tools\*.h %VCPKG_TGT_DIR%\webrtc\include\webrtc\rtc_tools +xcopy /v /s /i %WEBRTC_SRC_DIR%\stats\*.h %VCPKG_TGT_DIR%\webrtc\include\webrtc\stats +xcopy /v /s /i %WEBRTC_SRC_DIR%\system_wrappers\*.h %VCPKG_TGT_DIR%\webrtc\include\webrtc\system_wrappers +xcopy /v /s /i %WEBRTC_SRC_DIR%\third_party\abseil-cpp\absl\*.h %VCPKG_TGT_DIR%\webrtc\include\webrtc\absl +xcopy /v /s /i %WEBRTC_SRC_DIR%\third_party\libyuv\include\libyuv\*.h %VCPKG_TGT_DIR%\webrtc\include\webrtc\libyuv +xcopy /v /s /i %WEBRTC_SRC_DIR%\video\*.h %VCPKG_TGT_DIR%\webrtc\include\webrtc\video diff --git a/cmake/ports/webrtc/portfile.cmake b/cmake/ports/webrtc/portfile.cmake index fdc653d6a5e..3e81b9fd9c6 100644 --- a/cmake/ports/webrtc/portfile.cmake +++ b/cmake/ports/webrtc/portfile.cmake @@ -1,5 +1,5 @@ include(vcpkg_common_functions) -set(WEBRTC_VERSION 20190626) +set(WEBRTC_VERSION 20210105) set(MASTER_COPY_SOURCE_PATH ${CURRENT_BUILDTREES_DIR}/src) file(READ "${VCPKG_ROOT_DIR}/_env/EXTERNAL_BUILD_ASSETS.txt" EXTERNAL_BUILD_ASSETS) @@ -9,9 +9,9 @@ if (ANDROID) elseif (WIN32) vcpkg_download_distfile( WEBRTC_SOURCE_ARCHIVE - URLS "${EXTERNAL_BUILD_ASSETS}/seth/webrtc-20190626-windows.zip" - SHA512 c0848eddb1579b3bb0496b8785e24f30470f3c477145035fd729264a326a467b9467ae9f426aa5d72d168ad9e9bf2c279150744832736bdf39064d24b04de1a3 - FILENAME webrtc-20190626-windows.zip + URLS "${EXTERNAL_BUILD_ASSETS}/dependencies/vcpkg/webrtc-m84-20210105-windows.zip" + SHA512 12847f7e9df2e0539a6b017db88012a8978b1aa37ff2e8dbf019eb7438055395fdda3a74dc669b0a30330973a83bc57e86eca6f59b1c9eff8e2145a7ea4a532a + FILENAME webrtc-m84-20210105-windows.zip ) elseif (APPLE) vcpkg_download_distfile( diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index c03f37576db..c25dd61f688 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1176,7 +1176,9 @@ void AudioClient::configureWebrtc() { config.high_pass_filter.enabled = false; config.echo_canceller.enabled = true; config.echo_canceller.mobile_mode = false; +#if defined(WEBRTC_LEGACY) config.echo_canceller.use_legacy_aec = false; +#endif config.noise_suppression.enabled = false; config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kModerate; config.voice_detection.enabled = false; diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index 19ccb587a44..e1185acd6a6 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -58,6 +58,7 @@ #include "HifiAudioDeviceInfo.h" #if defined(WEBRTC_AUDIO) +# define WEBRTC_APM_DEBUG_DUMP 0 # include # include "modules/audio_processing/audio_processing_impl.h" #endif diff --git a/libraries/shared/src/shared/WebRTC.h b/libraries/shared/src/shared/WebRTC.h index e99c6430456..f2b64d7a2a9 100644 --- a/libraries/shared/src/shared/WebRTC.h +++ b/libraries/shared/src/shared/WebRTC.h @@ -22,6 +22,7 @@ #if defined(Q_OS_MAC) # define WEBRTC_AUDIO 1 # define WEBRTC_POSIX 1 +# define WEBRTC_LEGACY 1 #elif defined(Q_OS_WIN) # define WEBRTC_AUDIO 1 # define WEBRTC_DATA_CHANNEL 1 @@ -32,9 +33,11 @@ // I don't yet have a working libwebrtc for android // # define WEBRTC_AUDIO 1 // # define WEBRTC_POSIX 1 +// # define WEBRTC_LEGACY 1 #elif defined(Q_OS_LINUX) # define WEBRTC_AUDIO 1 # define WEBRTC_POSIX 1 +# define WEBRTC_LEGACY 1 #endif #endif // hifi_WebRTC_h From 9472d689c80bae18a1ebc2cea33ad85979d4d923 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Fri, 21 May 2021 11:51:30 +1200 Subject: [PATCH 05/82] Update to the latest Vircadia-Web master --- vircadia-web | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vircadia-web b/vircadia-web index d2e383cab81..ac1f13a39c7 160000 --- a/vircadia-web +++ b/vircadia-web @@ -1 +1 @@ -Subproject commit d2e383cab811018422433fe0d8f224e53e0506cf +Subproject commit ac1f13a39c702ee54bf2cda8bc35e5d34f7f0756 From aa1c32ec53142f436529e9e079178c7a8aeec332 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sat, 29 May 2021 09:25:15 +1200 Subject: [PATCH 06/82] Rename new domain server environment variable --- libraries/networking/src/DomainHandler.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/DomainHandler.h b/libraries/networking/src/DomainHandler.h index a8c316572a0..05cf8f26d57 100644 --- a/libraries/networking/src/DomainHandler.h +++ b/libraries/networking/src/DomainHandler.h @@ -44,9 +44,9 @@ const unsigned short DEFAULT_DOMAIN_SERVER_PORT = const unsigned short DEFAULT_DOMAIN_SERVER_WS_PORT = QProcessEnvironment::systemEnvironment() - .contains("HIFI_DOMAIN_SERVER_WS_PORT") + .contains("VIRCADIA_DOMAIN_SERVER_WS_PORT") ? QProcessEnvironment::systemEnvironment() - .value("HIFI_DOMAIN_SERVER_WS_PORT") + .value("VIRCADIA_DOMAIN_SERVER_WS_PORT") .toUShort() : 40102; // TCP From 6c3762846808c891d77413b172ba93d52a075603 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Mon, 31 May 2021 12:29:48 +1200 Subject: [PATCH 07/82] Add WebRTC data channel --- domain-server/src/DomainServer.cpp | 17 +- domain-server/src/DomainServer.h | 7 +- libraries/networking/CMakeLists.txt | 3 +- libraries/networking/src/NodeType.h | 6 +- .../src/webrtc/WebRTCDataChannels.cpp | 475 ++++++++++++++++++ .../src/webrtc/WebRTCDataChannels.h | 278 ++++++++++ .../src/webrtc/WebRTCSignalingServer.cpp | 4 +- .../src/webrtc/WebRTCSignalingServer.h | 12 +- libraries/shared/src/shared/WebRTC.h | 4 +- 9 files changed, 791 insertions(+), 15 deletions(-) create mode 100644 libraries/networking/src/webrtc/WebRTCDataChannels.cpp create mode 100644 libraries/networking/src/webrtc/WebRTCDataChannels.h diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 02fd810b0ef..c7a5700de27 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -165,8 +165,9 @@ bool DomainServer::forwardMetaverseAPIRequest(HTTPConnection* connection, DomainServer::DomainServer(int argc, char* argv[]) : QCoreApplication(argc, argv), _gatekeeper(this), -#ifdef WEBRTC_DATA_CHANNEL +#ifdef WEBRTC_DATA_CHANNELS _webrtcSignalingServer(QHostAddress::AnyIPv4, DEFAULT_DOMAIN_SERVER_WS_PORT, this), + _webrtcDataChannels(NodeType::DomainServer, this), #endif _httpManager(QHostAddress::AnyIPv4, DOMAIN_SERVER_HTTP_PORT, QString("%1/resources/web/").arg(QCoreApplication::applicationDirPath()), this) @@ -251,6 +252,8 @@ DomainServer::DomainServer(int argc, char* argv[]) : updateDownstreamNodes(); updateUpstreamNodes(); + setUpWebRTC(); + if (_type != NonMetaverse) { // if we have a metaverse domain, we'll use an access token for API calls resetAccountManagerAccessToken(); @@ -3136,6 +3139,18 @@ void DomainServer::updateUpstreamNodes() { updateReplicationNodes(Upstream); } +void DomainServer::setUpWebRTC() { + + // Inbound WebRTC signaling messages received from a client. + connect(&_webrtcSignalingServer, &WebRTCSignalingServer::messageReceived, + &_webrtcDataChannels, &WebRTCDataChannels::onSignalingMessage); + + // Outbound WebRTC signaling messages being sent to a client. + connect(&_webrtcDataChannels, &WebRTCDataChannels::signalingMessage, + &_webrtcSignalingServer, &WebRTCSignalingServer::sendMessage); + +} + void DomainServer::initializeExporter() { static const QString ENABLE_EXPORTER = "monitoring.enable_prometheus_exporter"; static const QString EXPORTER_PORT = "monitoring.prometheus_exporter_port"; diff --git a/domain-server/src/DomainServer.h b/domain-server/src/DomainServer.h index 29142505a89..23e12993747 100644 --- a/domain-server/src/DomainServer.h +++ b/domain-server/src/DomainServer.h @@ -28,6 +28,7 @@ #include #include #include +#include #include #include "AssetsBackupHandler.h" @@ -143,6 +144,9 @@ private slots: void updateReplicatedNodes(); void updateDownstreamNodes(); void updateUpstreamNodes(); + + void setUpWebRTC(); + void initializeExporter(); void initializeMetadataExporter(); @@ -314,8 +318,9 @@ private slots: QThread _assetClientThread; -#ifdef WEBRTC_DATA_CHANNEL +#ifdef WEBRTC_DATA_CHANNELS WebRTCSignalingServer _webrtcSignalingServer; + WebRTCDataChannels _webrtcDataChannels; #endif }; diff --git a/libraries/networking/CMakeLists.txt b/libraries/networking/CMakeLists.txt index 50382cda991..1835c7a6cd6 100644 --- a/libraries/networking/CMakeLists.txt +++ b/libraries/networking/CMakeLists.txt @@ -11,7 +11,8 @@ endif () if (WIN32) # we need ws2_32.lib on windows, but it's static so we don't bubble it up - target_link_libraries(${TARGET_NAME} ws2_32.lib) + # Libraries needed for WebRTC: security.log winmm.lib + target_link_libraries(${TARGET_NAME} ws2_32.lib security.lib winmm.lib) elseif(APPLE) # IOKit is needed for getting machine fingerprint find_library(FRAMEWORK_IOKIT IOKit) diff --git a/libraries/networking/src/NodeType.h b/libraries/networking/src/NodeType.h index 2b2cc4e0115..8539ce8fb3a 100644 --- a/libraries/networking/src/NodeType.h +++ b/libraries/networking/src/NodeType.h @@ -4,6 +4,7 @@ // // Created by Stephen Birarda on 05/29/15. // Copyright 2015 High Fidelity, Inc. +// Copyright 2021 Vircadia contributors. // // Distributed under the Apache License, Version 2.0. // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html @@ -14,6 +15,10 @@ #pragma once +/// @file +/// @brief NodeType + +/// @brief An 8-bit value identifying the type of a node - domain server, audio mixer, etc. typedef quint8 NodeType_t; namespace NodeType { @@ -37,7 +42,6 @@ namespace NodeType { NodeType_t upstreamType(NodeType_t primaryType); NodeType_t downstreamType(NodeType_t primaryType); - NodeType_t fromString(QString type); } diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp new file mode 100644 index 00000000000..321b844eb05 --- /dev/null +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp @@ -0,0 +1,475 @@ +// +// WebRTCDataChannels.cpp +// libraries/networking/src/webrtc +// +// Created by David Rowe on 21 May 2021. +// Copyright 2021 Vircadia contributors. +// + +#include "WebRTCDataChannels.h" + +#if defined(WEBRTC_DATA_CHANNELS) + +#include +#include + +#include "../NetworkLogging.h" + + +// References: +// - https://webrtc.github.io/webrtc-org/native-code/native-apis/ +// - https://webrtc.googlesource.com/src/+/master/api/peer_connection_interface.h + +const std::string ICE_SERVER_URI = "stun://ice.vircadia.com:7337"; + +#define WEBRTC_DEBUG + + +void WDCSetSessionDescriptionObserver::OnSuccess() { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCSetSessionDescriptionObserver::OnSuccess()"; +#endif +} + +void WDCSetSessionDescriptionObserver::OnFailure(RTCError error) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCSetSessionDescriptionObserver::OnFailure() :" << error.message(); +#endif +} + + +WDCCreateSessionDescriptionObserver::WDCCreateSessionDescriptionObserver(WDCConnection* parent) : + _parent(parent) +{ } + +void WDCCreateSessionDescriptionObserver::OnSuccess(SessionDescriptionInterface* description) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCCreateSessionDescriptionObserver::OnSuccess()"; +#endif + _parent->sendAnswer(description); + _parent->setLocalDescription(description); +} + +void WDCCreateSessionDescriptionObserver::OnFailure(RTCError error) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCCreateSessionDescriptionObserver::OnFailure() :" << error.message(); +#endif +} + + +WDCPeerConnectionObserver::WDCPeerConnectionObserver(WDCConnection* parent) : + _parent(parent) +{ } + +void WDCPeerConnectionObserver::OnSignalingChange(PeerConnectionInterface::SignalingState newState) { +#ifdef WEBRTC_DEBUG + QStringList states{ + "Stable", + "HaveLocalOffer", + "HaveLocalPrAnswer", + "HaveRemoteOffer", + "HaveRemotePrAnswer", + "Closed" + }; + qCDebug(networking_webrtc) << "WDCPeerConnectionObserver::OnSignalingChange()" << newState << states[newState]; +#endif +} + +void WDCPeerConnectionObserver::OnRenegotiationNeeded() { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCPeerConnectionObserver::OnRenegotiationNeeded()"; +#endif +} + +void WDCPeerConnectionObserver::OnIceGatheringChange(PeerConnectionInterface::IceGatheringState newState) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCPeerConnectionObserver::OnIceGatheringChange()" << newState; +#endif +} + +void WDCPeerConnectionObserver::OnIceCandidate(const IceCandidateInterface* candidate) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCPeerConnectionObserver::OnIceCandidate()"; +#endif + _parent->sendIceCandidate(candidate); +} + +void WDCPeerConnectionObserver::OnDataChannel(rtc::scoped_refptr dataChannel) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCPeerConnectionObserver::OnDataChannel()"; +#endif + _parent->onDataChannelOpened(dataChannel); +} + +void WDCPeerConnectionObserver::OnConnectionChange(PeerConnectionInterface::PeerConnectionState newState) { +} + + +WDCDataChannelObserver::WDCDataChannelObserver(WDCConnection* parent) : + _parent(parent) +{ } + +void WDCDataChannelObserver::OnStateChange() { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCDataChannelObserver::OnStateChange()"; +#endif + _parent->onDataChannelStateChanged(); +} + +void WDCDataChannelObserver::OnMessage(const DataBuffer& buffer) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCDataChannelObserver::OnMessage()"; +#endif + _parent->onDataChannelMessageReceived(buffer); +} + + +WDCConnection::WDCConnection(quint16 webSocketID, WebRTCDataChannels* parent) : + _webSocketID(webSocketID), + _parent(parent) +{ +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WebRTCDataChannels::WebRTCDataChannels()"; +#endif + + // Create observers. + _setSessionDescriptionObserver = new rtc::RefCountedObject(); + _createSessionDescriptionObserver = new rtc::RefCountedObject(this); + _dataChannelObserver = std::make_shared(this); + _peerConnectionObserver = std::make_shared(this); + + // Create new peer connection. + _peerConnection = _parent->createPeerConnection(_peerConnectionObserver); +}; + +void WDCConnection::setRemoteDescription(QJsonObject& description) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCConnection::setRemoteDescription() :" << description; +#endif + + SdpParseError sdpParseError; + auto sessionDescription = CreateSessionDescription( + description.value("type").toString().toStdString(), + description.value("sdp").toString().toStdString(), + &sdpParseError); + if (!sessionDescription) { + qCWarning(networking_webrtc) << "Error creating WebRTC remote description:" + << QString::fromStdString(sdpParseError.description); + return; + } + +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "3. Set remote description:" << sessionDescription; +#endif + _peerConnection->SetRemoteDescription(_setSessionDescriptionObserver, sessionDescription); +} + +void WDCConnection::createAnswer() { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCConnection::createAnswer()"; + qCDebug(networking_webrtc) << "4.a Create answer"; +#endif + _peerConnection->CreateAnswer(_createSessionDescriptionObserver, PeerConnectionInterface::RTCOfferAnswerOptions()); +} + +void WDCConnection::sendAnswer(SessionDescriptionInterface* description) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCConnection::sendAnswer()"; + qCDebug(networking_webrtc) << "4.b Send answer to the remote peer"; +#endif + + QJsonObject jsonDescription; + std::string descriptionString; + description->ToString(&descriptionString); + jsonDescription.insert("sdp", QString::fromStdString(descriptionString)); + jsonDescription.insert("type", "answer"); + + QJsonObject jsonWebRTCPayload; + jsonWebRTCPayload.insert("description", jsonDescription); + + QJsonObject jsonObject; + jsonObject.insert("from", QString(_parent->getNodeType())); + jsonObject.insert("to", _webSocketID); + jsonObject.insert("data", jsonWebRTCPayload); + + _parent->sendSignalingMessage(jsonObject); +} + +void WDCConnection::setLocalDescription(SessionDescriptionInterface* description) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCConnection::setLocalDescription()"; + qCDebug(networking_webrtc) << "5. Set local description"; +#endif + _peerConnection->SetLocalDescription(_setSessionDescriptionObserver, description); +} + +void WDCConnection::addIceCandidate(QJsonObject& data) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCConnection::addIceCandidate()"; +#endif + + SdpParseError sdpParseError; + auto iceCandidate = CreateIceCandidate( + data.value("sdpMid").toString().toStdString(), + data.value("sdpMLineIndex").toInt(), + data.value("candidate").toString().toStdString(), + &sdpParseError); + if (!iceCandidate) { + qCWarning(networking_webrtc) << "Error adding WebRTC ICE candidate:" + << QString::fromStdString(sdpParseError.description); + return; + } + +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "6. Add ICE candidate"; +#endif + _peerConnection->AddIceCandidate(iceCandidate); +} + +void WDCConnection::sendIceCandidate(const IceCandidateInterface* candidate) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCConnection::sendIceCandidate()"; +#endif + + std::string candidateString; + candidate->ToString(&candidateString); + QJsonObject jsonCandidate; + jsonCandidate.insert("candidate", QString::fromStdString(candidateString)); + jsonCandidate.insert("sdpMid", QString::fromStdString(candidate->sdp_mid())); + jsonCandidate.insert("sdpMLineIndex", candidate->sdp_mline_index()); + + QJsonObject jsonWebRTCData; + jsonWebRTCData.insert("candidate", jsonCandidate); + + QJsonObject jsonObject; + jsonObject.insert("from", QString(_parent->getNodeType())); + jsonObject.insert("to", _webSocketID); + jsonObject.insert("data", jsonWebRTCData); + QJsonDocument jsonDocument = QJsonDocument(jsonObject); + +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "7. Send ICE candidate to the remote peer"; +#endif + _parent->sendSignalingMessage(jsonObject); +} + +void WDCConnection::onDataChannelOpened(rtc::scoped_refptr dataChannel) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCConnection::onDataChannelOpened() :" + << dataChannel->id() + << QString::fromStdString(dataChannel->label()) + << QString::fromStdString(dataChannel->protocol()) + << dataChannel->negotiated() + << dataChannel->maxRetransmitTime() + << dataChannel->maxRetransmits() + << dataChannel->maxPacketLifeTime().value_or(-1) + << dataChannel->maxRetransmitsOpt().value_or(-1); +#endif + + _dataChannel = dataChannel; + _dataChannelID = dataChannel->id(); + _dataChannel->RegisterObserver(_dataChannelObserver.get()); + + _parent->onDataChannelOpened(this, _dataChannelID); +} + +void WDCConnection::onDataChannelStateChanged() { + auto state = _dataChannel->state(); +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCConnection::dataChannelStateChanged() :" << (int)state + << DataChannelInterface::DataStateString(state); +#endif + if (state == DataChannelInterface::kClosed) { + _parent->onDataChannelClosed(this, _dataChannelID); + } +} + +void WDCConnection::onDataChannelMessageReceived(const DataBuffer& buffer) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCConnection::onDataChannelMessageReceived()"; +#endif + + auto byteArray = QByteArray(buffer.data.data(), (int)buffer.data.size()); + + // Echo message back to sender. + if (byteArray.startsWith("echo:")) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "Echo message back"; +#endif + _parent->sendDataMessage(_dataChannelID, byteArray); // Use parent method to exercise the code stack. + return; + } + + _parent->emitDataMessage(_dataChannelID, byteArray); +} + +bool WDCConnection::sendDataMessage(const DataBuffer& buffer) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCConnection::sendDataMessage()"; +#endif + const int MAX_WEBRTC_BUFFER_SIZE = 16 * 1024 * 1024; // 16MB + if (_dataChannel->buffered_amount() + buffer.size() > MAX_WEBRTC_BUFFER_SIZE) { + // Don't send, otherwise the data channel will be closed. + return false; + } + return _dataChannel->Send(buffer); +} + + +WebRTCDataChannels::WebRTCDataChannels(NodeType_t nodeType, QObject* parent) : + _nodeType(nodeType), + _parent(parent) +{ +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WebRTCDataChannels::WebRTCDataChannels()"; +#endif + + // Create a peer connection factory. +#ifdef WEBRTC_DEBUG + // Numbers are per WebRTC's peer_connection_interface.h. + qCDebug(networking_webrtc) << "1. Create a new PeerConnectionFactoryInterface"; +#endif + _rtcNetworkThread = rtc::Thread::CreateWithSocketServer(); + _rtcNetworkThread->Start(); + _rtcWorkerThread = rtc::Thread::Create(); + _rtcWorkerThread->Start(); + _rtcSignalingThread = rtc::Thread::Create(); + _rtcSignalingThread->Start(); + PeerConnectionFactoryDependencies dependencies; + dependencies.network_thread = _rtcNetworkThread.get(); + dependencies.worker_thread = _rtcWorkerThread.get(); + dependencies.signaling_thread = _rtcSignalingThread.get(); + _peerConnectionFactory = CreateModularPeerConnectionFactory(std::move(dependencies)); + if (!_peerConnectionFactory) { + qCWarning(networking_webrtc) << "Failed to create WebRTC peer connection factory"; + } +} + +WebRTCDataChannels::~WebRTCDataChannels() { + QHashIterator i(_connectionsByDataChannel); + while (i.hasNext()) { + i.next(); + delete i.value(); + } + _connectionsByWebSocket.clear(); + _connectionsByDataChannel.clear(); + + _peerConnectionFactory = nullptr; + _rtcSignalingThread->Stop(); + _rtcSignalingThread = nullptr; + _rtcWorkerThread->Stop(); + _rtcWorkerThread = nullptr; + _rtcNetworkThread->Stop(); + _rtcNetworkThread = nullptr; +} + +void WebRTCDataChannels::onDataChannelOpened(WDCConnection* connection, int dataChannelID) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WebRTCDataChannels::onDataChannelOpened() :" << dataChannelID; +#endif + _connectionsByDataChannel.insert(dataChannelID, connection); +} + +void WebRTCDataChannels::onDataChannelClosed(WDCConnection* connection, int dataChannelID) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WebRTCDataChannels::onDataChannelClosed() :" << dataChannelID; +#endif + + // Delete WDCConnection. + _connectionsByWebSocket.remove(connection->getWebSocketID()); + _connectionsByDataChannel.remove(dataChannelID); + delete connection; +} + +void WebRTCDataChannels::onSignalingMessage(const QJsonObject& message) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WebRTCDataChannel::onSignalingMessage()" << message; +#endif + + // Validate message. + const int MAX_DEBUG_DETAIL_LENGTH = 64; + auto data = message.value("data").isObject() ? message.value("data").toObject() : QJsonObject(); + int from = message.value("from").isDouble() ? (quint16)(message.value("from").toInt()) : 0; + if (from <= 0 || from > MAXUINT16 || !data.contains("description") && !data.contains("candidate")) { + qCWarning(networking_webrtc) << "Unexpected signaling message:" + << QJsonDocument(message).toJson(QJsonDocument::Compact).left(MAX_DEBUG_DETAIL_LENGTH); + return; + } + + // Find or create a connection. + WDCConnection* connection; + if (_connectionsByWebSocket.contains(from)) { + connection = _connectionsByWebSocket.value(from); + } else { + connection = new WDCConnection(from, this); + _connectionsByWebSocket.insert(from, connection); + } + + // Set the remote description and reply with an answer. + if (data.contains("description")) { + auto description = data.value("description").toObject(); + if (description.value("type").toString() == "offer") { + connection->setRemoteDescription(description); + connection->createAnswer(); + } else { + qCWarning(networking_webrtc) << "Unexpected signaling description:" + << QJsonDocument(description).toJson(QJsonDocument::Compact).left(MAX_DEBUG_DETAIL_LENGTH); + } + } + + // Add a remote ICE candidate. + if (data.contains("candidate")) { + connection->addIceCandidate(data); + } + +} + +void WebRTCDataChannels::sendSignalingMessage(const QJsonObject& message) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WebRTCDataChannels::sendSignalingMessage() :" << QJsonDocument(message).toJson(QJsonDocument::Compact); +#endif + emit signalingMessage(message); +} + +void WebRTCDataChannels::emitDataMessage(int dataChannelID, const QByteArray& byteArray) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WebRTCDataChannels::emitDataMessage() :" << dataChannelID; +#endif + emit dataMessage(dataChannelID, byteArray); +} + +bool WebRTCDataChannels::sendDataMessage(int dataChannelID, const QByteArray& byteArray) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WebRTCDataChannels::sendDataMessage() :" << dataChannelID; +#endif + + // Find connection. + if (!_connectionsByDataChannel.contains(dataChannelID)) { + qCWarning(networking_webrtc) << "Could not find data channel to send message on!"; + return false; + } + + auto connection = _connectionsByDataChannel.value(dataChannelID); + DataBuffer buffer(byteArray.toStdString(), true); + return connection->sendDataMessage(buffer); +} + +rtc::scoped_refptr WebRTCDataChannels::createPeerConnection( + const std::shared_ptr peerConnectionObserver) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WebRTCDataChannels::createPeerConnection()"; +#endif + + PeerConnectionInterface::RTCConfiguration configuration; + PeerConnectionInterface::IceServer iceServer; + iceServer.uri = ICE_SERVER_URI; + configuration.servers.push_back(iceServer); + +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "2. Create a new PeerConnection"; +#endif + return _peerConnectionFactory->CreatePeerConnection(configuration, nullptr, nullptr, peerConnectionObserver.get()); +} + + +#endif // WEBRTC_DATA_CHANNELS diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.h b/libraries/networking/src/webrtc/WebRTCDataChannels.h new file mode 100644 index 00000000000..9ea75bc5ff7 --- /dev/null +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.h @@ -0,0 +1,278 @@ +// +// WebRTCDataChannels.h +// libraries/networking/src/webrtc +// +// Created by David Rowe on 21 May 2021. +// Copyright 2021 Vircadia contributors. +// + +#ifndef vircadia_WebRTCDataChannels_h +#define vircadia_WebRTCDataChannels_h + +#include + +#if defined(WEBRTC_DATA_CHANNELS) + + +#include +#include + +#undef emit // Avoid conflict between Qt signals/slots and the WebRTC library's. +#include +#define emit + +#include "../NodeType.h" + +using namespace webrtc; + +class WebRTCDataChannels; +class WDCConnection; + + +// A WebRTC data channel session description observer. +class WDCSetSessionDescriptionObserver : public SetSessionDescriptionObserver { +public: + // The call to SetLocalDescription or SetRemoteDescription succeeded. + void OnSuccess() override; + + // The call to SetLocalDescription or SetRemoteDescription failed. + void OnFailure(RTCError error) override; +}; + + +// A WebRTC data channel create session description observer. +class WDCCreateSessionDescriptionObserver : public CreateSessionDescriptionObserver { +public: + WDCCreateSessionDescriptionObserver(WDCConnection* parent); + + // The call to CreateAnswer succeeded. + void OnSuccess(SessionDescriptionInterface* desc) override; + + // The call to CreateAnswer failed. + void OnFailure(RTCError error) override; + +private: + WDCConnection* _parent; +}; + + +// A WebRTC data channel peer connection observer. +class WDCPeerConnectionObserver : public PeerConnectionObserver { +public: + WDCPeerConnectionObserver(WDCConnection* parent); + + // Triggered when the SignalingState changed. + void OnSignalingChange(PeerConnectionInterface::SignalingState newState) override; + + // Triggered when renegotiation is needed. For example, an ICE restart has begun. + void OnRenegotiationNeeded() override; + + // Called any time the IceGatheringState changes. + void OnIceGatheringChange(PeerConnectionInterface::IceGatheringState newState) override; + + // A new ICE candidate has been gathered. + void OnIceCandidate(const IceCandidateInterface* candidate) override; + + // Triggered when a remote peer opens a data channel. + void OnDataChannel(rtc::scoped_refptr dataChannel) override; + + // Called any time the PeerConnectionState changes. + void OnConnectionChange(PeerConnectionInterface::PeerConnectionState newState) override; + +private: + WDCConnection* _parent; +}; + + +// A WebRTC data channel observer. +class WDCDataChannelObserver : public DataChannelObserver { +public: + WDCDataChannelObserver(WDCConnection* parent); + + // The data channel state changed. + void OnStateChange() override; + + // A data buffer was successfully received. + void OnMessage(const DataBuffer& buffer) override; + +private: + WDCConnection* _parent; +}; + + +/// @brief A WebRTC data channel connection. +/// @details Opens and manages a WebRTC data channel connection. +class WDCConnection { + +public: + /// @brief Constructs a new WDCConnection and opens a WebRTC data connection. + /// @param webSocketID The signaling channel that initiated the opening of the WebRTC data channel. + /// @param parent The parent WebRTCDataChannels object. + WDCConnection(quint16 webSocketID, WebRTCDataChannels* parent); + + /// @brief Gets the WebSocket ID. + /// @return The ID of the WebSocket. + quint16 getWebSocketID() { return _webSocketID; } + + /// @brief Gets the WebRTC data channel ID. + /// @return The WebRTC data channel ID. `-1` if not open yet. + int getDataChannelID() { return _dataChannelID; } + + + /// @brief Sets the remote session description received from the remote client via the signaling channel. + /// @param description The remote session description. + void setRemoteDescription(QJsonObject& description); + + /// @brief Creates an answer to an offer received from the remote client via the signaling channel. + void createAnswer(); + + /// @brief Sends an answer to the remote client via the signaling channel. + /// @param description The answer. + void sendAnswer(SessionDescriptionInterface* description); + + /// @brief Sets the local session description on the WebRTC data channel being connected. + /// @param description The local session description. + void setLocalDescription(SessionDescriptionInterface* description); + + /// @brief Adds an ICE candidate received from the remote client via the signaling channel. + /// @param data The ICE candidate. + void addIceCandidate(QJsonObject& data); + + /// @brief Sends an ICE candidate to the remote vlient via the signaling channel. + /// @param candidate The ICE candidate. + void sendIceCandidate(const IceCandidateInterface* candidate); + + /// @brief Handles the WebRTC data channel being opened. + /// @param dataChannel The WebRTC data channel. + void onDataChannelOpened(rtc::scoped_refptr dataChannel); + + /// @brief Handles a change in the state of the WebRTC data channel. + void onDataChannelStateChanged(); + + + /// @brief Handles a message being received on the WebRTC data channel. + /// @param buffer The message received. + void onDataChannelMessageReceived(const DataBuffer& buffer); + + /// @brief Sends a message on the WebRTC data channel. + /// @param buffer The message to send. + /// @return `true` if the message was sent, otherwise `false`. + bool sendDataMessage(const DataBuffer& buffer); + +private: + WebRTCDataChannels* _parent; + quint16 _webSocketID { 0 }; + int _dataChannelID { -1 }; + + rtc::scoped_refptr _setSessionDescriptionObserver { nullptr }; + rtc::scoped_refptr _createSessionDescriptionObserver { nullptr }; + + std::shared_ptr _dataChannelObserver { nullptr }; + rtc::scoped_refptr _dataChannel { nullptr }; + + std::shared_ptr _peerConnectionObserver { nullptr }; + rtc::scoped_refptr _peerConnection { nullptr }; +}; + + +/// @brief Manages WebRTC data channels on the domain server or an assignment clients that Interface clients can connect to. +/// +/// @details Presents multiple individual WebRTC data channels as a single one-to-many WebRTCDataChannels object. Interface +/// clients may use WebRTC data channels for Vircadia protocol network communications instead of UDP. +/// A WebRTCSignalingServer is used in the process of setting up a WebRTC data channel between an Interface client and the +/// domain server or assignment client. +/// The Interface client initiates the connection - including initiating the data channel - and the domain server or assignment +/// client responds. +/// +/// Additionally, for debugging purposes, instead of containing a Vircadia protocol payload, a WebRTC message may be an echo +/// request. This is bounced back to the client. +/// +class WebRTCDataChannels : public QObject { + Q_OBJECT + +public: + + /// @brief Constructs a new WebRTCDataChannels object. + /// @paramm nodeType The type of node that the WebRTCDataChannels object is being used in. + /// @param parent The parent Qt object. + WebRTCDataChannels(NodeType_t nodeType, QObject* parent); + + /// @brief Destroys a WebRTCDataChannels object. + ~WebRTCDataChannels(); + + /// @brief Returns the type of node that the WebRTCDataChannels object is being used in. + /// @return The type of node. + NodeType_t getNodeType() { + return _nodeType; + } + + /// @brief Handles a WebRTC data channel opening. + /// @param connection The WebRTC data channel connection. + /// @param dataChannelID The WebRTC data channel ID. + void onDataChannelOpened(WDCConnection* connection, int dataChannelID); + + /// @brief Handles a WebRTC data channel closing. + /// @param connection The WebRTC data channel connection. + /// @param dataChannelID The WebRTC data channel ID. + void onDataChannelClosed(WDCConnection* connection, int dataChannelID); + + /// @brief Emits a signalingMessage received for the Interface client. + /// @param message The WebRTC signaling message to send. + void sendSignalingMessage(const QJsonObject& message); + + /// @brief Emits a dataMessage received from the Interface client. + /// @param dataChannelID The WebRTC data channel the message was received on. + /// @param byteArray The data message received. + void emitDataMessage(int dataChannelID, const QByteArray& byteArray); + + /// @brief Sends a data message to an Interface client. + /// @param dataChannelID The WebRTC channel ID of the Interface client. + /// @param message The data message to send. + /// @return `true` if the data message was sent, otherwise `false`. + bool sendDataMessage(int dataChannelID, const QByteArray& message); + + /// @brief Creates a new WebRTC peer connection for connecting to an Interface client. + /// @param peerConnectionObserver An observer to monitor the WebRTC peer connection. + /// @return The new WebRTC peer connection. + rtc::scoped_refptr createPeerConnection( + const std::shared_ptr peerConnectionObserver); + +public slots: + + /// @brief Handles a WebRTC signaling message received from the Interface client. + /// @param message The WebRTC signaling message. + void onSignalingMessage(const QJsonObject& message); + +signals: + + /// @brief A WebRTC signaling message to be sent to the Interface client. + /// @description This message is for the WebRTCSignalingServer to send. + /// @param message The WebRTC signaling message to send. + void signalingMessage(const QJsonObject& message); + + /// @brief A WebRTC data message received from the Interface client. + /// @description This message is for handling at a higher level in the Vircadia protocol. + /// @param dataChannelID The WebRTC data channel ID. + /// @param byteArray The Vircadia protocol message. + void dataMessage(int dataChannelID, const QByteArray& byteArray); + +private: + + QObject* _parent; + + NodeType_t _nodeType; + + std::unique_ptr _rtcNetworkThread { nullptr }; + std::unique_ptr _rtcWorkerThread { nullptr }; + std::unique_ptr _rtcSignalingThread { nullptr }; + + rtc::scoped_refptr _peerConnectionFactory { nullptr }; + + QHash _connectionsByWebSocket; + QHash _connectionsByDataChannel; +}; + + +#endif // WEBRTC_DATA_CHANNELS + +#endif // vircadia_WebRTCDataChannels_h diff --git a/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp b/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp index 66b1d6616c0..8c661bbf8cc 100644 --- a/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp +++ b/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp @@ -8,7 +8,7 @@ #include "WebRTCSignalingServer.h" -#if defined(WEBRTC_DATA_CHANNEL) +#if defined(WEBRTC_DATA_CHANNELS) #include #include @@ -93,4 +93,4 @@ void WebRTCSignalingServer::newWebSocketConnection() { _webSockets.insert(webSocket->peerPort(), webSocket); } -#endif // WEBRTC_DATA_CHANNEL +#endif // WEBRTC_DATA_CHANNELS diff --git a/libraries/networking/src/webrtc/WebRTCSignalingServer.h b/libraries/networking/src/webrtc/WebRTCSignalingServer.h index 41d79dbd571..9f4214d3c79 100644 --- a/libraries/networking/src/webrtc/WebRTCSignalingServer.h +++ b/libraries/networking/src/webrtc/WebRTCSignalingServer.h @@ -2,18 +2,16 @@ // WebRTCSignalingServer.h // libraries/networking/src/webrtc // -// Provides a signaling channel for setting up WebRTC connections between the Web app and the domain servers and mixers. -// // Created by David Rowe on 16 May 2021. // Copyright 2021 Vircadia contributors. // -#ifndef vircadia_SignalingServer_h -#define vircadia_SignalingServer_h +#ifndef vircadia_WebRTCSignalingServer_h +#define vircadia_WebRTCSignalingServer_h #include -#if defined(WEBRTC_DATA_CHANNEL) +#if defined(WEBRTC_DATA_CHANNELS) #include #include @@ -99,6 +97,6 @@ private slots: }; -#endif // WEBRTC_DATA_CHANNEL +#endif // WEBRTC_DATA_CHANNELS -#endif // vircadia_SignalingServer_h +#endif // vircadia_WebRTCSignalingServer_h diff --git a/libraries/shared/src/shared/WebRTC.h b/libraries/shared/src/shared/WebRTC.h index f2b64d7a2a9..888877eadb3 100644 --- a/libraries/shared/src/shared/WebRTC.h +++ b/libraries/shared/src/shared/WebRTC.h @@ -17,7 +17,7 @@ #endif // WEBRTC_AUDIO: WebRTC audio features, e.g., echo canceling. -// WEBRTC_DATA_CHANNEL: WebRTC client-server connections in parallel with UDP. +// WEBRTC_DATA_CHANNELS: WebRTC client-server connections in parallel with UDP. #if defined(Q_OS_MAC) # define WEBRTC_AUDIO 1 @@ -25,7 +25,7 @@ # define WEBRTC_LEGACY 1 #elif defined(Q_OS_WIN) # define WEBRTC_AUDIO 1 -# define WEBRTC_DATA_CHANNEL 1 +# define WEBRTC_DATA_CHANNELS 1 # define WEBRTC_WIN 1 # define NOMINMAX 1 # define WIN32_LEAN_AND_MEAN 1 From e6c49cf407f395bdbb4ede7dcfee7b4706e3ddc3 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Tue, 1 Jun 2021 09:32:28 +1200 Subject: [PATCH 08/82] Fix data channel ID --- .../networking/src/webrtc/WebRTCDataChannels.cpp | 16 +++++++++++----- .../networking/src/webrtc/WebRTCDataChannels.h | 12 +++++++++--- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp index 321b844eb05..92cba6fd96f 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp @@ -129,7 +129,7 @@ WDCConnection::WDCConnection(quint16 webSocketID, WebRTCDataChannels* parent) : _parent(parent) { #ifdef WEBRTC_DEBUG - qCDebug(networking_webrtc) << "WebRTCDataChannels::WebRTCDataChannels()"; + qCDebug(networking_webrtc) << "WDCConnection::WDCConnection() :" << webSocketID; #endif // Create observers. @@ -267,7 +267,7 @@ void WDCConnection::onDataChannelOpened(rtc::scoped_refptr #endif _dataChannel = dataChannel; - _dataChannelID = dataChannel->id(); + _dataChannelID = _parent->getNewDataChannelID(); // Not dataChannel->id() because it's only unique per peer connection. _dataChannel->RegisterObserver(_dataChannelObserver.get()); _parent->onDataChannelOpened(this, _dataChannelID); @@ -346,7 +346,7 @@ WebRTCDataChannels::WebRTCDataChannels(NodeType_t nodeType, QObject* parent) : } WebRTCDataChannels::~WebRTCDataChannels() { - QHashIterator i(_connectionsByDataChannel); + QHashIterator i(_connectionsByDataChannel); while (i.hasNext()) { i.next(); delete i.value(); @@ -363,14 +363,20 @@ WebRTCDataChannels::~WebRTCDataChannels() { _rtcNetworkThread = nullptr; } -void WebRTCDataChannels::onDataChannelOpened(WDCConnection* connection, int dataChannelID) { +quint16 WebRTCDataChannels::getNewDataChannelID() { + static const int QUINT16_LIMIT = std::numeric_limits::max() + 1; + _lastDataChannelID = std::max((_lastDataChannelID + 1) % QUINT16_LIMIT, 1); + return _lastDataChannelID; +} + +void WebRTCDataChannels::onDataChannelOpened(WDCConnection* connection, quint16 dataChannelID) { #ifdef WEBRTC_DEBUG qCDebug(networking_webrtc) << "WebRTCDataChannels::onDataChannelOpened() :" << dataChannelID; #endif _connectionsByDataChannel.insert(dataChannelID, connection); } -void WebRTCDataChannels::onDataChannelClosed(WDCConnection* connection, int dataChannelID) { +void WebRTCDataChannels::onDataChannelClosed(WDCConnection* connection, quint16 dataChannelID) { #ifdef WEBRTC_DEBUG qCDebug(networking_webrtc) << "WebRTCDataChannels::onDataChannelClosed() :" << dataChannelID; #endif diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.h b/libraries/networking/src/webrtc/WebRTCDataChannels.h index 9ea75bc5ff7..ca14e9ae818 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.h +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.h @@ -206,15 +206,19 @@ class WebRTCDataChannels : public QObject { return _nodeType; } + /// @brief Get a new data channel ID to uniquely identify a WDCConnection. + /// @return A new data channel ID. + quint16 getNewDataChannelID(); + /// @brief Handles a WebRTC data channel opening. /// @param connection The WebRTC data channel connection. /// @param dataChannelID The WebRTC data channel ID. - void onDataChannelOpened(WDCConnection* connection, int dataChannelID); + void onDataChannelOpened(WDCConnection* connection, quint16 dataChannelID); /// @brief Handles a WebRTC data channel closing. /// @param connection The WebRTC data channel connection. /// @param dataChannelID The WebRTC data channel ID. - void onDataChannelClosed(WDCConnection* connection, int dataChannelID); + void onDataChannelClosed(WDCConnection* connection, quint16 dataChannelID); /// @brief Emits a signalingMessage received for the Interface client. /// @param message The WebRTC signaling message to send. @@ -268,8 +272,10 @@ public slots: rtc::scoped_refptr _peerConnectionFactory { nullptr }; + quint16 _lastDataChannelID { 0 }; + QHash _connectionsByWebSocket; - QHash _connectionsByDataChannel; + QHash _connectionsByDataChannel; }; From 63ec9332901c1a0025e0080396fbe329672bc03e Mon Sep 17 00:00:00 2001 From: David Rowe Date: Tue, 1 Jun 2021 09:34:17 +1200 Subject: [PATCH 09/82] Code tidying --- .../networking/src/webrtc/WebRTCDataChannels.cpp | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp index 92cba6fd96f..497ecf9a555 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp @@ -280,6 +280,11 @@ void WDCConnection::onDataChannelStateChanged() { << DataChannelInterface::DataStateString(state); #endif if (state == DataChannelInterface::kClosed) { + _dataChannel->Close(); + _dataChannel = nullptr; + // WEBRTC FIXME: The following line causes the _peerConnectionFactory to fail. + //_peerConnection->Close(); + //_peerConnection = nullptr; _parent->onDataChannelClosed(this, _dataChannelID); } } @@ -346,6 +351,9 @@ WebRTCDataChannels::WebRTCDataChannels(NodeType_t nodeType, QObject* parent) : } WebRTCDataChannels::~WebRTCDataChannels() { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WebRTCDataChannels::~WebRTCDataChannels()"; +#endif QHashIterator i(_connectionsByDataChannel); while (i.hasNext()) { i.next(); @@ -384,7 +392,8 @@ void WebRTCDataChannels::onDataChannelClosed(WDCConnection* connection, quint16 // Delete WDCConnection. _connectionsByWebSocket.remove(connection->getWebSocketID()); _connectionsByDataChannel.remove(dataChannelID); - delete connection; + // WEBRTC FIXME: The following line causes the _peerConnectionFactory to fail. + //delete connection; } void WebRTCDataChannels::onSignalingMessage(const QJsonObject& message) { @@ -474,7 +483,9 @@ rtc::scoped_refptr WebRTCDataChannels::createPeerConnec #ifdef WEBRTC_DEBUG qCDebug(networking_webrtc) << "2. Create a new PeerConnection"; #endif - return _peerConnectionFactory->CreatePeerConnection(configuration, nullptr, nullptr, peerConnectionObserver.get()); + PeerConnectionDependencies dependencies(peerConnectionObserver.get()); + auto result = _peerConnectionFactory->CreatePeerConnection(configuration, std::move(dependencies)); + return result; } From c28f4749ed8ca3fe996afdba7be53864e22e6938 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Tue, 1 Jun 2021 09:34:38 +1200 Subject: [PATCH 10/82] Doxygen tidying --- .../src/webrtc/WebRTCDataChannels.h | 57 ++++++++++++------- .../src/webrtc/WebRTCSignalingServer.h | 12 ++-- 2 files changed, 44 insertions(+), 25 deletions(-) diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.h b/libraries/networking/src/webrtc/WebRTCDataChannels.h index ca14e9ae818..d9bb213a1df 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.h +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.h @@ -29,26 +29,33 @@ class WebRTCDataChannels; class WDCConnection; -// A WebRTC data channel session description observer. +/// @addtogroup Networking +/// @{ + +/// @brief A WebRTC session description observer. class WDCSetSessionDescriptionObserver : public SetSessionDescriptionObserver { public: - // The call to SetLocalDescription or SetRemoteDescription succeeded. + + /// @brief The call to SetLocalDescription or SetRemoteDescription succeeded. void OnSuccess() override; - // The call to SetLocalDescription or SetRemoteDescription failed. + /// @brief The call to SetLocalDescription or SetRemoteDescription failed. + /// @param error Error information. void OnFailure(RTCError error) override; }; -// A WebRTC data channel create session description observer. +/// @brief A WebRTC create session description observer. class WDCCreateSessionDescriptionObserver : public CreateSessionDescriptionObserver { public: WDCCreateSessionDescriptionObserver(WDCConnection* parent); - // The call to CreateAnswer succeeded. + /// @brief The call to CreateAnswer succeeded. + /// @param The session description. void OnSuccess(SessionDescriptionInterface* desc) override; - // The call to CreateAnswer failed. + //@ @brief The call to CreateAnswer failed. + /// @param error Error information. void OnFailure(RTCError error) override; private: @@ -56,27 +63,32 @@ class WDCCreateSessionDescriptionObserver : public CreateSessionDescriptionObser }; -// A WebRTC data channel peer connection observer. +/// @brief A WebRTC peer connection observer. class WDCPeerConnectionObserver : public PeerConnectionObserver { public: WDCPeerConnectionObserver(WDCConnection* parent); - // Triggered when the SignalingState changed. + /// @brief Called when the SignalingState changes. + /// @param newState The new signaling state. void OnSignalingChange(PeerConnectionInterface::SignalingState newState) override; - // Triggered when renegotiation is needed. For example, an ICE restart has begun. + /// @brief Called when renegotiation is needed. For example, an ICE restart has begun. void OnRenegotiationNeeded() override; - // Called any time the IceGatheringState changes. + /// @brief Called when the ICE gather state changes. + /// @param newState The new ICE gathering state. void OnIceGatheringChange(PeerConnectionInterface::IceGatheringState newState) override; - // A new ICE candidate has been gathered. + /// @brief Called when a new ICE candidate has been gathered. + /// @param candidate The new ICE candidate. void OnIceCandidate(const IceCandidateInterface* candidate) override; - // Triggered when a remote peer opens a data channel. + /// @brief Called when a remote peer opens a data channel. + /// @param dataChannel The data channel. void OnDataChannel(rtc::scoped_refptr dataChannel) override; - // Called any time the PeerConnectionState changes. + /// @brief Called when the peer connection state changes. + /// @param newState The new peer connection state. void OnConnectionChange(PeerConnectionInterface::PeerConnectionState newState) override; private: @@ -84,15 +96,16 @@ class WDCPeerConnectionObserver : public PeerConnectionObserver { }; -// A WebRTC data channel observer. +/// @brief A WebRTC data channel observer. class WDCDataChannelObserver : public DataChannelObserver { public: WDCDataChannelObserver(WDCConnection* parent); - // The data channel state changed. + /// @brief The data channel state changed. void OnStateChange() override; - // A data buffer was successfully received. + /// @brief A data channel message was received. + /// @param The message received. void OnMessage(const DataBuffer& buffer) override; private: @@ -175,7 +188,7 @@ class WDCConnection { }; -/// @brief Manages WebRTC data channels on the domain server or an assignment clients that Interface clients can connect to. +/// @brief Manages WebRTC data channels on the domain server or an assignment client that Interface clients can connect to. /// /// @details Presents multiple individual WebRTC data channels as a single one-to-many WebRTCDataChannels object. Interface /// clients may use WebRTC data channels for Vircadia protocol network communications instead of UDP. @@ -193,7 +206,7 @@ class WebRTCDataChannels : public QObject { public: /// @brief Constructs a new WebRTCDataChannels object. - /// @paramm nodeType The type of node that the WebRTCDataChannels object is being used in. + /// @param nodeType The type of node that the WebRTCDataChannels object is being used in. /// @param parent The parent Qt object. WebRTCDataChannels(NodeType_t nodeType, QObject* parent); @@ -220,7 +233,7 @@ class WebRTCDataChannels : public QObject { /// @param dataChannelID The WebRTC data channel ID. void onDataChannelClosed(WDCConnection* connection, quint16 dataChannelID); - /// @brief Emits a signalingMessage received for the Interface client. + /// @brief Emits a signalingMessage to be sent to the Interface client. /// @param message The WebRTC signaling message to send. void sendSignalingMessage(const QJsonObject& message); @@ -250,12 +263,12 @@ public slots: signals: /// @brief A WebRTC signaling message to be sent to the Interface client. - /// @description This message is for the WebRTCSignalingServer to send. + /// @details This message is for the WebRTCSignalingServer to send. /// @param message The WebRTC signaling message to send. void signalingMessage(const QJsonObject& message); /// @brief A WebRTC data message received from the Interface client. - /// @description This message is for handling at a higher level in the Vircadia protocol. + /// @details This message is for handling at a higher level in the Vircadia protocol. /// @param dataChannelID The WebRTC data channel ID. /// @param byteArray The Vircadia protocol message. void dataMessage(int dataChannelID, const QByteArray& byteArray); @@ -279,6 +292,8 @@ public slots: }; +/// @} + #endif // WEBRTC_DATA_CHANNELS #endif // vircadia_WebRTCDataChannels_h diff --git a/libraries/networking/src/webrtc/WebRTCSignalingServer.h b/libraries/networking/src/webrtc/WebRTCSignalingServer.h index 9f4214d3c79..f2e8594b91f 100644 --- a/libraries/networking/src/webrtc/WebRTCSignalingServer.h +++ b/libraries/networking/src/webrtc/WebRTCSignalingServer.h @@ -19,8 +19,11 @@ #include "../HifiSockAddr.h" -/// @brief WebRTC signaling server that Interface clients can use to initiate WebRTC connections to the domain server and -/// assignment clients. +/// @addtogroup Networking +/// @{ + +/// @brief Provides a WebRTC signaling server that Interface clients can use to initiate WebRTC connections to the domain server +/// and its assignment clients. /// /// @details The signaling server is expected to be hosted in the domain server. It provides a WebSocket for Interface clients /// to use in the WebRTC signaling handshake process to establish WebRTC data channel connections to each of the domain server @@ -48,14 +51,14 @@ /// | `to` | WebSocket port number | /// | `from` | NodeType | /// | [`data`] | WebRTC payload | -/// | [`echo`] | Echo request | +/// | [`echo`] | Echo response | /// class WebRTCSignalingServer : public QObject { Q_OBJECT public: - /// @brief Constructs a new WebRTCSignalingServer. + /// @brief Constructs a new WebRTCSignalingServer object. /// @param address The IP address to use for the WebSocket. /// @param port The port to use for the WebSocket. /// @param parent Qt parent object. @@ -96,6 +99,7 @@ private slots: QTimer* _isWebSocketServerListeningTimer; }; +/// @} #endif // WEBRTC_DATA_CHANNELS From 645dc265ab5a87792ba055a9770f369c0c59c45d Mon Sep 17 00:00:00 2001 From: David Rowe Date: Tue, 1 Jun 2021 15:41:25 +1200 Subject: [PATCH 11/82] Fix non-WebRTC domain server builds --- domain-server/src/DomainServer.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index c7a5700de27..4f80e82681d 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -3140,6 +3140,7 @@ void DomainServer::updateUpstreamNodes() { } void DomainServer::setUpWebRTC() { +#ifdef WEBRTC_DATA_CHANNELS // Inbound WebRTC signaling messages received from a client. connect(&_webrtcSignalingServer, &WebRTCSignalingServer::messageReceived, @@ -3149,6 +3150,7 @@ void DomainServer::setUpWebRTC() { connect(&_webrtcDataChannels, &WebRTCDataChannels::signalingMessage, &_webrtcSignalingServer, &WebRTCSignalingServer::sendMessage); +#endif } void DomainServer::initializeExporter() { From 1dcee6cc1dbf5fb12f97796baf3c3a9979053285 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Wed, 2 Jun 2021 23:00:19 +1200 Subject: [PATCH 12/82] Conditionally include WebRTCSignalingServer.h --- domain-server/src/DomainServer.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/domain-server/src/DomainServer.h b/domain-server/src/DomainServer.h index 29142505a89..2100a9ba141 100644 --- a/domain-server/src/DomainServer.h +++ b/domain-server/src/DomainServer.h @@ -28,7 +28,9 @@ #include #include #include +#if defined(WEBRTC_DATA_CHANNEL) #include +#endif #include "AssetsBackupHandler.h" #include "DomainGatekeeper.h" From 28c408de9839dbad34b987607bbc7c6c20e53e0f Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sat, 5 Jun 2021 18:23:37 +1200 Subject: [PATCH 13/82] Typo --- domain-server/src/DomainServer.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/domain-server/src/DomainServer.h b/domain-server/src/DomainServer.h index 5d2ac762acf..eccf67d5b62 100644 --- a/domain-server/src/DomainServer.h +++ b/domain-server/src/DomainServer.h @@ -28,7 +28,7 @@ #include #include #include -#if defined(WEBRTC_DATA_CHANNEL) +#if defined(WEBRTC_DATA_CHANNELS) #include #include #endif From c9d4fe8558a0ebbaedddfd6187af0758f552aa7c Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sat, 26 Jun 2021 17:23:58 +1200 Subject: [PATCH 14/82] Remove unused code --- libraries/networking/src/HifiSockAddr.cpp | 10 ---------- libraries/networking/src/HifiSockAddr.h | 1 - libraries/networking/src/NodeList.cpp | 1 - libraries/networking/src/PacketReceiver.cpp | 4 +--- 4 files changed, 1 insertion(+), 15 deletions(-) diff --git a/libraries/networking/src/HifiSockAddr.cpp b/libraries/networking/src/HifiSockAddr.cpp index 086dd084892..434f8daa3da 100644 --- a/libraries/networking/src/HifiSockAddr.cpp +++ b/libraries/networking/src/HifiSockAddr.cpp @@ -73,16 +73,6 @@ HifiSockAddr::HifiSockAddr(const QString& hostname, quint16 hostOrderPort, bool } } -HifiSockAddr::HifiSockAddr(const sockaddr* sockaddr) { - _address = QHostAddress(sockaddr); - - if (sockaddr->sa_family == AF_INET) { - _port = ntohs(reinterpret_cast(sockaddr)->sin_port); - } else { - _port = ntohs(reinterpret_cast(sockaddr)->sin6_port); - } -} - void HifiSockAddr::swap(HifiSockAddr& otherSockAddr) { using std::swap; diff --git a/libraries/networking/src/HifiSockAddr.h b/libraries/networking/src/HifiSockAddr.h index dcf7f9a6a94..fc9fdd15a58 100644 --- a/libraries/networking/src/HifiSockAddr.h +++ b/libraries/networking/src/HifiSockAddr.h @@ -26,7 +26,6 @@ class HifiSockAddr : public QObject { HifiSockAddr(const QHostAddress& address, quint16 port); HifiSockAddr(const HifiSockAddr& otherSockAddr); HifiSockAddr(const QString& hostname, quint16 hostOrderPort, bool shouldBlockForLookup = false); - HifiSockAddr(const sockaddr* sockaddr); bool isNull() const { return _address.isNull() && _port == 0; } void clear() { _address.clear(); _port = 0;} diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index 31534f34f4f..90027460617 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -452,7 +452,6 @@ void NodeList::sendDomainServerCheckIn() { packetStream << hardwareAddress; // now add the machine fingerprint - auto accountManager = DependencyManager::get(); packetStream << FingerprintUtils::getMachineFingerprint(); platform::json all = platform::getAll(); diff --git a/libraries/networking/src/PacketReceiver.cpp b/libraries/networking/src/PacketReceiver.cpp index c13fb8566b7..18009f52411 100644 --- a/libraries/networking/src/PacketReceiver.cpp +++ b/libraries/networking/src/PacketReceiver.cpp @@ -139,9 +139,7 @@ void PacketReceiver::handleVerifiedPacket(std::unique_ptr packet) { if (_shouldDropPackets) { return; } - - auto nodeList = DependencyManager::get(); - + // setup an NLPacket from the packet we were passed auto nlPacket = NLPacket::fromBase(std::move(packet)); auto receivedMessage = QSharedPointer::create(*nlPacket); From 5b937a158014b0070466107145c7351d94f74ea4 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sat, 26 Jun 2021 20:07:26 +1200 Subject: [PATCH 15/82] WebRTCSignalingServer and WebRTCDataChannels improvements --- .../src/webrtc/WebRTCDataChannels.cpp | 47 ++++++++---- .../src/webrtc/WebRTCDataChannels.h | 75 +++++++++++-------- .../src/webrtc/WebRTCSignalingServer.cpp | 24 +++--- .../src/webrtc/WebRTCSignalingServer.h | 13 ++-- 4 files changed, 98 insertions(+), 61 deletions(-) diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp index 497ecf9a555..fb0cd9cf6e1 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp @@ -24,6 +24,8 @@ const std::string ICE_SERVER_URI = "stun://ice.vircadia.com:7337"; #define WEBRTC_DEBUG +using namespace webrtc; + void WDCSetSessionDescriptionObserver::OnSuccess() { #ifdef WEBRTC_DEBUG @@ -308,6 +310,13 @@ void WDCConnection::onDataChannelMessageReceived(const DataBuffer& buffer) { _parent->emitDataMessage(_dataChannelID, byteArray); } +qint64 WDCConnection::getBufferedAmount() const { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCConnection::getBufferedAmount()"; +#endif + return _dataChannel->buffered_amount(); +} + bool WDCConnection::sendDataMessage(const DataBuffer& buffer) { #ifdef WEBRTC_DEBUG qCDebug(networking_webrtc) << "WDCConnection::sendDataMessage()"; @@ -321,12 +330,13 @@ bool WDCConnection::sendDataMessage(const DataBuffer& buffer) { } -WebRTCDataChannels::WebRTCDataChannels(NodeType_t nodeType, QObject* parent) : - _nodeType(nodeType), - _parent(parent) +WebRTCDataChannels::WebRTCDataChannels(QObject* parent, NodeType_t nodeType) : + QObject(parent), + _parent(parent), + _nodeType(nodeType) { #ifdef WEBRTC_DEBUG - qCDebug(networking_webrtc) << "WebRTCDataChannels::WebRTCDataChannels()"; + qCDebug(networking_webrtc) << "WebRTCDataChannels::WebRTCDataChannels()" << nodeType << NodeType::getNodeTypeName(nodeType); #endif // Create a peer connection factory. @@ -354,14 +364,7 @@ WebRTCDataChannels::~WebRTCDataChannels() { #ifdef WEBRTC_DEBUG qCDebug(networking_webrtc) << "WebRTCDataChannels::~WebRTCDataChannels()"; #endif - QHashIterator i(_connectionsByDataChannel); - while (i.hasNext()) { - i.next(); - delete i.value(); - } - _connectionsByWebSocket.clear(); - _connectionsByDataChannel.clear(); - + reset(); _peerConnectionFactory = nullptr; _rtcSignalingThread->Stop(); _rtcSignalingThread = nullptr; @@ -371,6 +374,16 @@ WebRTCDataChannels::~WebRTCDataChannels() { _rtcNetworkThread = nullptr; } +void WebRTCDataChannels::reset() { + QHashIterator i(_connectionsByDataChannel); + while (i.hasNext()) { + i.next(); + delete i.value(); + } + _connectionsByWebSocket.clear(); + _connectionsByDataChannel.clear(); +} + quint16 WebRTCDataChannels::getNewDataChannelID() { static const int QUINT16_LIMIT = std::numeric_limits::max() + 1; _lastDataChannelID = std::max((_lastDataChannelID + 1) % QUINT16_LIMIT, 1); @@ -448,7 +461,7 @@ void WebRTCDataChannels::sendSignalingMessage(const QJsonObject& message) { void WebRTCDataChannels::emitDataMessage(int dataChannelID, const QByteArray& byteArray) { #ifdef WEBRTC_DEBUG - qCDebug(networking_webrtc) << "WebRTCDataChannels::emitDataMessage() :" << dataChannelID; + qCDebug(networking_webrtc) << "WebRTCDataChannels::emitDataMessage() :" << dataChannelID << byteArray; #endif emit dataMessage(dataChannelID, byteArray); } @@ -469,6 +482,14 @@ bool WebRTCDataChannels::sendDataMessage(int dataChannelID, const QByteArray& by return connection->sendDataMessage(buffer); } +/// @brief Gets the number of bytes waiting to be written on a data channel. +/// @param port The data channel ID. +/// @return The number of bytes waiting to be written on the data channel. +qint64 WebRTCDataChannels::getBufferedAmount(int dataChannelID) const { + auto connection = _connectionsByDataChannel.value(dataChannelID); + return connection->getBufferedAmount(); +} + rtc::scoped_refptr WebRTCDataChannels::createPeerConnection( const std::shared_ptr peerConnectionObserver) { #ifdef WEBRTC_DEBUG diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.h b/libraries/networking/src/webrtc/WebRTCDataChannels.h index d9bb213a1df..21463b1c5f5 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.h +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.h @@ -23,8 +23,6 @@ #include "../NodeType.h" -using namespace webrtc; - class WebRTCDataChannels; class WDCConnection; @@ -33,7 +31,7 @@ class WDCConnection; /// @{ /// @brief A WebRTC session description observer. -class WDCSetSessionDescriptionObserver : public SetSessionDescriptionObserver { +class WDCSetSessionDescriptionObserver : public webrtc::SetSessionDescriptionObserver { public: /// @brief The call to SetLocalDescription or SetRemoteDescription succeeded. @@ -41,22 +39,22 @@ class WDCSetSessionDescriptionObserver : public SetSessionDescriptionObserver { /// @brief The call to SetLocalDescription or SetRemoteDescription failed. /// @param error Error information. - void OnFailure(RTCError error) override; + void OnFailure(webrtc::RTCError error) override; }; /// @brief A WebRTC create session description observer. -class WDCCreateSessionDescriptionObserver : public CreateSessionDescriptionObserver { +class WDCCreateSessionDescriptionObserver : public webrtc::CreateSessionDescriptionObserver { public: WDCCreateSessionDescriptionObserver(WDCConnection* parent); /// @brief The call to CreateAnswer succeeded. /// @param The session description. - void OnSuccess(SessionDescriptionInterface* desc) override; + void OnSuccess(webrtc::SessionDescriptionInterface* desc) override; //@ @brief The call to CreateAnswer failed. /// @param error Error information. - void OnFailure(RTCError error) override; + void OnFailure(webrtc::RTCError error) override; private: WDCConnection* _parent; @@ -64,32 +62,32 @@ class WDCCreateSessionDescriptionObserver : public CreateSessionDescriptionObser /// @brief A WebRTC peer connection observer. -class WDCPeerConnectionObserver : public PeerConnectionObserver { +class WDCPeerConnectionObserver : public webrtc::PeerConnectionObserver { public: WDCPeerConnectionObserver(WDCConnection* parent); /// @brief Called when the SignalingState changes. /// @param newState The new signaling state. - void OnSignalingChange(PeerConnectionInterface::SignalingState newState) override; + void OnSignalingChange(webrtc::PeerConnectionInterface::SignalingState newState) override; /// @brief Called when renegotiation is needed. For example, an ICE restart has begun. void OnRenegotiationNeeded() override; /// @brief Called when the ICE gather state changes. /// @param newState The new ICE gathering state. - void OnIceGatheringChange(PeerConnectionInterface::IceGatheringState newState) override; + void OnIceGatheringChange(webrtc::PeerConnectionInterface::IceGatheringState newState) override; /// @brief Called when a new ICE candidate has been gathered. /// @param candidate The new ICE candidate. - void OnIceCandidate(const IceCandidateInterface* candidate) override; + void OnIceCandidate(const webrtc::IceCandidateInterface* candidate) override; /// @brief Called when a remote peer opens a data channel. /// @param dataChannel The data channel. - void OnDataChannel(rtc::scoped_refptr dataChannel) override; + void OnDataChannel(rtc::scoped_refptr dataChannel) override; /// @brief Called when the peer connection state changes. /// @param newState The new peer connection state. - void OnConnectionChange(PeerConnectionInterface::PeerConnectionState newState) override; + void OnConnectionChange(webrtc::PeerConnectionInterface::PeerConnectionState newState) override; private: WDCConnection* _parent; @@ -97,7 +95,7 @@ class WDCPeerConnectionObserver : public PeerConnectionObserver { /// @brief A WebRTC data channel observer. -class WDCDataChannelObserver : public DataChannelObserver { +class WDCDataChannelObserver : public webrtc::DataChannelObserver { public: WDCDataChannelObserver(WDCConnection* parent); @@ -106,7 +104,7 @@ class WDCDataChannelObserver : public DataChannelObserver { /// @brief A data channel message was received. /// @param The message received. - void OnMessage(const DataBuffer& buffer) override; + void OnMessage(const webrtc::DataBuffer& buffer) override; private: WDCConnection* _parent; @@ -125,11 +123,11 @@ class WDCConnection { /// @brief Gets the WebSocket ID. /// @return The ID of the WebSocket. - quint16 getWebSocketID() { return _webSocketID; } + quint16 getWebSocketID() const { return _webSocketID; } /// @brief Gets the WebRTC data channel ID. /// @return The WebRTC data channel ID. `-1` if not open yet. - int getDataChannelID() { return _dataChannelID; } + int getDataChannelID() const { return _dataChannelID; } /// @brief Sets the remote session description received from the remote client via the signaling channel. @@ -141,11 +139,11 @@ class WDCConnection { /// @brief Sends an answer to the remote client via the signaling channel. /// @param description The answer. - void sendAnswer(SessionDescriptionInterface* description); + void sendAnswer(webrtc::SessionDescriptionInterface* description); /// @brief Sets the local session description on the WebRTC data channel being connected. /// @param description The local session description. - void setLocalDescription(SessionDescriptionInterface* description); + void setLocalDescription(webrtc::SessionDescriptionInterface* description); /// @brief Adds an ICE candidate received from the remote client via the signaling channel. /// @param data The ICE candidate. @@ -153,11 +151,11 @@ class WDCConnection { /// @brief Sends an ICE candidate to the remote vlient via the signaling channel. /// @param candidate The ICE candidate. - void sendIceCandidate(const IceCandidateInterface* candidate); + void sendIceCandidate(const webrtc::IceCandidateInterface* candidate); /// @brief Handles the WebRTC data channel being opened. /// @param dataChannel The WebRTC data channel. - void onDataChannelOpened(rtc::scoped_refptr dataChannel); + void onDataChannelOpened(rtc::scoped_refptr dataChannel); /// @brief Handles a change in the state of the WebRTC data channel. void onDataChannelStateChanged(); @@ -165,12 +163,17 @@ class WDCConnection { /// @brief Handles a message being received on the WebRTC data channel. /// @param buffer The message received. - void onDataChannelMessageReceived(const DataBuffer& buffer); + void onDataChannelMessageReceived(const webrtc::DataBuffer& buffer); + + /// @brief Gets the number of bytes waiting to be sent on the WebRTC data channel. + /// @return The number of bytes waiting to be sent on the WebRTC data channel. + qint64 getBufferedAmount() const; + /// @brief Sends a message on the WebRTC data channel. /// @param buffer The message to send. /// @return `true` if the message was sent, otherwise `false`. - bool sendDataMessage(const DataBuffer& buffer); + bool sendDataMessage(const webrtc::DataBuffer& buffer); private: WebRTCDataChannels* _parent; @@ -181,10 +184,10 @@ class WDCConnection { rtc::scoped_refptr _createSessionDescriptionObserver { nullptr }; std::shared_ptr _dataChannelObserver { nullptr }; - rtc::scoped_refptr _dataChannel { nullptr }; + rtc::scoped_refptr _dataChannel { nullptr }; std::shared_ptr _peerConnectionObserver { nullptr }; - rtc::scoped_refptr _peerConnection { nullptr }; + rtc::scoped_refptr _peerConnection { nullptr }; }; @@ -206,20 +209,25 @@ class WebRTCDataChannels : public QObject { public: /// @brief Constructs a new WebRTCDataChannels object. - /// @param nodeType The type of node that the WebRTCDataChannels object is being used in. /// @param parent The parent Qt object. - WebRTCDataChannels(NodeType_t nodeType, QObject* parent); + /// @param nodeType The type of node that the WebRTCDataChannels object is being used in. + WebRTCDataChannels(QObject* parent, NodeType_t nodeType); /// @brief Destroys a WebRTCDataChannels object. ~WebRTCDataChannels(); - /// @brief Returns the type of node that the WebRTCDataChannels object is being used in. + /// @brief Gets the type of node that the WebRTCDataChannels object is being used in. /// @return The type of node. NodeType_t getNodeType() { return _nodeType; } + /// @brief Immediately closes all connections and resets the socket. + void reset(); + /// @brief Get a new data channel ID to uniquely identify a WDCConnection. + /// @details This ID is assigned by WebRTCDataChannels; it is not the WebRTC data channel ID because that is only + /// unique within a peer connection. /// @return A new data channel ID. quint16 getNewDataChannelID(); @@ -248,10 +256,15 @@ class WebRTCDataChannels : public QObject { /// @return `true` if the data message was sent, otherwise `false`. bool sendDataMessage(int dataChannelID, const QByteArray& message); + /// @brief Gets the number of bytes waiting to be sent on a data channel. + /// @param dataChannelID The data channel ID. + /// @return The number of bytes waiting to be sent on the data channel. + qint64 getBufferedAmount(int dataChannelID) const; + /// @brief Creates a new WebRTC peer connection for connecting to an Interface client. /// @param peerConnectionObserver An observer to monitor the WebRTC peer connection. /// @return The new WebRTC peer connection. - rtc::scoped_refptr createPeerConnection( + rtc::scoped_refptr createPeerConnection( const std::shared_ptr peerConnectionObserver); public slots: @@ -283,9 +296,9 @@ public slots: std::unique_ptr _rtcWorkerThread { nullptr }; std::unique_ptr _rtcSignalingThread { nullptr }; - rtc::scoped_refptr _peerConnectionFactory { nullptr }; + rtc::scoped_refptr _peerConnectionFactory { nullptr }; - quint16 _lastDataChannelID { 0 }; + quint16 _lastDataChannelID { 0 }; // First data channel ID is 1. QHash _connectionsByWebSocket; QHash _connectionsByDataChannel; diff --git a/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp b/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp index 8c661bbf8cc..a11d09ea41e 100644 --- a/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp +++ b/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp @@ -19,33 +19,33 @@ const int WEBRTC_SOCKET_CHECK_INTERVAL_IN_MS = 30000; -WebRTCSignalingServer::WebRTCSignalingServer(const QHostAddress& address, quint16 port, QObject* parent) : +WebRTCSignalingServer::WebRTCSignalingServer(QObject* parent) : QObject(parent), - _address(address), - _port(port), _webSocketServer(new QWebSocketServer(QStringLiteral("WebRTC Signaling Server"), QWebSocketServer::NonSecureMode, this)) { connect(_webSocketServer, &QWebSocketServer::newConnection, this, &WebRTCSignalingServer::newWebSocketConnection); - bindSocket(); - // Automatically recover from network interruptions. _isWebSocketServerListeningTimer = new QTimer(this); connect(_isWebSocketServerListeningTimer, &QTimer::timeout, this, &WebRTCSignalingServer::checkWebSocketServerIsListening); _isWebSocketServerListeningTimer->start(WEBRTC_SOCKET_CHECK_INTERVAL_IN_MS); } +bool WebRTCSignalingServer::bind(const QHostAddress& address, quint16 port) { + _address = address; + _port = port; + auto success = _webSocketServer->listen(_address, _port); + if (!success) { + qCWarning(networking_webrtc) << "Failed to open WebSocket for WebRTC signaling."; + } + return success; +} + void WebRTCSignalingServer::checkWebSocketServerIsListening() { if (!_webSocketServer->isListening()) { qCWarning(networking_webrtc) << "WebSocket on port " << QString::number(_port) << " is no longer listening"; _webSockets.clear(); - bindSocket(); - } -} - -void WebRTCSignalingServer::bindSocket() { - if (!_webSocketServer->listen(_address, _port)) { - qCWarning(networking_webrtc) << "Failed to open WebSocket for WebRTC signaling."; + _webSocketServer->listen(_address, _port); } } diff --git a/libraries/networking/src/webrtc/WebRTCSignalingServer.h b/libraries/networking/src/webrtc/WebRTCSignalingServer.h index f2e8594b91f..e32133dd17c 100644 --- a/libraries/networking/src/webrtc/WebRTCSignalingServer.h +++ b/libraries/networking/src/webrtc/WebRTCSignalingServer.h @@ -59,10 +59,14 @@ class WebRTCSignalingServer : public QObject { public: /// @brief Constructs a new WebRTCSignalingServer object. - /// @param address The IP address to use for the WebSocket. - /// @param port The port to use for the WebSocket. /// @param parent Qt parent object. - WebRTCSignalingServer(const QHostAddress& address, quint16 port, QObject* parent = nullptr); + WebRTCSignalingServer(QObject* parent); + + /// @brief Binds the WebRTC signaling server's WebSocket to an address and port. + /// @param address The address to use for the WebSocket. + /// @param port The port to use for the WebSocket. + /// @return true if the WebSocket was successfully bound, false if it wasn't. + bool bind(const QHostAddress& address, quint16 port); public slots: @@ -88,11 +92,10 @@ private slots: private: void checkWebSocketServerIsListening(); - void bindSocket(); QWebSocketServer* _webSocketServer; QHostAddress _address; - const quint16 _port; + quint16 _port { 0 }; QHash _webSockets; // client WebSocket port, client WebSocket object From d65ecead9f2d7cc68e3f428948f751bd60ee0fd5 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sat, 26 Jun 2021 21:20:26 +1200 Subject: [PATCH 16/82] Abstract WebRTCDataChannel into a QUdpSocket-style WebRTCSocket --- domain-server/src/DomainServer.cpp | 20 --- domain-server/src/DomainServer.h | 12 -- libraries/networking/src/udt/Constants.h | 2 + .../networking/src/webrtc/WebRTCSocket.cpp | 158 +++++++++++++++++ .../networking/src/webrtc/WebRTCSocket.h | 162 ++++++++++++++++++ 5 files changed, 322 insertions(+), 32 deletions(-) create mode 100644 libraries/networking/src/webrtc/WebRTCSocket.cpp create mode 100644 libraries/networking/src/webrtc/WebRTCSocket.h diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 4f80e82681d..1a31e0869c0 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -165,10 +165,6 @@ bool DomainServer::forwardMetaverseAPIRequest(HTTPConnection* connection, DomainServer::DomainServer(int argc, char* argv[]) : QCoreApplication(argc, argv), _gatekeeper(this), -#ifdef WEBRTC_DATA_CHANNELS - _webrtcSignalingServer(QHostAddress::AnyIPv4, DEFAULT_DOMAIN_SERVER_WS_PORT, this), - _webrtcDataChannels(NodeType::DomainServer, this), -#endif _httpManager(QHostAddress::AnyIPv4, DOMAIN_SERVER_HTTP_PORT, QString("%1/resources/web/").arg(QCoreApplication::applicationDirPath()), this) { @@ -252,8 +248,6 @@ DomainServer::DomainServer(int argc, char* argv[]) : updateDownstreamNodes(); updateUpstreamNodes(); - setUpWebRTC(); - if (_type != NonMetaverse) { // if we have a metaverse domain, we'll use an access token for API calls resetAccountManagerAccessToken(); @@ -3139,20 +3133,6 @@ void DomainServer::updateUpstreamNodes() { updateReplicationNodes(Upstream); } -void DomainServer::setUpWebRTC() { -#ifdef WEBRTC_DATA_CHANNELS - - // Inbound WebRTC signaling messages received from a client. - connect(&_webrtcSignalingServer, &WebRTCSignalingServer::messageReceived, - &_webrtcDataChannels, &WebRTCDataChannels::onSignalingMessage); - - // Outbound WebRTC signaling messages being sent to a client. - connect(&_webrtcDataChannels, &WebRTCDataChannels::signalingMessage, - &_webrtcSignalingServer, &WebRTCSignalingServer::sendMessage); - -#endif -} - void DomainServer::initializeExporter() { static const QString ENABLE_EXPORTER = "monitoring.enable_prometheus_exporter"; static const QString EXPORTER_PORT = "monitoring.prometheus_exporter_port"; diff --git a/domain-server/src/DomainServer.h b/domain-server/src/DomainServer.h index eccf67d5b62..c6a928dd0c3 100644 --- a/domain-server/src/DomainServer.h +++ b/domain-server/src/DomainServer.h @@ -27,11 +27,6 @@ #include #include #include -#include -#if defined(WEBRTC_DATA_CHANNELS) -#include -#include -#endif #include "AssetsBackupHandler.h" #include "DomainGatekeeper.h" @@ -147,8 +142,6 @@ private slots: void updateDownstreamNodes(); void updateUpstreamNodes(); - void setUpWebRTC(); - void initializeExporter(); void initializeMetadataExporter(); @@ -319,11 +312,6 @@ private slots: std::unordered_map> _pendingContentFiles; QThread _assetClientThread; - -#ifdef WEBRTC_DATA_CHANNELS - WebRTCSignalingServer _webrtcSignalingServer; - WebRTCDataChannels _webrtcDataChannels; -#endif }; diff --git a/libraries/networking/src/udt/Constants.h b/libraries/networking/src/udt/Constants.h index 243fa4edda0..d6b208a0128 100644 --- a/libraries/networking/src/udt/Constants.h +++ b/libraries/networking/src/udt/Constants.h @@ -26,6 +26,8 @@ namespace udt { static const int CONNECTION_SEND_BUFFER_SIZE_PACKETS = 8192; static const int UDP_SEND_BUFFER_SIZE_BYTES = 1048576; static const int UDP_RECEIVE_BUFFER_SIZE_BYTES = 1048576; + static const int WEBRTC_SEND_BUFFER_SIZE_BYTES = 1048576; + static const int WEBRTC_RECEIVE_BUFFER_SIZE_BYTES = 1048576; static const int DEFAULT_SYN_INTERVAL_USECS = 10 * 1000; diff --git a/libraries/networking/src/webrtc/WebRTCSocket.cpp b/libraries/networking/src/webrtc/WebRTCSocket.cpp new file mode 100644 index 00000000000..5c6e3ce292b --- /dev/null +++ b/libraries/networking/src/webrtc/WebRTCSocket.cpp @@ -0,0 +1,158 @@ +// +// WebRTCSocket.cpp +// libraries/networking/src/webrtc +// +// Created by David Rowe on 21 Jun 2021. +// Copyright 2021 Vircadia contributors. +// + +#include "WebRTCSocket.h" + +#if defined(WEBRTC_DATA_CHANNELS) + +#include "../NetworkLogging.h" +#include "../udt/Constants.h" + + +WebRTCSocket::WebRTCSocket(QObject* parent, NodeType_t nodeType) : + QObject(parent), + _parent(parent), + _signalingServer(this /*, QHostAddress::AnyIPv4, DEFAULT_DOMAIN_SERVER_WS_PORT*/), + _dataChannels(this, nodeType) +{ + // Connect WebRTC signaling server and data channels. + connect(&_signalingServer, &WebRTCSignalingServer::messageReceived, + &_dataChannels, &WebRTCDataChannels::onSignalingMessage); + connect(&_dataChannels, &WebRTCDataChannels::signalingMessage, + &_signalingServer, &WebRTCSignalingServer::sendMessage); + + // Route received data channel messages. + connect(&_dataChannels, &WebRTCDataChannels::dataMessage, this, &WebRTCSocket::onDataChannelReceivedMessage); +} + +void WebRTCSocket::setSocketOption(QAbstractSocket::SocketOption option, const QVariant& value) { + clearError(); + switch (option) { + case QAbstractSocket::SocketOption::ReceiveBufferSizeSocketOption: + case QAbstractSocket::SocketOption::SendBufferSizeSocketOption: + // WebRTC doesn't provide access to setting these buffer sizes. + break; + default: + setError(QAbstractSocket::SocketError::UnsupportedSocketOperationError, "Failed to set socket option"); + qCCritical(networking_webrtc) << "WebRTCSocket::setSocketOption() not implemented for option:" << option; + } + +} + +QVariant WebRTCSocket::socketOption(QAbstractSocket::SocketOption option) { + clearError(); + switch (option) { + case QAbstractSocket::SocketOption::ReceiveBufferSizeSocketOption: + // WebRTC doesn't provide access to the receive buffer size. Just use the default buffer size. + return udt::WEBRTC_RECEIVE_BUFFER_SIZE_BYTES; + case QAbstractSocket::SocketOption::SendBufferSizeSocketOption: + // WebRTC doesn't provide access to the send buffer size though it's probably 16MB. Just use the default buffer size. + return udt::WEBRTC_SEND_BUFFER_SIZE_BYTES; + default: + setError(QAbstractSocket::SocketError::UnsupportedSocketOperationError, "Failed to get socket option"); + qCCritical(networking_webrtc) << "WebRTCSocket::getSocketOption() not implemented for option:" << option; + } + + return QVariant(); +} + +bool WebRTCSocket::bind(const QHostAddress& address, quint16 port, QAbstractSocket::BindMode mode) { + // WebRTC data channels aren't bound to ports so just treat this as a successful operation. + auto wasBound = _isBound; + _isBound = _signalingServer.bind(address, port); + if (_isBound != wasBound) { + emit stateChanged(_isBound ? QAbstractSocket::BoundState : QAbstractSocket::UnconnectedState); + } + return _isBound; +} + +QAbstractSocket::SocketState WebRTCSocket::state() const { + return _isBound ? QAbstractSocket::BoundState : QAbstractSocket::UnconnectedState; +} + +void WebRTCSocket::abort() { + _dataChannels.reset(); +} + + +qint64 WebRTCSocket::writeDatagram(const QByteArray& datagram, quint16 port) { + clearError(); + if (_dataChannels.sendDataMessage(port, datagram)) { + return datagram.length(); + } + setError(QAbstractSocket::SocketError::UnknownSocketError, "Failed to write datagram"); + return -1; +} + +qint64 WebRTCSocket::bytesToWrite(quint16 port) const { + return _dataChannels.getBufferedAmount(port); +} + + +bool WebRTCSocket::hasPendingDatagrams() const { + return _receivedQueue.length() > 0; +} + +qint64 WebRTCSocket::pendingDatagramSize() const { + if (_receivedQueue.length() > 0) { + return _receivedQueue.head().second.length(); + } + return -1; +} + +qint64 WebRTCSocket::readDatagram(char* data, qint64 maxSize, QHostAddress* address, quint16* port) { + clearError(); + if (_receivedQueue.length() > 0) { + auto datagram = _receivedQueue.dequeue(); + auto length = std::min((qint64)datagram.second.length(), maxSize); + + if (data) { + memcpy(data, datagram.second.constData(), length); + } + + if (address) { + // WEBRTC TODO: Use signaling channel's remote WebSocket address? Or remote data channel address? + *address = QHostAddress::AnyIPv4; + } + + if (port) { + *port = datagram.first; + } + + return length; + } + setError(QAbstractSocket::SocketError::UnknownSocketError, "Failed to read datagram"); + return -1; +} + + +QAbstractSocket::SocketError WebRTCSocket::error() const { + return _lastErrorType; +} + +QString WebRTCSocket::errorString() const { + return _lastErrorString; +} + + +void WebRTCSocket::setError(QAbstractSocket::SocketError errorType, QString errorString) { + _lastErrorType = errorType; +} + +void WebRTCSocket::clearError() { + _lastErrorType = QAbstractSocket::SocketError(); + _lastErrorString = QString(); +} + + +void WebRTCSocket::onDataChannelReceivedMessage(int dataChannelID, const QByteArray& message) { + _receivedQueue.enqueue(QPair(dataChannelID, message)); + emit readyRead(); +} + +#endif // WEBRTC_DATA_CHANNELS diff --git a/libraries/networking/src/webrtc/WebRTCSocket.h b/libraries/networking/src/webrtc/WebRTCSocket.h new file mode 100644 index 00000000000..ed33608859f --- /dev/null +++ b/libraries/networking/src/webrtc/WebRTCSocket.h @@ -0,0 +1,162 @@ +// +// WebRTCSocket.h +// libraries/networking/src/webrtc +// +// Created by David Rowe on 21 Jun 2021. +// Copyright 2021 Vircadia contributors. +// + +#ifndef vircadia_WebRTCSocket_h +#define vircadia_WebRTCSocket_h + +#include + +#if defined(WEBRTC_DATA_CHANNELS) + +#include +#include +#include + +#include "WebRTCDataChannels.h" +#include "WebRTCSignalingServer.h" + +/// @addtogroup Networking +/// @{ + + +/// @brief Provides a QUdpSocket-style interface for using WebRTCDataChannels. +class WebRTCSocket : public QObject { + Q_OBJECT + +public: + + /// @brief Constructs a new WebRTCSocket object. + /// @param parent Qt parent object. + /// @param nodeType The type of node that the WebRTCsocket object is being used in. + WebRTCSocket(QObject* parent, NodeType_t nodeType); + + + /// @brief Nominally sets the value of a socket option. + /// @details Only SendBufferSizeSocketOption and ReceiveBufferSizeSocketOption options are handled + /// and for these no action is taken because these buffer sizes are not configurable in WebRTC. + /// Included for compatibility with the QUdpSocket interface. + /// @param option The socket option. + /// @param value The value of the socket option. + void setSocketOption(QAbstractSocket::SocketOption option, const QVariant& value); + + /// @brief Nominally gets the value of a socket option. + /// @details Only SendBufferSizeSocketOption and ReceiveBufferSizeSocketOption options are handled + /// and for these only default values are returned because these buffer sizes are not configurable in WebRTC. + /// Included for compatibility with the QUdpSocket interface. + /// @param option The socket option. + /// @return The value of the socket option. + QVariant socketOption(QAbstractSocket::SocketOption option); + + /// @brief Binds the WebRTC socket's signaling server to an address and port. + /// @details Note: WebRTC data connections aren't bound to an address or port. Their ports are negotiated as part of the + /// WebRTC peer connection process. + /// @param address The address to use for the signaling server. + /// @param port The port to use for the signaling server. + /// @param mode The bind mode. (Not used: included for compatibility with the QUdpSocket interface.) + /// @return true if the signaling server was successfully bound, false if it wasn't. + bool bind(const QHostAddress& address, quint16 port = 0, QAbstractSocket::BindMode mode + = QAbstractSocket::DefaultForPlatform); + + /// @brief Gets the state of the socket. + /// @details In particular, QAbstractSocket::BoundState is returned if the socket is bound, + /// QAbstractSocket::UnconnectedState if it isn't. + /// @return The state of the socket. + QAbstractSocket::SocketState state() const; + + /// @brief Immediately closes all connections and resets the socket. + void abort(); + + /// @brief Nominally gets the host port number. + /// @details + /// Included for compatibility with the QUdpSocket interface. + /// @return 0 + quint16 localPort() const { return 0; } + + /// @brief Nominally gets the socket descriptor. + /// @details + /// Included for compatibility with the QUdpSocket interface. + /// @return -1 + qintptr socketDescriptor() const { return -1; } + + + /// @brief Sends a datagram to the host on a data channel. + /// @param datagram The datagram to send. + /// @param port The data channel ID. + /// @return The number of bytes if successfully sent, otherwise -1. + qint64 writeDatagram(const QByteArray& datagram, quint16 port); + + /// @brief Gets the number of bytes waiting to be written. + /// @param port The data channel ID. + /// @return The number of bytes waiting to be written. + qint64 bytesToWrite(quint16 port) const; + + /// @brief Gets whether there's a datagram waiting to be read. + /// @return true if there's a datagram waiting to be read, false if there isn't. + bool hasPendingDatagrams() const; + + /// @brief Gets the size of the first pending datagram. + /// @return the size of the first pending datagram; -1 if there is no pending datagram. + qint64 pendingDatagramSize() const; + + /// @brief Reads the next datagram, up to a maximum number of bytes. + /// @details Any remaining data in the datagram is lost. + /// @param data The destination to read the datagram into. + /// @param maxSize The maximum number of bytes to read. + /// @param address The destination to put the IP address that the datagram was read from. (Not currently set.) + /// @param port The destination to put the data channel ID that the datagram was read from. + /// @return The number of bytes read on success; -1 if reading unsuccessful. + qint64 readDatagram(char* data, qint64 maxSize, QHostAddress* address = nullptr, quint16* port = nullptr); + + + /// @brief Gets the type of error that last occurred. + /// @return The type of error that last occurred. + QAbstractSocket::SocketError error() const; + + /// @brief Gets the description of the error that last occurred. + /// @return The description of the error that last occurred. + QString errorString() const; + +public slots: + + /// @brief Handles the WebRTC data channel receiving a message. + /// @param dataChannelID The data channel that the message was received on. + /// @param message The message that was received. + /// @detail Queues the message to be read via readDatagram. + void onDataChannelReceivedMessage(int dataChannelID, const QByteArray& message); + +signals: + + /// @brief Emitted when the state of the socket changes. + void stateChanged(QAbstractSocket::SocketState socketState); + + /// @brief Emitted each time new data becomes available for reading. + void readyRead(); + +private: + + void setError(QAbstractSocket::SocketError errorType, QString errorString); + void clearError(); + + QObject* _parent; + WebRTCSignalingServer _signalingServer; + WebRTCDataChannels _dataChannels; + + bool _isBound { false }; + + QQueue> _receivedQueue; // Messages received are queued for reading from the "socket". + + QAbstractSocket::SocketError _lastErrorType { QAbstractSocket::UnknownSocketError }; + QString _lastErrorString; +}; + + +/// @} + +#endif // WEBRTC_DATA_CHANNELS + +#endif // vircadia_WebRTCSocket_h From 5a5cb6488cfec51c33c0388e2e6eb904cc9d8fec Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sat, 26 Jun 2021 22:07:01 +1200 Subject: [PATCH 17/82] Multiplex UDP and WebRTC sockets in a QUdpSocket-style NetworkSocket --- assignment-client/src/AssignmentClient.cpp | 5 +- .../src/AssignmentClientMonitor.cpp | 2 +- domain-server/src/DomainServer.cpp | 8 +- ice-server/src/IceServer.cpp | 2 +- interface/src/ui/PreferencesDialog.cpp | 4 +- libraries/networking/src/DomainHandler.cpp | 5 +- libraries/networking/src/HifiSockAddr.cpp | 30 +- libraries/networking/src/HifiSockAddr.h | 13 +- libraries/networking/src/LimitedNodeList.cpp | 40 +-- libraries/networking/src/LimitedNodeList.h | 9 +- libraries/networking/src/NodeList.cpp | 6 +- libraries/networking/src/NodeList.h | 4 +- libraries/networking/src/SocketType.h | 39 +++ .../networking/src/udt/NetworkSocket.cpp | 281 ++++++++++++++++++ libraries/networking/src/udt/NetworkSocket.h | 163 ++++++++++ libraries/networking/src/udt/Socket.cpp | 90 +++--- libraries/networking/src/udt/Socket.h | 20 +- tools/ice-client/src/ICEClientApp.cpp | 18 +- 18 files changed, 633 insertions(+), 106 deletions(-) create mode 100644 libraries/networking/src/SocketType.h create mode 100644 libraries/networking/src/udt/NetworkSocket.cpp create mode 100644 libraries/networking/src/udt/NetworkSocket.h diff --git a/assignment-client/src/AssignmentClient.cpp b/assignment-client/src/AssignmentClient.cpp index adc7f5e3c5a..196e9a20b76 100644 --- a/assignment-client/src/AssignmentClient.cpp +++ b/assignment-client/src/AssignmentClient.cpp @@ -82,7 +82,7 @@ AssignmentClient::AssignmentClient(Assignment::Type requestAssignmentType, QStri _assignmentServerHostname = assignmentServerHostname; } - _assignmentServerSocket = HifiSockAddr(_assignmentServerHostname, assignmentServerPort, true); + _assignmentServerSocket = HifiSockAddr(SocketType::UDP, _assignmentServerHostname, assignmentServerPort, true); if (_assignmentServerSocket.isNull()) { qCCritical(assignment_client) << "PAGE: Couldn't resolve domain server address" << _assignmentServerHostname; } @@ -110,7 +110,8 @@ AssignmentClient::AssignmentClient(Assignment::Type requestAssignmentType, QStri // did we get an assignment-client monitor port? if (assignmentMonitorPort > 0) { - _assignmentClientMonitorSocket = HifiSockAddr(DEFAULT_ASSIGNMENT_CLIENT_MONITOR_HOSTNAME, assignmentMonitorPort); + _assignmentClientMonitorSocket = HifiSockAddr(SocketType::UDP, DEFAULT_ASSIGNMENT_CLIENT_MONITOR_HOSTNAME, + assignmentMonitorPort); _assignmentClientMonitorSocket.setObjectName("AssignmentClientMonitor"); qCDebug(assignment_client) << "Assignment-client monitor socket is" << _assignmentClientMonitorSocket; diff --git a/assignment-client/src/AssignmentClientMonitor.cpp b/assignment-client/src/AssignmentClientMonitor.cpp index 68c0dfc9fdd..a085d1d3f49 100644 --- a/assignment-client/src/AssignmentClientMonitor.cpp +++ b/assignment-client/src/AssignmentClientMonitor.cpp @@ -69,7 +69,7 @@ AssignmentClientMonitor::AssignmentClientMonitor(const unsigned int numAssignmen // create a NodeList so we can receive stats from children DependencyManager::registerInheritance(); auto addressManager = DependencyManager::set(); - auto nodeList = DependencyManager::set(listenPort); + auto nodeList = DependencyManager::set(NodeType::Unassigned, listenPort); auto& packetReceiver = DependencyManager::get()->getPacketReceiver(); packetReceiver.registerListener(PacketType::AssignmentClientStatus, diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 1a31e0869c0..dca17df3402 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -731,10 +731,11 @@ void DomainServer::setupNodeListAndAssignments() { // check for scripts the user wants to persist from their domain-server config populateStaticScriptedAssignmentsFromSettings(); - auto nodeList = DependencyManager::set(domainServerPort, domainServerDTLSPort); + auto nodeList = DependencyManager::set(NodeType::DomainServer, domainServerPort, domainServerDTLSPort); // no matter the local port, save it to shared mem so that local assignment clients can ask what it is - nodeList->putLocalPortIntoSharedMemory(DOMAIN_SERVER_LOCAL_PORT_SMEM_KEY, this, nodeList->getSocketLocalPort()); + nodeList->putLocalPortIntoSharedMemory(DOMAIN_SERVER_LOCAL_PORT_SMEM_KEY, this, + nodeList->getSocketLocalPort(SocketType::UDP)); // store our local http ports in shared memory quint16 localHttpPort = DOMAIN_SERVER_HTTP_PORT; @@ -3041,6 +3042,7 @@ ReplicationServerInfo serverInformationFromSettings(QVariantMap serverMap, Repli // read the address and port and construct a HifiSockAddr from them serverInfo.sockAddr = { + SocketType::UDP, serverMap[REPLICATION_SERVER_ADDRESS].toString(), (quint16) serverMap[REPLICATION_SERVER_PORT].toString().toInt() }; @@ -3621,7 +3623,7 @@ void DomainServer::randomizeICEServerAddress(bool shouldTriggerHostLookup) { indexToTry = distribution(generator); } - _iceServerSocket = HifiSockAddr { candidateICEAddresses[indexToTry], ICE_SERVER_DEFAULT_PORT }; + _iceServerSocket = HifiSockAddr { SocketType::UDP, candidateICEAddresses[indexToTry], ICE_SERVER_DEFAULT_PORT }; qCInfo(domain_server_ice) << "Set candidate ice-server socket to" << _iceServerSocket; // clear our number of hearbeat denials, this should be re-set on ice-server change diff --git a/ice-server/src/IceServer.cpp b/ice-server/src/IceServer.cpp index a8e50d31a96..373d33013db 100644 --- a/ice-server/src/IceServer.cpp +++ b/ice-server/src/IceServer.cpp @@ -37,7 +37,7 @@ IceServer::IceServer(int argc, char* argv[]) : { // start the ice-server socket qDebug() << "ice-server socket is listening on" << ICE_SERVER_DEFAULT_PORT; - _serverSocket.bind(QHostAddress::AnyIPv4, ICE_SERVER_DEFAULT_PORT); + _serverSocket.bind(SocketType::UDP, QHostAddress::AnyIPv4, ICE_SERVER_DEFAULT_PORT); // set processPacket as the verified packet callback for the udt::Socket _serverSocket.setPacketHandler([this](std::unique_ptr packet) { processPacket(std::move(packet)); }); diff --git a/interface/src/ui/PreferencesDialog.cpp b/interface/src/ui/PreferencesDialog.cpp index c2f1abdd94f..9c1c4fa7847 100644 --- a/interface/src/ui/PreferencesDialog.cpp +++ b/interface/src/ui/PreferencesDialog.cpp @@ -551,7 +551,7 @@ void setupPreferences() { auto getter = [nodeListWeak] { auto nodeList = nodeListWeak.lock(); if (nodeList) { - return static_cast(nodeList->getSocketLocalPort()); + return static_cast(nodeList->getSocketLocalPort(SocketType::UDP)); } else { return -1; } @@ -559,7 +559,7 @@ void setupPreferences() { auto setter = [nodeListWeak](int preset) { auto nodeList = nodeListWeak.lock(); if (nodeList) { - nodeList->setSocketLocalPort(static_cast(preset)); + nodeList->setSocketLocalPort(SocketType::UDP, static_cast(preset)); } }; auto preference = new IntSpinnerPreference(NETWORKING, "Listening Port", getter, setter); diff --git a/libraries/networking/src/DomainHandler.cpp b/libraries/networking/src/DomainHandler.cpp index 5a1d8fb4a09..a77cde4ecc0 100644 --- a/libraries/networking/src/DomainHandler.cpp +++ b/libraries/networking/src/DomainHandler.cpp @@ -37,7 +37,7 @@ DomainHandler::DomainHandler(QObject* parent) : QObject(parent), - _sockAddr(HifiSockAddr(QHostAddress::Null, DEFAULT_DOMAIN_SERVER_PORT)), + _sockAddr(HifiSockAddr(SocketType::UDP, QHostAddress::Null, DEFAULT_DOMAIN_SERVER_PORT)), _icePeer(this), _settingsTimer(this), _apiRefreshTimer(this) @@ -282,7 +282,8 @@ void DomainHandler::setIceServerHostnameAndID(const QString& iceServerHostname, HifiSockAddr* replaceableSockAddr = &_iceServerSockAddr; replaceableSockAddr->~HifiSockAddr(); - replaceableSockAddr = new (replaceableSockAddr) HifiSockAddr(iceServerHostname, ICE_SERVER_DEFAULT_PORT); + replaceableSockAddr = new (replaceableSockAddr) HifiSockAddr(SocketType::UDP, iceServerHostname, + ICE_SERVER_DEFAULT_PORT); _iceServerSockAddr.setObjectName("IceServer"); auto nodeList = DependencyManager::get(); diff --git a/libraries/networking/src/HifiSockAddr.cpp b/libraries/networking/src/HifiSockAddr.cpp index 434f8daa3da..091ccd6ed6a 100644 --- a/libraries/networking/src/HifiSockAddr.cpp +++ b/libraries/networking/src/HifiSockAddr.cpp @@ -27,21 +27,22 @@ int hifiSockAddrMetaTypeId = qRegisterMetaType(); HifiSockAddr::HifiSockAddr() : + _socketType(SocketType::Unknown), _address(), _port(0) { - } -HifiSockAddr::HifiSockAddr(const QHostAddress& address, quint16 port) : +HifiSockAddr::HifiSockAddr(SocketType socketType, const QHostAddress& address, quint16 port) : + _socketType(socketType), _address(address), _port(port) { - } HifiSockAddr::HifiSockAddr(const HifiSockAddr& otherSockAddr) : QObject(), + _socketType(otherSockAddr._socketType), _address(otherSockAddr._address), _port(otherSockAddr._port) { @@ -50,12 +51,14 @@ HifiSockAddr::HifiSockAddr(const HifiSockAddr& otherSockAddr) : HifiSockAddr& HifiSockAddr::operator=(const HifiSockAddr& rhsSockAddr) { setObjectName(rhsSockAddr.objectName()); + _socketType = rhsSockAddr._socketType; _address = rhsSockAddr._address; _port = rhsSockAddr._port; return *this; } -HifiSockAddr::HifiSockAddr(const QString& hostname, quint16 hostOrderPort, bool shouldBlockForLookup) : +HifiSockAddr::HifiSockAddr(SocketType socketType, const QString& hostname, quint16 hostOrderPort, bool shouldBlockForLookup) : + _socketType(socketType), _address(hostname), _port(hostOrderPort) { @@ -75,7 +78,8 @@ HifiSockAddr::HifiSockAddr(const QString& hostname, quint16 hostOrderPort, bool void HifiSockAddr::swap(HifiSockAddr& otherSockAddr) { using std::swap; - + + swap(_socketType, otherSockAddr._socketType); swap(_address, otherSockAddr._address); swap(_port, otherSockAddr._port); @@ -86,7 +90,7 @@ void HifiSockAddr::swap(HifiSockAddr& otherSockAddr) { } bool HifiSockAddr::operator==(const HifiSockAddr& rhsSockAddr) const { - return _address == rhsSockAddr._address && _port == rhsSockAddr._port; + return _socketType == rhsSockAddr._socketType && _address == rhsSockAddr._address && _port == rhsSockAddr._port; } void HifiSockAddr::handleLookupResult(const QHostInfo& hostInfo) { @@ -108,7 +112,7 @@ void HifiSockAddr::handleLookupResult(const QHostInfo& hostInfo) { } QString HifiSockAddr::toString() const { - return _address.toString() + ":" + QString::number(_port); + return socketTypeToString(_socketType) + " " + _address.toString() + ":" + QString::number(_port); } bool HifiSockAddr::hasPrivateAddress() const { @@ -124,17 +128,27 @@ bool HifiSockAddr::hasPrivateAddress() const { } QDebug operator<<(QDebug debug, const HifiSockAddr& sockAddr) { - debug.nospace() << sockAddr._address.toString().toLocal8Bit().constData() << ":" << sockAddr._port; + debug.nospace() << socketTypeToString(sockAddr._socketType).toLocal8Bit().constData() << " " + << sockAddr._address.toString().toLocal8Bit().constData() << ":" << sockAddr._port; return debug.space(); } QDataStream& operator<<(QDataStream& dataStream, const HifiSockAddr& sockAddr) { + // Don't include socketType because it can be implied from the type of connection used. + // WEBRTC TODO: Reconsider this. dataStream << sockAddr._address << sockAddr._port; return dataStream; } QDataStream& operator>>(QDataStream& dataStream, HifiSockAddr& sockAddr) { + // Don't include socketType because it can be implied from the type of connection used. + // WEBRTC TODO: Reconsider this. dataStream >> sockAddr._address >> sockAddr._port; + + // Set default for non-WebRTC code. + // WEBRTC TODO: Reconsider this. + sockAddr.setSocketType(SocketType::UDP); + return dataStream; } diff --git a/libraries/networking/src/HifiSockAddr.h b/libraries/networking/src/HifiSockAddr.h index fc9fdd15a58..e4c5c0c6cd8 100644 --- a/libraries/networking/src/HifiSockAddr.h +++ b/libraries/networking/src/HifiSockAddr.h @@ -19,13 +19,16 @@ struct sockaddr; #include +#include "SocketType.h" + + class HifiSockAddr : public QObject { Q_OBJECT public: HifiSockAddr(); - HifiSockAddr(const QHostAddress& address, quint16 port); + HifiSockAddr(SocketType socketType, const QHostAddress& address, quint16 port); HifiSockAddr(const HifiSockAddr& otherSockAddr); - HifiSockAddr(const QString& hostname, quint16 hostOrderPort, bool shouldBlockForLookup = false); + HifiSockAddr(SocketType socketType, const QString& hostname, quint16 hostOrderPort, bool shouldBlockForLookup = false); bool isNull() const { return _address.isNull() && _port == 0; } void clear() { _address.clear(); _port = 0;} @@ -36,6 +39,10 @@ class HifiSockAddr : public QObject { bool operator==(const HifiSockAddr& rhsSockAddr) const; bool operator!=(const HifiSockAddr& rhsSockAddr) const { return !(*this == rhsSockAddr); } + SocketType getSocketType() const { return _socketType; } + SocketType* getSocketTypePointer() { return &_socketType; } + void setSocketType(const SocketType socketType) { _socketType = socketType; } + const QHostAddress& getAddress() const { return _address; } QHostAddress* getAddressPointer() { return &_address; } void setAddress(const QHostAddress& address) { _address = address; } @@ -52,6 +59,7 @@ class HifiSockAddr : public QObject { bool hasPrivateAddress() const; // checks if the address behind this sock addr is private per RFC 1918 friend QDebug operator<<(QDebug debug, const HifiSockAddr& sockAddr); + friend QDataStream& operator<<(QDataStream& dataStream, const HifiSockAddr& sockAddr); friend QDataStream& operator>>(QDataStream& dataStream, HifiSockAddr& sockAddr); @@ -61,6 +69,7 @@ private slots: void lookupCompleted(); void lookupFailed(); private: + SocketType _socketType { SocketType::Unknown }; QHostAddress _address; quint16 _port; }; diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index 653611ae8c2..275324b865b 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -49,18 +49,18 @@ static Setting::Handle LIMITED_NODELIST_LOCAL_PORT("LimitedNodeList.Loc using namespace std::chrono_literals; static const std::chrono::milliseconds CONNECTION_RATE_INTERVAL_MS = 1s; -LimitedNodeList::LimitedNodeList(int socketListenPort, int dtlsListenPort) : - _nodeSocket(this), +LimitedNodeList::LimitedNodeList(char ownerType, int socketListenPort, int dtlsListenPort) : + _nodeSocket(this, true, ownerType), _packetReceiver(new PacketReceiver(this)) { qRegisterMetaType("ConnectionStep"); auto port = (socketListenPort != INVALID_PORT) ? socketListenPort : LIMITED_NODELIST_LOCAL_PORT.get(); - _nodeSocket.bind(QHostAddress::AnyIPv4, port); - quint16 assignedPort = _nodeSocket.localPort(); + _nodeSocket.bind(SocketType::UDP, QHostAddress::AnyIPv4, port); + quint16 assignedPort = _nodeSocket.localPort(SocketType::UDP); if (socketListenPort != INVALID_PORT && socketListenPort != 0 && socketListenPort != assignedPort) { - qCCritical(networking) << "PAGE: NodeList is unable to assign requested port of" << socketListenPort; + qCCritical(networking) << "PAGE: NodeList is unable to assign requested UDP port of" << socketListenPort; } - qCDebug(networking) << "NodeList socket is listening on" << assignedPort; + qCDebug(networking) << "NodeList UDP socket is listening on" << assignedPort; if (dtlsListenPort != INVALID_PORT) { // only create the DTLS socket during constructor if a custom port is passed @@ -73,6 +73,8 @@ LimitedNodeList::LimitedNodeList(int socketListenPort, int dtlsListenPort) : qCDebug(networking) << "NodeList DTLS socket is listening on" << _dtlsSocket->localPort(); } + _nodeSocket.bind(SocketType::WebRTC, QHostAddress::AnyIPv4, DEFAULT_DOMAIN_SERVER_WS_PORT); + // check for local socket updates every so often const int LOCAL_SOCKET_UPDATE_INTERVAL_MSECS = 5 * 1000; QTimer* localSocketUpdate = new QTimer(this); @@ -204,15 +206,19 @@ void LimitedNodeList::setPermissions(const NodePermissions& newPermissions) { } } -void LimitedNodeList::setSocketLocalPort(quint16 socketLocalPort) { +void LimitedNodeList::setSocketLocalPort(SocketType socketType, quint16 socketLocalPort) { if (QThread::currentThread() != thread()) { QMetaObject::invokeMethod(this, "setSocketLocalPort", Qt::QueuedConnection, Q_ARG(quint16, socketLocalPort)); return; } - if (_nodeSocket.localPort() != socketLocalPort) { - _nodeSocket.rebind(socketLocalPort); - LIMITED_NODELIST_LOCAL_PORT.set(socketLocalPort); + if (_nodeSocket.localPort(socketType) != socketLocalPort) { + _nodeSocket.rebind(socketType, socketLocalPort); + if (socketType == SocketType::UDP) { + LIMITED_NODELIST_LOCAL_PORT.set(socketLocalPort); + } else { + // WEBRTC TODO: Add WebRTC equivalent? + } } } @@ -1105,7 +1111,7 @@ void LimitedNodeList::processSTUNResponse(std::unique_ptr packe _publicSockAddr.getAddress().toString().toLocal8Bit().constData(), _publicSockAddr.getPort()); - _publicSockAddr = HifiSockAddr(newPublicAddress, newPublicPort); + _publicSockAddr = HifiSockAddr(SocketType::UDP, newPublicAddress, newPublicPort); if (!_hasCompletedInitialSTUN) { // if we're here we have definitely completed our initial STUN sequence @@ -1186,7 +1192,7 @@ void LimitedNodeList::stopInitialSTUNUpdate(bool success) { qCDebug(networking) << "LimitedNodeList public socket will be set with local port and null QHostAddress."; // reset the public address and port to a null address - _publicSockAddr = HifiSockAddr(QHostAddress(), _nodeSocket.localPort()); + _publicSockAddr = HifiSockAddr(SocketType::UDP, QHostAddress(), _nodeSocket.localPort(SocketType::UDP)); // we have changed the publicSockAddr, so emit our signal emit publicSockAddrChanged(_publicSockAddr); @@ -1213,7 +1219,7 @@ void LimitedNodeList::stopInitialSTUNUpdate(bool success) { void LimitedNodeList::updateLocalSocket() { // when update is called, if the local socket is empty then start with the guessed local socket if (_localSockAddr.isNull()) { - setLocalSocket(HifiSockAddr { getGuessedLocalAddress(), _nodeSocket.localPort() }); + setLocalSocket(HifiSockAddr { SocketType::UDP, getGuessedLocalAddress(), _nodeSocket.localPort(SocketType::UDP) }); } // attempt to use Google's DNS to confirm that local IP @@ -1237,7 +1243,7 @@ void LimitedNodeList::connectedForLocalSocketTest() { auto localHostAddress = localIPTestSocket->localAddress(); if (localHostAddress.protocol() == QAbstractSocket::IPv4Protocol) { - setLocalSocket(HifiSockAddr { localHostAddress, _nodeSocket.localPort() }); + setLocalSocket(HifiSockAddr { SocketType::UDP, localHostAddress, _nodeSocket.localPort(SocketType::UDP) }); _hasTCPCheckedLocalSocket = true; } @@ -1253,7 +1259,7 @@ void LimitedNodeList::errorTestingLocalSocket() { // error connecting to the test socket - if we've never set our local socket using this test socket // then use our possibly updated guessed local address as fallback if (!_hasTCPCheckedLocalSocket) { - setLocalSocket(HifiSockAddr { getGuessedLocalAddress(), _nodeSocket.localPort() }); + setLocalSocket(HifiSockAddr { SocketType::UDP, getGuessedLocalAddress(), _nodeSocket.localPort(SocketType::UDP) }); qCCritical(networking) << "PAGE: Can't connect to Google DNS service via TCP, falling back to guessed local address" << getLocalSockAddr(); } @@ -1273,8 +1279,8 @@ void LimitedNodeList::setLocalSocket(const HifiSockAddr& sockAddr) { _localSockAddr = sockAddr; if (_hasTCPCheckedLocalSocket) { // Force a port change for NAT: reset("local socket change"); - _nodeSocket.rebind(0); - _localSockAddr.setPort(_nodeSocket.localPort()); + _nodeSocket.rebind(SocketType::UDP, 0); + _localSockAddr.setPort(_nodeSocket.localPort(SocketType::UDP)); qCInfo(networking) << "Local port changed to" << _localSockAddr.getPort(); } } diff --git a/libraries/networking/src/LimitedNodeList.h b/libraries/networking/src/LimitedNodeList.h index 189f3e1b087..670d843dd3f 100644 --- a/libraries/networking/src/LimitedNodeList.h +++ b/libraries/networking/src/LimitedNodeList.h @@ -135,8 +135,8 @@ class LimitedNodeList : public QObject, public Dependency { bool getThisNodeCanGetAndSetPrivateUserData() const { return _permissions.can(NodePermissions::Permission::canGetAndSetPrivateUserData); } bool getThisNodeCanRezAvatarEntities() const { return _permissions.can(NodePermissions::Permission::canRezAvatarEntities); } - quint16 getSocketLocalPort() const { return _nodeSocket.localPort(); } - Q_INVOKABLE void setSocketLocalPort(quint16 socketLocalPort); + quint16 getSocketLocalPort(SocketType socketType) const { return _nodeSocket.localPort(socketType); } + Q_INVOKABLE void setSocketLocalPort(SocketType socketType, quint16 socketLocalPort); QUdpSocket& getDTLSSocket(); @@ -413,7 +413,8 @@ protected slots: QUuid connectionSecretUUID; }; - LimitedNodeList(int socketListenPort = INVALID_PORT, int dtlsListenPort = INVALID_PORT); + LimitedNodeList(char ownerType = NodeType::DomainServer, int socketListenPort = INVALID_PORT, + int dtlsListenPort = INVALID_PORT); LimitedNodeList(LimitedNodeList const&) = delete; // Don't implement, needed to avoid copies of singleton void operator=(LimitedNodeList const&) = delete; // Don't implement, needed to avoid copies of singleton @@ -446,7 +447,7 @@ protected slots: QUdpSocket* _dtlsSocket { nullptr }; HifiSockAddr _localSockAddr; HifiSockAddr _publicSockAddr; - HifiSockAddr _stunSockAddr { STUN_SERVER_HOSTNAME, STUN_SERVER_PORT }; + HifiSockAddr _stunSockAddr { SocketType::UDP, STUN_SERVER_HOSTNAME, STUN_SERVER_PORT }; bool _hasTCPCheckedLocalSocket { false }; bool _useAuthentication { true }; diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index 90027460617..a6600991999 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -50,7 +50,7 @@ const int KEEPALIVE_PING_INTERVAL_MS = 1000; const int MAX_SYSTEM_INFO_SIZE = 1000; NodeList::NodeList(char newOwnerType, int socketListenPort, int dtlsListenPort) : - LimitedNodeList(socketListenPort, dtlsListenPort), + LimitedNodeList(newOwnerType, socketListenPort, dtlsListenPort), _ownerType(newOwnerType), _nodeTypesOfInterest(), _domainHandler(this), @@ -890,6 +890,10 @@ void NodeList::parseNodeFromPacketStream(QDataStream& packetStream) { info.publicSocket.setAddress(_domainHandler.getIP()); } + // WEBRTC TODO: Handle WebRTC-connected nodes. Probably need to include SocketType in HifiSockAddr << and >> + info.publicSocket.setSocketType(SocketType::UDP); + info.localSocket.setSocketType(SocketType::UDP); + addNewNode(info); } diff --git a/libraries/networking/src/NodeList.h b/libraries/networking/src/NodeList.h index 59b3815fbac..2ffac59702d 100644 --- a/libraries/networking/src/NodeList.h +++ b/libraries/networking/src/NodeList.h @@ -153,7 +153,9 @@ private slots: void maybeSendIgnoreSetToNode(SharedNodePointer node); private: - NodeList() : LimitedNodeList(INVALID_PORT, INVALID_PORT) { assert(false); } // Not implemented, needed for DependencyManager templates compile + NodeList() : LimitedNodeList(NodeType::Unassigned, INVALID_PORT, INVALID_PORT) { + assert(false); // Not implemented, needed for DependencyManager templates compile + } NodeList(char ownerType, int socketListenPort = INVALID_PORT, int dtlsListenPort = INVALID_PORT); NodeList(NodeList const&) = delete; // Don't implement, needed to avoid copies of singleton void operator=(NodeList const&) = delete; // Don't implement, needed to avoid copies of singleton diff --git a/libraries/networking/src/SocketType.h b/libraries/networking/src/SocketType.h new file mode 100644 index 00000000000..c63ee0cf348 --- /dev/null +++ b/libraries/networking/src/SocketType.h @@ -0,0 +1,39 @@ +// +// SocketType.h +// libraries/networking/src +// +// Created by David Rowe on 17 May 2021. +// Copyright 2021 Vircadia contributors. +// +// Handles UDP and WebRTC sockets in parallel. +// +// Distributed under the Apache License, Version 2.0. +// See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html +// + +#ifndef vircadia_SocketType_h +#define vircadia_SocketType_h + +/// @addtogroup Networking +/// @{ + + +/// @brief The network socket type. +enum class SocketType { + Unknown, ///< Unknown socket type. + UDP, ///< UDP socket. + WebRTC ///< WebRTC socket. +}; + +/// @brief Returns the name of a SocketType value as a string. +/// @param socketType The SocketType value. +/// @return The name of the SocketType value as a string. +static QString socketTypeToString(SocketType socketType) { + static QStringList SOCKET_TYPE_STRINGS { "Unknown", "UDP", "WebRTC" }; + return SOCKET_TYPE_STRINGS[(int)socketType]; +} + + +/// @} + +#endif // vircadia_SocketType_h diff --git a/libraries/networking/src/udt/NetworkSocket.cpp b/libraries/networking/src/udt/NetworkSocket.cpp new file mode 100644 index 00000000000..3fb039187ec --- /dev/null +++ b/libraries/networking/src/udt/NetworkSocket.cpp @@ -0,0 +1,281 @@ +// +// NetworkSocket.cpp +// libraries/networking/src/udt +// +// Created by David Rowe on 21 Jun 2021. +// Copyright 2021 Vircadia contributors. +// + +#include "NetworkSocket.h" + +#include "../NetworkLogging.h" + + +NetworkSocket::NetworkSocket(QObject* parent, NodeType_t nodeType) : + QObject(parent), + _parent(parent), + _udpSocket(this) +#if defined(WEBRTC_DATA_CHANNELS) + , + _webrtcSocket(this, nodeType) +#endif +{ + connect(&_udpSocket, &QUdpSocket::readyRead, this, &NetworkSocket::readyRead); + connect(&_udpSocket, &QAbstractSocket::stateChanged, this, &NetworkSocket::onUDPStateChanged); + connect(&_udpSocket, &QAbstractSocket::errorOccurred, this, &NetworkSocket::onUDPSocketError); +#if defined(WEBRTC_DATA_CHANNELS) + connect(&_webrtcSocket, &WebRTCSocket::readyRead, this, &NetworkSocket::readyRead); + connect(&_webrtcSocket, &WebRTCSocket::stateChanged, this, &NetworkSocket::onWebRTCStateChanged); + // WEBRTC TODO: Add similar for errorOccurred +#endif +} + + +void NetworkSocket::setSocketOption(SocketType socketType, QAbstractSocket::SocketOption option, const QVariant& value) { + switch (socketType) { + case SocketType::UDP: + _udpSocket.setSocketOption(option, value); + break; +#if defined(WEBRTC_DATA_CHANNELS) + case SocketType::WebRTC: + _webrtcSocket.setSocketOption(option, value); + break; +#endif + default: + qCCritical(networking) << "Socket type not specified in setSocketOption()"; + } +} + +QVariant NetworkSocket::socketOption(SocketType socketType, QAbstractSocket::SocketOption option) { + switch (socketType) { + case SocketType::UDP: + return _udpSocket.socketOption(option); +#if defined(WEBRTC_DATA_CHANNELS) + case SocketType::WebRTC: + return _webrtcSocket.socketOption(option); +#endif + default: + qCCritical(networking) << "Socket type not specified in socketOption()"; + return ""; + } +} + + +void NetworkSocket::bind(SocketType socketType, const QHostAddress& address, quint16 port) { + switch (socketType) { + case SocketType::UDP: + _udpSocket.bind(address, port); + break; +#if defined(WEBRTC_DATA_CHANNELS) + case SocketType::WebRTC: + _webrtcSocket.bind(address, port); + break; +#endif + default: + qCCritical(networking) << "Socket type not specified in bind()"; + } +} + +void NetworkSocket::abort(SocketType socketType) { + switch (socketType) { + case SocketType::UDP: + _udpSocket.abort(); + break; +#if defined(WEBRTC_DATA_CHANNELS) + case SocketType::WebRTC: + _webrtcSocket.abort(); + break; +#endif + default: + qCCritical(networking) << "Socket type not specified in abort()"; + } +} + + +quint16 NetworkSocket::localPort(SocketType socketType) const { + switch (socketType) { + case SocketType::UDP: + return _udpSocket.localPort(); +#if defined(WEBRTC_DATA_CHANNELS) + case SocketType::WebRTC: + return _webrtcSocket.localPort(); +#endif + default: + qCCritical(networking) << "Socket type not specified in localPort()"; + return 0; + } +} + +qintptr NetworkSocket::socketDescriptor(SocketType socketType) const { + switch (socketType) { + case SocketType::UDP: + return _udpSocket.socketDescriptor(); +#if defined(WEBRTC_DATA_CHANNELS) + case SocketType::WebRTC: + return _webrtcSocket.socketDescriptor(); + return 0; +#endif + default: + qCCritical(networking) << "Socket type not specified in socketDescriptor()"; + return 0; + } +} + + +qint64 NetworkSocket::writeDatagram(const QByteArray& datagram, const HifiSockAddr& sockAddr) { + switch (sockAddr.getSocketType()) { + case SocketType::UDP: + // WEBRTC TODO: The Qt documentation says that the following call shouldn't be used if the UDP socket is connected!!! + // https://doc.qt.io/qt-5/qudpsocket.html#writeDatagram + return _udpSocket.writeDatagram(datagram, sockAddr.getAddress(), sockAddr.getPort()); +#if defined(WEBRTC_DATA_CHANNELS) + case SocketType::WebRTC: + return _webrtcSocket.writeDatagram(datagram, sockAddr.getPort()); +#endif + default: + qCCritical(networking) << "Socket type not specified in writeDatagram() address"; + return 0; + } +} + +qint64 NetworkSocket::bytesToWrite(SocketType socketType, quint16 port) const { + switch (socketType) { + case SocketType::UDP: + return _udpSocket.bytesToWrite(); +#if defined(WEBRTC_DATA_CHANNELS) + case SocketType::WebRTC: + return _webrtcSocket.bytesToWrite(port); +#endif + default: + qCCritical(networking) << "Socket type not specified in bytesToWrite()"; + return 0; + } +} + + +bool NetworkSocket::hasPendingDatagrams() const { + return +#if defined(WEBRTC_DATA_CHANNELS) + _webrtcSocket.hasPendingDatagrams() || +#endif + _udpSocket.hasPendingDatagrams(); +} + +qint64 NetworkSocket::pendingDatagramSize() { +#if defined(WEBRTC_DATA_CHANNELS) + // Alternate socket types, remembering the socket type used so that the same socket type is used next readDatagram(). + if (_lastSocketTypeRead == SocketType::UDP) { + if (_webrtcSocket.hasPendingDatagrams()) { + _pendingDatagramSizeSocketType = SocketType::WebRTC; + return _webrtcSocket.pendingDatagramSize(); + } else { + _pendingDatagramSizeSocketType = SocketType::UDP; + return _udpSocket.pendingDatagramSize(); + } + } else { + if (_udpSocket.hasPendingDatagrams()) { + _pendingDatagramSizeSocketType = SocketType::UDP; + return _udpSocket.pendingDatagramSize(); + } else { + _pendingDatagramSizeSocketType = SocketType::WebRTC; + return _webrtcSocket.pendingDatagramSize(); + } + } +#else + return _udpSocket.pendingDatagramSize(); +#endif +} + +qint64 NetworkSocket::readDatagram(char* data, qint64 maxSize, HifiSockAddr* sockAddr) { +#if defined(WEBRTC_DATA_CHANNELS) + // Read per preceding pendingDatagramSize() if any, otherwise alternate socket types. + if (_pendingDatagramSizeSocketType == SocketType::UDP + || _pendingDatagramSizeSocketType == SocketType::Unknown && _lastSocketTypeRead == SocketType::WebRTC) { + _lastSocketTypeRead = SocketType::UDP; + _pendingDatagramSizeSocketType = SocketType::Unknown; + if (sockAddr) { + sockAddr->setSocketType(SocketType::UDP); + return _udpSocket.readDatagram(data, maxSize, sockAddr->getAddressPointer(), sockAddr->getPortPointer()); + } else { + return _udpSocket.readDatagram(data, maxSize); + } + } else { + _lastSocketTypeRead = SocketType::WebRTC; + _pendingDatagramSizeSocketType = SocketType::Unknown; + if (sockAddr) { + sockAddr->setSocketType(SocketType::WebRTC); + return _webrtcSocket.readDatagram(data, maxSize, sockAddr->getAddressPointer(), sockAddr->getPortPointer()); + } else { + return _webrtcSocket.readDatagram(data, maxSize); + } + } +#else + if (sockAddr) { + sockAddr->setSocketType(SocketType::UDP); + return _udpSocket.readDatagram(data, maxSize, sockAddr->getAddressPointer(), sockAddr->getPortPointer()); + } else { + return _udpSocket.readDatagram(data, maxSize); + } +#endif +} + + +QAbstractSocket::SocketState NetworkSocket::state(SocketType socketType) const { + switch (socketType) { + case SocketType::UDP: + return _udpSocket.state(); +#if defined(WEBRTC_DATA_CHANNELS) + case SocketType::WebRTC: + return _webrtcSocket.state(); +#endif + default: + qCCritical(networking) << "Socket type not specified in state()"; + return QAbstractSocket::SocketState::UnconnectedState; + } +} + + +QAbstractSocket::SocketError NetworkSocket::error(SocketType socketType) const { + switch (socketType) { + case SocketType::UDP: + return _udpSocket.error(); +#if defined(WEBRTC_DATA_CHANNELS) + case SocketType::WebRTC: + return _webrtcSocket.error(); +#endif + default: + qCCritical(networking) << "Socket type not specified in error()"; + return QAbstractSocket::SocketError::UnknownSocketError; + } +} + +QString NetworkSocket::errorString(SocketType socketType) const { + switch (socketType) { + case SocketType::UDP: + return _udpSocket.errorString(); +#if defined(WEBRTC_DATA_CHANNELS) + case SocketType::WebRTC: + return _webrtcSocket.errorString(); +#endif + default: + qCCritical(networking) << "Socket type not specified in errorString()"; + return ""; + } +} + + +void NetworkSocket::onUDPStateChanged(QAbstractSocket::SocketState socketState) { + emit stateChanged(SocketType::UDP, socketState); +} + +void NetworkSocket::onWebRTCStateChanged(QAbstractSocket::SocketState socketState) { + emit stateChanged(SocketType::WebRTC, socketState); +} + +void NetworkSocket::onUDPSocketError(QAbstractSocket::SocketError socketError) { + emit NetworkSocket::socketError(SocketType::UDP, socketError); +} + +void NetworkSocket::onWebRTCSocketError(QAbstractSocket::SocketError socketError) { + emit NetworkSocket::socketError(SocketType::WebRTC, socketError); +} diff --git a/libraries/networking/src/udt/NetworkSocket.h b/libraries/networking/src/udt/NetworkSocket.h new file mode 100644 index 00000000000..b946ab0bd09 --- /dev/null +++ b/libraries/networking/src/udt/NetworkSocket.h @@ -0,0 +1,163 @@ +// +// NetworkSocket.h +// libraries/networking/src/udt +// +// Created by David Rowe on 21 Jun 2021. +// Copyright 2021 Vircadia contributors. +// + +#ifndef vircadia_NetworkSocket_h +#define vircadia_NetworkSocket_h + +#include +#include + +#include + +#include "../HifiSockAddr.h" +#include "../NodeType.h" +#include "../SocketType.h" +#if defined(WEBRTC_DATA_CHANNELS) +#include "../webrtc/WebRTCSocket.h" +#endif + +/// @addtogroup Networking +/// @{ + + +/// @brief Multiplexes a QUdpSocket and a WebRTCSocket so that they appear as a single QUdpSocket-style socket. +class NetworkSocket : public QObject { + Q_OBJECT + +public: + + /// @brief Constructs a new NetworkSocket object. + /// @param parent Qt parent object. + /// @param nodeType The type of node that the NetworkSocket object is being used in. + NetworkSocket(QObject* parent, NodeType_t nodeType); + + + /// @brief Set the value of a UDP or WebRTC socket option. + /// @param socketType The type of socket for which to set the option value. + /// @param option The option to set the value of. + /// @param value The option value. + void setSocketOption(SocketType socketType, QAbstractSocket::SocketOption option, const QVariant& value); + + /// @brief Gets the value of a UDP or WebRTC socket option. + /// @param socketType The type of socket for which to get the option value. + /// @param option The option to get the value of. + /// @return The option value. + QVariant socketOption(SocketType socketType, QAbstractSocket::SocketOption option); + + + /// @brief Binds the UDP or WebRTC socket to an address and port. + /// @param socketType The type of socket to bind. + /// @param address The address to bind to. + /// @param port The port to bind to. + void bind(SocketType socketType, const QHostAddress& address, quint16 port = 0); + + /// @brief Immediately closes and resets the socket. + /// @param socketType The type of socket to close and reset. + void abort(SocketType socketType); + + + /// @brief Gets the UDP or WebRTC local port number. + /// @param socketType The type of socket for which to the get local port number. + /// @return The UDP or WebRTC local port number if available, otherwise 0. + quint16 localPort(SocketType socketType) const; + + /// @brief Returns the native socket descriptor of the UDP or WebRTC socket. + /// @param socketType The type of socket to get the socket descriptor for. + /// @return The native socket descriptor if available, otherwise -1. + qintptr socketDescriptor(SocketType socketType) const; + + + /// @brief Sends a datagram to a network address. + /// @param datagram The datagram to send. + /// @param sockAddr The address to send to. + /// @return The number of bytes if successfully sent, otherwise -1. + qint64 writeDatagram(const QByteArray& datagram, const HifiSockAddr& sockAddr); + + /// @brief Gets the number of bytes waiting to be written. + /// @detail For UDP, there's a single buffer used for all destinations. For WebRTC, each destination has its own buffer. + /// @param socketType The type of socket for which to get the number of bytes waiting to be written. + /// @param port If a WebRTC socket, the data channel for which to get the number of bytes waiting. + /// @return The number of bytes waiting to be written. + qint64 bytesToWrite(SocketType socketType, quint16 port = 0) const; + + + /// @brief Gets whether there is a pending datagram waiting to be read. + /// @return true if there is a datagram waiting to be read, false if there isn't. + bool hasPendingDatagrams() const; + + /// @brief Gets the size of the next pending datagram, alternating between socket types if both have datagrams to read. + /// @return The size of the next pendign datagram. + qint64 pendingDatagramSize(); + + /// @brief Reads the next datagram per the most recent pendingDatagramSize call if made, otherwise alternating between + /// socket types if both have datagrams to read. + /// @param data The destination to write the data into. + /// @param maxSize The maximum number of bytes to read. + /// @param sockAddr The destination to write the source network address into. + /// @return The number of bytes if successfully read, otherwise -1. + qint64 readDatagram(char* data, qint64 maxSize, HifiSockAddr* sockAddr = nullptr); + + + /// @brief Gets the state of the UDP or WebRTC socket. + /// @param socketType The type of socket for which to get the state. + /// @return The socket state. + QAbstractSocket::SocketState state(SocketType socketType) const; + + + /// @brief Gets the type of error that last occurred. + /// @param socketType The type of socket for which to get the last error. + /// @return The type of error that last occurred + QAbstractSocket::SocketError error(SocketType socketType) const; + + /// @brief Gets the description of the error that last occurred. + /// @param socketType The type of socket for which to get the last error's description. + /// @return The description of the error that last occurred. + QString errorString(SocketType socketType) const; + +signals: + + /// @brief Emitted each time new data becomes available for reading. + void readyRead(); + + /// @brief Emitted when the state of the underlying UDP or WebRTC socket changes. + /// @param socketType The type of socket that changed state. + /// @param socketState The socket's new state. + void stateChanged(SocketType socketType, QAbstractSocket::SocketState socketState); + + /// @brief + /// @param socketType + /// @param socketError + void socketError(SocketType socketType, QAbstractSocket::SocketError socketError); + +private slots: + + void onUDPStateChanged(QAbstractSocket::SocketState socketState); + void onWebRTCStateChanged(QAbstractSocket::SocketState socketState); + + void onUDPSocketError(QAbstractSocket::SocketError socketError); + void onWebRTCSocketError(QAbstractSocket::SocketError socketError); + +private: + + QObject* _parent; + + QUdpSocket _udpSocket; +#if defined(WEBRTC_DATA_CHANNELS) + WebRTCSocket _webrtcSocket; +#endif + +#if defined(WEBRTC_DATA_CHANNELS) + SocketType _pendingDatagramSizeSocketType { SocketType::Unknown }; + SocketType _lastSocketTypeRead { SocketType::Unknown }; +#endif +}; + + +/// @} + +#endif // vircadia_NetworkSocket_h diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 017855d6283..5365bc1e6c2 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -39,18 +39,17 @@ using namespace udt; #endif -Socket::Socket(QObject* parent, bool shouldChangeSocketOptions) : +Socket::Socket(QObject* parent, bool shouldChangeSocketOptions, NodeType_t nodeType) : QObject(parent), - _udpSocket(parent), + _networkSocket(parent, nodeType), _readyReadBackupTimer(new QTimer(this)), _shouldChangeSocketOptions(shouldChangeSocketOptions) { - connect(&_udpSocket, &QUdpSocket::readyRead, this, &Socket::readPendingDatagrams); + connect(&_networkSocket, &NetworkSocket::readyRead, this, &Socket::readPendingDatagrams); // make sure we hear about errors and state changes from the underlying socket - connect(&_udpSocket, SIGNAL(error(QAbstractSocket::SocketError)), - this, SLOT(handleSocketError(QAbstractSocket::SocketError))); - connect(&_udpSocket, &QAbstractSocket::stateChanged, this, &Socket::handleStateChanged); + connect(&_networkSocket, &NetworkSocket::socketError, this, &Socket::handleSocketError); + connect(&_networkSocket, &NetworkSocket::stateChanged, this, &Socket::handleStateChanged); // in order to help track down the zombie server bug, add a timer to check if we missed a readyRead const int READY_READ_BACKUP_CHECK_MSECS = 2 * 1000; @@ -58,19 +57,21 @@ Socket::Socket(QObject* parent, bool shouldChangeSocketOptions) : _readyReadBackupTimer->start(READY_READ_BACKUP_CHECK_MSECS); } -void Socket::bind(const QHostAddress& address, quint16 port) { - - _udpSocket.bind(address, port); +void Socket::bind(SocketType socketType, const QHostAddress& address, quint16 port) { + _networkSocket.bind(socketType, address, port); if (_shouldChangeSocketOptions) { - setSystemBufferSizes(); + setSystemBufferSizes(socketType); + if (socketType == SocketType::WebRTC) { + return; + } #if defined(Q_OS_LINUX) - auto sd = _udpSocket.socketDescriptor(); + auto sd = _networkSocket.socketDescriptor(socketType); int val = IP_PMTUDISC_DONT; setsockopt(sd, IPPROTO_IP, IP_MTU_DISCOVER, &val, sizeof(val)); #elif defined(Q_OS_WIN) - auto sd = _udpSocket.socketDescriptor(); + auto sd = _networkSocket.socketDescriptor(socketType); int val = 0; // false if (setsockopt(sd, IPPROTO_IP, IP_DONTFRAGMENT, (const char *)&val, sizeof(val))) { auto wsaErr = WSAGetLastError(); @@ -80,16 +81,16 @@ void Socket::bind(const QHostAddress& address, quint16 port) { } } -void Socket::rebind() { - rebind(_udpSocket.localPort()); +void Socket::rebind(SocketType socketType) { + rebind(socketType, _networkSocket.localPort(socketType)); } -void Socket::rebind(quint16 localPort) { - _udpSocket.abort(); - bind(QHostAddress::AnyIPv4, localPort); +void Socket::rebind(SocketType socketType, quint16 localPort) { + _networkSocket.abort(socketType); + bind(socketType, QHostAddress::AnyIPv4, localPort); } -void Socket::setSystemBufferSizes() { +void Socket::setSystemBufferSizes(SocketType socketType) { for (int i = 0; i < 2; i++) { QAbstractSocket::SocketOption bufferOpt; QString bufferTypeString; @@ -98,20 +99,22 @@ void Socket::setSystemBufferSizes() { if (i == 0) { bufferOpt = QAbstractSocket::SendBufferSizeSocketOption; - numBytes = udt::UDP_SEND_BUFFER_SIZE_BYTES; + numBytes = socketType == SocketType::UDP + ? udt::UDP_SEND_BUFFER_SIZE_BYTES : udt::WEBRTC_SEND_BUFFER_SIZE_BYTES; bufferTypeString = "send"; } else { bufferOpt = QAbstractSocket::ReceiveBufferSizeSocketOption; - numBytes = udt::UDP_RECEIVE_BUFFER_SIZE_BYTES; + numBytes = socketType == SocketType::UDP + ? udt::UDP_RECEIVE_BUFFER_SIZE_BYTES : udt::WEBRTC_RECEIVE_BUFFER_SIZE_BYTES; bufferTypeString = "receive"; } - int oldBufferSize = _udpSocket.socketOption(bufferOpt).toInt(); + int oldBufferSize = _networkSocket.socketOption(socketType, bufferOpt).toInt(); if (oldBufferSize < numBytes) { - _udpSocket.setSocketOption(bufferOpt, QVariant(numBytes)); - int newBufferSize = _udpSocket.socketOption(bufferOpt).toInt(); + _networkSocket.setSocketOption(socketType, bufferOpt, QVariant(numBytes)); + int newBufferSize = _networkSocket.socketOption(socketType, bufferOpt).toInt(); qCDebug(networking) << "Changed socket" << bufferTypeString << "buffer size from" << oldBufferSize << "to" << newBufferSize << "bytes"; @@ -235,16 +238,18 @@ qint64 Socket::writeDatagram(const char* data, qint64 size, const HifiSockAddr& } qint64 Socket::writeDatagram(const QByteArray& datagram, const HifiSockAddr& sockAddr) { + auto socketType = sockAddr.getSocketType(); // don't attempt to write the datagram if we're unbound. Just drop it. - // _udpSocket.writeDatagram will return an error anyway, but there are + // _networkSocket.writeDatagram will return an error anyway, but there are // potential crashes in Qt when that happens. - if (_udpSocket.state() != QAbstractSocket::BoundState) { + if (_networkSocket.state(socketType) != QAbstractSocket::BoundState) { qCDebug(networking) << "Attempt to writeDatagram when in unbound state to" << sockAddr; return -1; } - qint64 bytesWritten = _udpSocket.writeDatagram(datagram, sockAddr.getAddress(), sockAddr.getPort()); - int pending = _udpSocket.bytesToWrite(); + qint64 bytesWritten = _networkSocket.writeDatagram(datagram, sockAddr); + + int pending = _networkSocket.bytesToWrite(socketType, sockAddr.getPort()); if (bytesWritten < 0 || pending) { int wsaError = 0; static std::atomic previousWsaError (0); @@ -252,8 +257,8 @@ qint64 Socket::writeDatagram(const QByteArray& datagram, const HifiSockAddr& soc wsaError = WSAGetLastError(); #endif QString errorString; - QDebug(&errorString) << "udt::writeDatagram (" << _udpSocket.state() << sockAddr << ") error - " - << wsaError << _udpSocket.error() << "(" << _udpSocket.errorString() << ")" + QDebug(&errorString) << "udt::writeDatagram (" << _networkSocket.state(socketType) << sockAddr << ") error - " + << wsaError << _networkSocket.error(socketType) << "(" << _networkSocket.errorString(socketType) << ")" << (pending ? "pending bytes:" : "pending:") << pending; if (previousWsaError.exchange(wsaError) != wsaError) { @@ -343,7 +348,7 @@ void Socket::messageFailed(Connection* connection, Packet::MessageNumber message } void Socket::checkForReadyReadBackup() { - if (_udpSocket.hasPendingDatagrams()) { + if (_networkSocket.hasPendingDatagrams()) { qCDebug(networking) << "Socket::checkForReadyReadBackup() detected blocked readyRead signal. Flushing pending datagrams."; // so that birarda can possibly figure out how the heck we get into this state in the first place @@ -357,8 +362,8 @@ void Socket::checkForReadyReadBackup() { // drop all of the pending datagrams on the floor int droppedCount = 0; - while (_udpSocket.hasPendingDatagrams()) { - _udpSocket.readDatagram(nullptr, 0); + while (_networkSocket.hasPendingDatagrams()) { + _networkSocket.readDatagram(nullptr, 0); ++droppedCount; } qCDebug(networking) << "Flushed" << droppedCount << "Packets"; @@ -371,8 +376,8 @@ void Socket::readPendingDatagrams() { const auto abortTime = system_clock::now() + MAX_PROCESS_TIME; int packetSizeWithHeader = -1; - while (_udpSocket.hasPendingDatagrams() && - (packetSizeWithHeader = _udpSocket.pendingDatagramSize()) != -1) { + while (_networkSocket.hasPendingDatagrams() && + (packetSizeWithHeader = _networkSocket.pendingDatagramSize()) != -1) { if (system_clock::now() > abortTime) { // We've been running for too long, stop processing packets for now // Once we've processed the event queue, we'll come back to packet processing @@ -397,8 +402,7 @@ void Socket::readPendingDatagrams() { auto buffer = std::unique_ptr(new char[packetSizeWithHeader]); // pull the datagram - auto sizeRead = _udpSocket.readDatagram(buffer.get(), packetSizeWithHeader, - senderSockAddr.getAddressPointer(), senderSockAddr.getPortPointer()); + auto sizeRead = _networkSocket.readDatagram(buffer.get(), packetSizeWithHeader, &senderSockAddr); // save information for this packet, in case it is the one that sticks readyRead _lastPacketSizeRead = sizeRead; @@ -540,17 +544,17 @@ std::vector Socket::getConnectionSockAddrs() { return addr; } -void Socket::handleSocketError(QAbstractSocket::SocketError socketError) { +void Socket::handleSocketError(SocketType socketType, QAbstractSocket::SocketError socketError) { int wsaError = 0; static std::atomic previousWsaError(0); #ifdef WIN32 wsaError = WSAGetLastError(); #endif - int pending = _udpSocket.bytesToWrite(); + int pending = _networkSocket.bytesToWrite(socketType); QString errorString; - QDebug(&errorString) << "udt::Socket (" << _udpSocket.state() << ") error - " << wsaError << socketError << - "(" << _udpSocket.errorString() << ")" << (pending ? "pending bytes:" : "pending:") - << pending; + QDebug(&errorString) << "udt::Socket (" << socketTypeToString(socketType) << _networkSocket.state(socketType) + << ") error - " << wsaError << socketError << "(" << _networkSocket.errorString(socketType) << ")" + << (pending ? "pending bytes:" : "pending:") << pending; if (previousWsaError.exchange(wsaError) != wsaError) { qCDebug(networking).noquote() << errorString; @@ -563,9 +567,9 @@ void Socket::handleSocketError(QAbstractSocket::SocketError socketError) { } } -void Socket::handleStateChanged(QAbstractSocket::SocketState socketState) { +void Socket::handleStateChanged(SocketType socketType, QAbstractSocket::SocketState socketState) { if (socketState != QAbstractSocket::BoundState) { - qCDebug(networking) << "udt::Socket state changed - state is now" << socketState; + qCDebug(networking) << socketTypeToString(socketType) << "socket state changed - state is now" << socketState; } } diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index 6cd2d256594..1b697cdb777 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -21,11 +21,11 @@ #include #include -#include #include "../HifiSockAddr.h" #include "TCPVegasCC.h" #include "Connection.h" +#include "NetworkSocket.h" //#define UDT_CONNECTION_DEBUG @@ -55,9 +55,9 @@ class Socket : public QObject { public: using StatsVector = std::vector>; - Socket(QObject* object = 0, bool shouldChangeSocketOptions = true); + Socket(QObject* object = 0, bool shouldChangeSocketOptions = true, NodeType_t nodeType = NodeType::Unassigned); - quint16 localPort() const { return _udpSocket.localPort(); } + quint16 localPort(SocketType socketType) const { return _networkSocket.localPort(socketType); } // Simple functions writing to the socket with no processing qint64 writeBasePacket(const BasePacket& packet, const HifiSockAddr& sockAddr); @@ -67,9 +67,9 @@ class Socket : public QObject { qint64 writeDatagram(const char* data, qint64 size, const HifiSockAddr& sockAddr); qint64 writeDatagram(const QByteArray& datagram, const HifiSockAddr& sockAddr); - void bind(const QHostAddress& address, quint16 port = 0); - void rebind(quint16 port); - void rebind(); + void bind(SocketType socketType, const QHostAddress& address, quint16 port = 0); + void rebind(SocketType socketType, quint16 port); + void rebind(SocketType socketType); void setPacketFilterOperator(PacketFilterOperator filterOperator) { _packetFilterOperator = filterOperator; } void setPacketHandler(PacketHandler handler) { _packetHandler = handler; } @@ -105,11 +105,11 @@ private slots: void readPendingDatagrams(); void checkForReadyReadBackup(); - void handleSocketError(QAbstractSocket::SocketError socketError); - void handleStateChanged(QAbstractSocket::SocketState socketState); + void handleSocketError(SocketType socketType, QAbstractSocket::SocketError socketError); + void handleStateChanged(SocketType socketType, QAbstractSocket::SocketState socketState); private: - void setSystemBufferSizes(); + void setSystemBufferSizes(SocketType socketType); Connection* findOrCreateConnection(const HifiSockAddr& sockAddr, bool filterCreation = false); // privatized methods used by UDTTest - they are private since they must be called on the Socket thread @@ -121,7 +121,7 @@ private slots: Q_INVOKABLE void writeReliablePacket(Packet* packet, const HifiSockAddr& sockAddr); Q_INVOKABLE void writeReliablePacketList(PacketList* packetList, const HifiSockAddr& sockAddr); - QUdpSocket _udpSocket { this }; + NetworkSocket _networkSocket; PacketFilterOperator _packetFilterOperator; PacketHandler _packetHandler; MessageHandler _messageHandler; diff --git a/tools/ice-client/src/ICEClientApp.cpp b/tools/ice-client/src/ICEClientApp.cpp index 0301fad6f43..42e38bd748c 100644 --- a/tools/ice-client/src/ICEClientApp.cpp +++ b/tools/ice-client/src/ICEClientApp.cpp @@ -62,7 +62,7 @@ ICEClientApp::ICEClientApp(int argc, char* argv[]) : const_cast(&networking())->setEnabled(QtWarningMsg, false); } - _stunSockAddr = HifiSockAddr(STUN_SERVER_HOSTNAME, STUN_SERVER_PORT, true); + _stunSockAddr = HifiSockAddr(SocketType::UDP, STUN_SERVER_HOSTNAME, STUN_SERVER_PORT, true); _cacheSTUNResult = parser.isSet(cacheSTUNOption); @@ -79,7 +79,7 @@ ICEClientApp::ICEClientApp(int argc, char* argv[]) : } } - _iceServerAddr = HifiSockAddr("127.0.0.1", ICE_SERVER_DEFAULT_PORT); + _iceServerAddr = HifiSockAddr(SocketType::UDP, "127.0.0.1", ICE_SERVER_DEFAULT_PORT); if (parser.isSet(iceServerAddressOption)) { // parse the IP and port combination for this target QString hostnamePortString = parser.value(iceServerAddressOption); @@ -96,7 +96,7 @@ ICEClientApp::ICEClientApp(int argc, char* argv[]) : QMetaObject::invokeMethod(this, "quit", Qt::QueuedConnection); } else { - _iceServerAddr = HifiSockAddr(address, port); + _iceServerAddr = HifiSockAddr(SocketType::UDP, address, port); } } @@ -132,7 +132,7 @@ void ICEClientApp::openSocket() { _socket = new udt::Socket(); unsigned int localPort = 0; - _socket->bind(QHostAddress::AnyIPv4, localPort); + _socket->bind(SocketType::UDP, QHostAddress::AnyIPv4, localPort); _socket->setPacketHandler([this](std::unique_ptr packet) { processPacket(std::move(packet)); }); _socket->addUnfilteredHandler(_stunSockAddr, [this](std::unique_ptr packet) { @@ -140,10 +140,10 @@ void ICEClientApp::openSocket() { }); if (_verbose) { - qDebug() << "local port is" << _socket->localPort(); + qDebug() << "local port is" << _socket->localPort(SocketType::UDP); } - _localSockAddr = HifiSockAddr("127.0.0.1", _socket->localPort()); - _publicSockAddr = HifiSockAddr("127.0.0.1", _socket->localPort()); + _localSockAddr = HifiSockAddr(SocketType::UDP, "127.0.0.1", _socket->localPort(SocketType::UDP)); + _publicSockAddr = HifiSockAddr(SocketType::UDP, "127.0.0.1", _socket->localPort(SocketType::UDP)); _domainPingCount = 0; } @@ -188,7 +188,7 @@ void ICEClientApp::doSomething() { if (_verbose) { qDebug() << "using cached STUN response"; } - _publicSockAddr.setPort(_socket->localPort()); + _publicSockAddr.setPort(_socket->localPort(SocketType::UDP)); setState(talkToIceServer); } @@ -303,7 +303,7 @@ void ICEClientApp::processSTUNResponse(std::unique_ptr packet) uint16_t newPublicPort; QHostAddress newPublicAddress; if (LimitedNodeList::parseSTUNResponse(packet.get(), newPublicAddress, newPublicPort)) { - _publicSockAddr = HifiSockAddr(newPublicAddress, newPublicPort); + _publicSockAddr = HifiSockAddr(SocketType::UDP, newPublicAddress, newPublicPort); if (_verbose) { qDebug() << "My public address is" << _publicSockAddr; } From 2a9ab98e41e9fdcfea1d57ae3a056ae21af94fd4 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sat, 26 Jun 2021 22:07:48 +1200 Subject: [PATCH 18/82] Miscellaneous tidying and fixes --- libraries/networking/src/AddressManager.h | 10 +++++----- libraries/networking/src/DomainHandler.cpp | 2 +- libraries/networking/src/NodeList.cpp | 10 +++++++--- libraries/networking/src/NodeList.h | 2 +- libraries/networking/src/udt/BasePacket.cpp | 2 +- libraries/networking/src/udt/NetworkSocket.h | 2 +- libraries/networking/src/udt/PacketHeaders.h | 2 ++ libraries/networking/src/webrtc/WebRTCSocket.h | 2 +- tools/doxygen/Doxyfile | 4 ++-- 9 files changed, 21 insertions(+), 15 deletions(-) diff --git a/libraries/networking/src/AddressManager.h b/libraries/networking/src/AddressManager.h index e8793edc1dc..74ec1c42667 100644 --- a/libraries/networking/src/AddressManager.h +++ b/libraries/networking/src/AddressManager.h @@ -252,8 +252,8 @@ public slots: * "127.0.0.1" or "localhost"), a file:/// address, a domain name, a named path * on a domain (starts with "/"), a position or position and orientation, or a user (starts with * "@"). - * @param {boolean} [fromSuggestions=false] - Set to true if the address is obtained from the "Goto" dialog. - * Helps ensure that user's location history is correctly maintained. + * @param {boolean} [fromSuggestions=false] - Set to true if the address is obtained from the "Explore" app. + * Helps ensure that the user's location history is correctly maintained. */ void handleLookupString(const QString& lookupString, bool fromSuggestions = false); @@ -390,10 +390,10 @@ public slots: void lookupResultIsNotFound(); /*@jsdoc - * Triggered when a request is made to go to an IP address. + * Triggered when a request is made to go to a URL or IP address. * @function location.possibleDomainChangeRequired - * @param {Url} domainURL - URL for domain - * @param {Uuid} domainID - The UUID of the domain to go to. + * @param {string} domainURL - The URL of the domain. + * @param {Uuid} domainID - The UUID of the domain to go to. May be "{@link Uuid|Uuid.NULL} if not yet known. * @returns {Signal} */ // No example because this function isn't typically used in scripts. diff --git a/libraries/networking/src/DomainHandler.cpp b/libraries/networking/src/DomainHandler.cpp index a77cde4ecc0..e30e046013d 100644 --- a/libraries/networking/src/DomainHandler.cpp +++ b/libraries/networking/src/DomainHandler.cpp @@ -368,7 +368,7 @@ void DomainHandler::setIsConnected(bool isConnected) { emit connectedToDomain(_domainURL); // FIXME: Reinstate the requestDomainSettings() call here in version 2021.2.0 instead of having it in - // NodeList::processDomainServerList(). + // NodeList::processDomainList(). /* if (_domainURL.scheme() == URL_SCHEME_HIFI && !_domainURL.host().isEmpty()) { // we've connected to new domain - time to ask it for global settings diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index a6600991999..15da4f6d43b 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -147,7 +147,7 @@ NodeList::NodeList(char newOwnerType, int socketListenPort, int dtlsListenPort) auto& packetReceiver = getPacketReceiver(); packetReceiver.registerListener(PacketType::DomainList, - PacketReceiver::makeUnsourcedListenerReference(this, &NodeList::processDomainServerList)); + PacketReceiver::makeUnsourcedListenerReference(this, &NodeList::processDomainList)); packetReceiver.registerListener(PacketType::Ping, PacketReceiver::makeSourcedListenerReference(this, &NodeList::processPingPacket)); packetReceiver.registerListener(PacketType::PingReply, @@ -357,7 +357,7 @@ void NodeList::sendDomainServerCheckIn() { if (publicSockAddr.isNull()) { // we don't know our public socket and we need to send it to the domain server - qCDebug(networking_ice) << "Waiting for inital public socket from STUN. Will not send domain-server check in."; + qCDebug(networking_ice) << "Waiting for initial public socket from STUN. Will not send domain-server check in."; } else if (domainHandlerIp.isNull() && _domainHandler.requiresICE()) { qCDebug(networking_ice) << "Waiting for ICE discovered domain-server socket. Will not send domain-server check in."; handleICEConnectionToDomainServer(); @@ -401,6 +401,8 @@ void NodeList::sendDomainServerCheckIn() { return; } + // WEBRTC TODO: Move code into packet library. And update reference in DomainConnectRequest.js. + auto domainPacket = NLPacket::create(domainPacketType); QDataStream packetStream(domainPacket.get()); @@ -709,7 +711,9 @@ void NodeList::processDomainServerConnectionTokenPacket(QSharedPointer message) { +void NodeList::processDomainList(QSharedPointer message) { + + // WEBRTC TODO: Move code into packet library. And update reference in DomainServerList.js. // parse header information QDataStream packetStream(message->getMessage()); diff --git a/libraries/networking/src/NodeList.h b/libraries/networking/src/NodeList.h index 2ffac59702d..9dea08c0a9c 100644 --- a/libraries/networking/src/NodeList.h +++ b/libraries/networking/src/NodeList.h @@ -109,7 +109,7 @@ public slots: void sendDomainServerCheckIn(); void handleDSPathQuery(const QString& newPath); - void processDomainServerList(QSharedPointer message); + void processDomainList(QSharedPointer message); void processDomainServerAddedNode(QSharedPointer message); void processDomainServerRemovedNode(QSharedPointer message); void processDomainServerPathResponse(QSharedPointer message); diff --git a/libraries/networking/src/udt/BasePacket.cpp b/libraries/networking/src/udt/BasePacket.cpp index 12a174b7d3e..777e7780a1f 100644 --- a/libraries/networking/src/udt/BasePacket.cpp +++ b/libraries/networking/src/udt/BasePacket.cpp @@ -57,7 +57,7 @@ BasePacket::BasePacket(qint64 size) { } // Sanity check - Q_ASSERT(size >= 0 || size < maxPayload); + Q_ASSERT(size >= 0 && size <= maxPayload); _packetSize = size; _packet.reset(new char[_packetSize]()); diff --git a/libraries/networking/src/udt/NetworkSocket.h b/libraries/networking/src/udt/NetworkSocket.h index b946ab0bd09..dfd9fe5e728 100644 --- a/libraries/networking/src/udt/NetworkSocket.h +++ b/libraries/networking/src/udt/NetworkSocket.h @@ -79,7 +79,7 @@ class NetworkSocket : public QObject { qint64 writeDatagram(const QByteArray& datagram, const HifiSockAddr& sockAddr); /// @brief Gets the number of bytes waiting to be written. - /// @detail For UDP, there's a single buffer used for all destinations. For WebRTC, each destination has its own buffer. + /// @details For UDP, there's a single buffer used for all destinations. For WebRTC, each destination has its own buffer. /// @param socketType The type of socket for which to get the number of bytes waiting to be written. /// @param port If a WebRTC socket, the data channel for which to get the number of bytes waiting. /// @return The number of bytes waiting to be written. diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index 819551045ac..d57fa9f663d 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -10,6 +10,8 @@ // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html // +// WEBRTC TODO: Rename / split up into files with better names. + #ifndef hifi_PacketHeaders_h #define hifi_PacketHeaders_h diff --git a/libraries/networking/src/webrtc/WebRTCSocket.h b/libraries/networking/src/webrtc/WebRTCSocket.h index ed33608859f..9b2011b6200 100644 --- a/libraries/networking/src/webrtc/WebRTCSocket.h +++ b/libraries/networking/src/webrtc/WebRTCSocket.h @@ -124,9 +124,9 @@ class WebRTCSocket : public QObject { public slots: /// @brief Handles the WebRTC data channel receiving a message. + /// @details Queues the message to be read via readDatagram. /// @param dataChannelID The data channel that the message was received on. /// @param message The message that was received. - /// @detail Queues the message to be read via readDatagram. void onDataChannelReceivedMessage(int dataChannelID, const QByteArray& message); signals: diff --git a/tools/doxygen/Doxyfile b/tools/doxygen/Doxyfile index ac8136a0c53..ee7ac1e2e15 100644 --- a/tools/doxygen/Doxyfile +++ b/tools/doxygen/Doxyfile @@ -509,7 +509,7 @@ EXTRACT_PACKAGE = NO # included in the documentation. # The default value is: NO. -EXTRACT_STATIC = NO +EXTRACT_STATIC = YES # If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined # locally in source files will be included in the documentation. If set to NO, @@ -794,7 +794,7 @@ CITE_BIB_FILES = # messages are off. # The default value is: NO. -QUIET = NO +QUIET = YES # The WARNINGS tag can be used to turn on/off the warning messages that are # generated to standard error (stderr) by doxygen. If WARNINGS is set to YES From 6b188d888ec5df42a13c6c527fdf845c02e23678 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Mon, 28 Jun 2021 11:15:18 +1200 Subject: [PATCH 19/82] Update to latest Vircadia-Web --- vircadia-web | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vircadia-web b/vircadia-web index ac1f13a39c7..ab6c8b1a54a 160000 --- a/vircadia-web +++ b/vircadia-web @@ -1 +1 @@ -Subproject commit ac1f13a39c702ee54bf2cda8bc35e5d34f7f0756 +Subproject commit ab6c8b1a54aec359b1894f70722f69cfec7f04f1 From d0c89c7d91bee8e4f6fba90a72fa92782efbd63a Mon Sep 17 00:00:00 2001 From: David Rowe Date: Wed, 30 Jun 2021 22:21:38 +1200 Subject: [PATCH 20/82] Tidying --- libraries/networking/src/SocketType.h | 8 ++++---- libraries/networking/src/webrtc/WebRTCSocket.cpp | 1 - libraries/networking/src/webrtc/WebRTCSocket.h | 1 - 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/libraries/networking/src/SocketType.h b/libraries/networking/src/SocketType.h index c63ee0cf348..a798fb59c6f 100644 --- a/libraries/networking/src/SocketType.h +++ b/libraries/networking/src/SocketType.h @@ -18,16 +18,16 @@ /// @{ -/// @brief The network socket type. +/// @brief The types of network socket. enum class SocketType { Unknown, ///< Unknown socket type. UDP, ///< UDP socket. - WebRTC ///< WebRTC socket. + WebRTC ///< WebRTC socket. A WebRTC data channel presented as a UDP-style socket. }; -/// @brief Returns the name of a SocketType value as a string. +/// @brief Returns the name of a SocketType value, e.g., "WebRTC". /// @param socketType The SocketType value. -/// @return The name of the SocketType value as a string. +/// @return The name of the SocketType value. static QString socketTypeToString(SocketType socketType) { static QStringList SOCKET_TYPE_STRINGS { "Unknown", "UDP", "WebRTC" }; return SOCKET_TYPE_STRINGS[(int)socketType]; diff --git a/libraries/networking/src/webrtc/WebRTCSocket.cpp b/libraries/networking/src/webrtc/WebRTCSocket.cpp index 5c6e3ce292b..b9eee027a17 100644 --- a/libraries/networking/src/webrtc/WebRTCSocket.cpp +++ b/libraries/networking/src/webrtc/WebRTCSocket.cpp @@ -16,7 +16,6 @@ WebRTCSocket::WebRTCSocket(QObject* parent, NodeType_t nodeType) : QObject(parent), - _parent(parent), _signalingServer(this /*, QHostAddress::AnyIPv4, DEFAULT_DOMAIN_SERVER_WS_PORT*/), _dataChannels(this, nodeType) { diff --git a/libraries/networking/src/webrtc/WebRTCSocket.h b/libraries/networking/src/webrtc/WebRTCSocket.h index 9b2011b6200..8d5d5fc3476 100644 --- a/libraries/networking/src/webrtc/WebRTCSocket.h +++ b/libraries/networking/src/webrtc/WebRTCSocket.h @@ -142,7 +142,6 @@ public slots: void setError(QAbstractSocket::SocketError errorType, QString errorString); void clearError(); - QObject* _parent; WebRTCSignalingServer _signalingServer; WebRTCDataChannels _dataChannels; From b7fdd5bb1ff796e392722c591d4e462c5b2b5270 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Thu, 1 Jul 2021 19:08:38 +1200 Subject: [PATCH 21/82] Remove extraneous Doxygen set-up instruction --- tools/doxygen/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/doxygen/README.md b/tools/doxygen/README.md index 658d8814aec..8b8fdc5c264 100644 --- a/tools/doxygen/README.md +++ b/tools/doxygen/README.md @@ -7,8 +7,6 @@ **Doxygen** ≥ 1.9.1 - https://www.doxygen.nl/ -Make a `/build/doxygen/` directory. - If you want to run Doxygen from a command prompt, add the Doxygen install's `/bin` directory to your system PATH. From f02ffe1ed921278a33f1cddf37be212090e359ed Mon Sep 17 00:00:00 2001 From: David Rowe Date: Fri, 2 Jul 2021 12:05:29 +1200 Subject: [PATCH 22/82] Use alternative signal/slot mechanism for Android compatibility --- libraries/networking/src/udt/NetworkSocket.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/udt/NetworkSocket.cpp b/libraries/networking/src/udt/NetworkSocket.cpp index 3fb039187ec..16f847a0681 100644 --- a/libraries/networking/src/udt/NetworkSocket.cpp +++ b/libraries/networking/src/udt/NetworkSocket.cpp @@ -22,7 +22,10 @@ NetworkSocket::NetworkSocket(QObject* parent, NodeType_t nodeType) : { connect(&_udpSocket, &QUdpSocket::readyRead, this, &NetworkSocket::readyRead); connect(&_udpSocket, &QAbstractSocket::stateChanged, this, &NetworkSocket::onUDPStateChanged); - connect(&_udpSocket, &QAbstractSocket::errorOccurred, this, &NetworkSocket::onUDPSocketError); + // Use old SIGNAL/SLOT mechanism for Android builds. + connect(&_udpSocket, SIGNAL(error(QAbstractSocket::SocketError)), + this, SLOT(onUDPSocketError(QAbstractSocket::SocketError))); + #if defined(WEBRTC_DATA_CHANNELS) connect(&_webrtcSocket, &WebRTCSocket::readyRead, this, &NetworkSocket::readyRead); connect(&_webrtcSocket, &WebRTCSocket::stateChanged, this, &NetworkSocket::onWebRTCStateChanged); From dacda8405c170d1ecd8ea333267deabd1f2e1d8a Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sat, 3 Jul 2021 08:43:31 +1200 Subject: [PATCH 23/82] Typo --- libraries/networking/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/CMakeLists.txt b/libraries/networking/CMakeLists.txt index 1835c7a6cd6..7a11329a14e 100644 --- a/libraries/networking/CMakeLists.txt +++ b/libraries/networking/CMakeLists.txt @@ -11,7 +11,7 @@ endif () if (WIN32) # we need ws2_32.lib on windows, but it's static so we don't bubble it up - # Libraries needed for WebRTC: security.log winmm.lib + # Libraries needed for WebRTC: security.lib winmm.lib target_link_libraries(${TARGET_NAME} ws2_32.lib security.lib winmm.lib) elseif(APPLE) # IOKit is needed for getting machine fingerprint From f6a8ae285db16d48de9cb35e23c49820bf1b5fb3 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sun, 4 Jul 2021 15:50:52 +1200 Subject: [PATCH 24/82] Fix WebRTC peer connection not being closed properly --- .../src/webrtc/WebRTCDataChannels.cpp | 94 +++++++++++++++---- .../src/webrtc/WebRTCDataChannels.h | 31 +++++- 2 files changed, 100 insertions(+), 25 deletions(-) diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp index 497ecf9a555..48ec1b7b9b8 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp @@ -22,7 +22,7 @@ const std::string ICE_SERVER_URI = "stun://ice.vircadia.com:7337"; -#define WEBRTC_DEBUG +// #define WEBRTC_DEBUG void WDCSetSessionDescriptionObserver::OnSuccess() { @@ -102,6 +102,10 @@ void WDCPeerConnectionObserver::OnDataChannel(rtc::scoped_refptronPeerConnectionStateChanged(newState); } @@ -253,6 +257,20 @@ void WDCConnection::sendIceCandidate(const IceCandidateInterface* candidate) { _parent->sendSignalingMessage(jsonObject); } +void WDCConnection::onPeerConnectionStateChanged(PeerConnectionInterface::PeerConnectionState state) { +#ifdef WEBRTC_DEBUG + const char* STATES[] = { + "New", + "Connecting", + "Connected", + "Disconnected", + "Failed", + "Closed" + }; + qCDebug(networking_webrtc) << "WDCConnection::onPeerConnectionStateChanged() :" << (int)state << STATES[(int)state]; +#endif +} + void WDCConnection::onDataChannelOpened(rtc::scoped_refptr dataChannel) { #ifdef WEBRTC_DEBUG qCDebug(networking_webrtc) << "WDCConnection::onDataChannelOpened() :" @@ -276,16 +294,19 @@ void WDCConnection::onDataChannelOpened(rtc::scoped_refptr void WDCConnection::onDataChannelStateChanged() { auto state = _dataChannel->state(); #ifdef WEBRTC_DEBUG - qCDebug(networking_webrtc) << "WDCConnection::dataChannelStateChanged() :" << (int)state + qCDebug(networking_webrtc) << "WDCConnection::onDataChannelStateChanged() :" << (int)state << DataChannelInterface::DataStateString(state); #endif if (state == DataChannelInterface::kClosed) { - _dataChannel->Close(); + // Close data channel. + _dataChannel->UnregisterObserver(); + _dataChannelObserver = nullptr; _dataChannel = nullptr; - // WEBRTC FIXME: The following line causes the _peerConnectionFactory to fail. - //_peerConnection->Close(); - //_peerConnection = nullptr; - _parent->onDataChannelClosed(this, _dataChannelID); +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "Disposed of data channel"; +#endif + // Close peer connection. + _parent->closePeerConnection(this); } } @@ -320,6 +341,18 @@ bool WDCConnection::sendDataMessage(const DataBuffer& buffer) { return _dataChannel->Send(buffer); } +void WDCConnection::closePeerConnection() { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCConnection::closePeerConnection()"; +#endif + _peerConnection->Close(); + _peerConnection = nullptr; + _peerConnectionObserver = nullptr; +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "Disposed of peer connection"; +#endif +} + WebRTCDataChannels::WebRTCDataChannels(NodeType_t nodeType, QObject* parent) : _nodeType(nodeType), @@ -348,6 +381,9 @@ WebRTCDataChannels::WebRTCDataChannels(NodeType_t nodeType, QObject* parent) : if (!_peerConnectionFactory) { qCWarning(networking_webrtc) << "Failed to create WebRTC peer connection factory"; } + + // Set up mechanism for closing peer connections. + connect(this, &WebRTCDataChannels::closePeerConnectionSoon, this, &WebRTCDataChannels::closePeerConnectionNow); } WebRTCDataChannels::~WebRTCDataChannels() { @@ -384,18 +420,6 @@ void WebRTCDataChannels::onDataChannelOpened(WDCConnection* connection, quint16 _connectionsByDataChannel.insert(dataChannelID, connection); } -void WebRTCDataChannels::onDataChannelClosed(WDCConnection* connection, quint16 dataChannelID) { -#ifdef WEBRTC_DEBUG - qCDebug(networking_webrtc) << "WebRTCDataChannels::onDataChannelClosed() :" << dataChannelID; -#endif - - // Delete WDCConnection. - _connectionsByWebSocket.remove(connection->getWebSocketID()); - _connectionsByDataChannel.remove(dataChannelID); - // WEBRTC FIXME: The following line causes the _peerConnectionFactory to fail. - //delete connection; -} - void WebRTCDataChannels::onSignalingMessage(const QJsonObject& message) { #ifdef WEBRTC_DEBUG qCDebug(networking_webrtc) << "WebRTCDataChannel::onSignalingMessage()" << message; @@ -481,12 +505,42 @@ rtc::scoped_refptr WebRTCDataChannels::createPeerConnec configuration.servers.push_back(iceServer); #ifdef WEBRTC_DEBUG - qCDebug(networking_webrtc) << "2. Create a new PeerConnection"; + qCDebug(networking_webrtc) << "2. Create a new peer connection"; #endif PeerConnectionDependencies dependencies(peerConnectionObserver.get()); auto result = _peerConnectionFactory->CreatePeerConnection(configuration, std::move(dependencies)); +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "Created peer connection"; +#endif return result; } +void WebRTCDataChannels::closePeerConnection(WDCConnection* connection) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WebRTCDataChannels::closePeerConnection()"; +#endif + // Use Qt's signals/slots mechanism to close the peer connection on its own call stack, separate from the DataChannel + // callback that initiated the peer connection. + // https://bugs.chromium.org/p/webrtc/issues/detail?id=3721 + emit closePeerConnectionSoon(connection); +} + + +void WebRTCDataChannels::closePeerConnectionNow(WDCConnection* connection) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WebRTCDataChannels::closePeerConnectionNow()"; +#endif + // Close the peer connection. + connection->closePeerConnection(); + + // Delete the WDCConnection. + _connectionsByWebSocket.remove(connection->getWebSocketID()); + _connectionsByDataChannel.remove(connection->getDataChannelID()); + delete connection; +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "Disposed of connection"; +#endif +} + #endif // WEBRTC_DATA_CHANNELS diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.h b/libraries/networking/src/webrtc/WebRTCDataChannels.h index d9bb213a1df..0b8caf80b5c 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.h +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.h @@ -48,6 +48,7 @@ class WDCSetSessionDescriptionObserver : public SetSessionDescriptionObserver { /// @brief A WebRTC create session description observer. class WDCCreateSessionDescriptionObserver : public CreateSessionDescriptionObserver { public: + WDCCreateSessionDescriptionObserver(WDCConnection* parent); /// @brief The call to CreateAnswer succeeded. @@ -118,6 +119,7 @@ class WDCDataChannelObserver : public DataChannelObserver { class WDCConnection { public: + /// @brief Constructs a new WDCConnection and opens a WebRTC data connection. /// @param webSocketID The signaling channel that initiated the opening of the WebRTC data channel. /// @param parent The parent WebRTCDataChannels object. @@ -155,6 +157,10 @@ class WDCConnection { /// @param candidate The ICE candidate. void sendIceCandidate(const IceCandidateInterface* candidate); + /// @brief Monitors the peer connection state. + /// @param state The new peer connection state. + void onPeerConnectionStateChanged(PeerConnectionInterface::PeerConnectionState state); + /// @brief Handles the WebRTC data channel being opened. /// @param dataChannel The WebRTC data channel. void onDataChannelOpened(rtc::scoped_refptr dataChannel); @@ -171,6 +177,9 @@ class WDCConnection { /// @param buffer The message to send. /// @return `true` if the message was sent, otherwise `false`. bool sendDataMessage(const DataBuffer& buffer); + + /// @brief Closes the WebRTC peer connection. + void closePeerConnection(); private: WebRTCDataChannels* _parent; @@ -228,11 +237,6 @@ class WebRTCDataChannels : public QObject { /// @param dataChannelID The WebRTC data channel ID. void onDataChannelOpened(WDCConnection* connection, quint16 dataChannelID); - /// @brief Handles a WebRTC data channel closing. - /// @param connection The WebRTC data channel connection. - /// @param dataChannelID The WebRTC data channel ID. - void onDataChannelClosed(WDCConnection* connection, quint16 dataChannelID); - /// @brief Emits a signalingMessage to be sent to the Interface client. /// @param message The WebRTC signaling message to send. void sendSignalingMessage(const QJsonObject& message); @@ -254,12 +258,24 @@ class WebRTCDataChannels : public QObject { rtc::scoped_refptr createPeerConnection( const std::shared_ptr peerConnectionObserver); + /// @brief Initiates closing the peer connection for a WebRTC data channel. + /// @details Emits a {@link WebRTCDataChannels.closePeerConnectionSoon} signal which is connected to + /// {@link WebRTCDataChannels.closePeerConnectionNow} in order to close the peer connection on a new call stack. This is + /// necessary to work around a WebRTC library limitation. + /// @param connection The WebRTC data channel connection. + void closePeerConnection(WDCConnection* connection); + public slots: /// @brief Handles a WebRTC signaling message received from the Interface client. /// @param message The WebRTC signaling message. void onSignalingMessage(const QJsonObject& message); + /// @brief Closes the peer connection for a WebRTC data channel. + /// @details Used by {@link WebRTCDataChannels.closePeerConnection}. + /// @param connection The WebRTC data channel connection. + void closePeerConnectionNow(WDCConnection* connection); + signals: /// @brief A WebRTC signaling message to be sent to the Interface client. @@ -273,6 +289,11 @@ public slots: /// @param byteArray The Vircadia protocol message. void dataMessage(int dataChannelID, const QByteArray& byteArray); + /// @brief Signals that the peer connection for a WebRTC data channel should be closed. + /// @details Used by {@link WebRTCDataChannels.closePeerConnection}. + /// @param connection The WebRTC data channel connection. + void closePeerConnectionSoon(WDCConnection* connection); + private: QObject* _parent; From e682336cc205ea4e0fc9b957efd32e3c34539e83 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sun, 4 Jul 2021 15:57:53 +1200 Subject: [PATCH 25/82] Add missing Doxygen --- libraries/networking/src/webrtc/WebRTCDataChannels.h | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.h b/libraries/networking/src/webrtc/WebRTCDataChannels.h index 0b8caf80b5c..3edf9f9b9aa 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.h +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.h @@ -49,6 +49,8 @@ class WDCSetSessionDescriptionObserver : public SetSessionDescriptionObserver { class WDCCreateSessionDescriptionObserver : public CreateSessionDescriptionObserver { public: + /// @brief Constructs a session description observer. + /// @param parent The parent connection object. WDCCreateSessionDescriptionObserver(WDCConnection* parent); /// @brief The call to CreateAnswer succeeded. @@ -67,6 +69,9 @@ class WDCCreateSessionDescriptionObserver : public CreateSessionDescriptionObser /// @brief A WebRTC peer connection observer. class WDCPeerConnectionObserver : public PeerConnectionObserver { public: + + /// @brief Constructs a peer connection observer. + /// @param parent The parent connection object. WDCPeerConnectionObserver(WDCConnection* parent); /// @brief Called when the SignalingState changes. @@ -100,6 +105,9 @@ class WDCPeerConnectionObserver : public PeerConnectionObserver { /// @brief A WebRTC data channel observer. class WDCDataChannelObserver : public DataChannelObserver { public: + + /// @brief Constructs a data channel observer. + /// @param parent The parent connection object. WDCDataChannelObserver(WDCConnection* parent); /// @brief The data channel state changed. From 7ecd9b6b8e83456715e90407667c91f35991e39e Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sun, 4 Jul 2021 16:38:47 +1200 Subject: [PATCH 26/82] Miscellaneous tidying --- .../networking/src/webrtc/WebRTCDataChannels.cpp | 12 +++++++----- libraries/networking/src/webrtc/WebRTCDataChannels.h | 4 ++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp index 48ec1b7b9b8..69c954ad4ba 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp @@ -21,6 +21,7 @@ // - https://webrtc.googlesource.com/src/+/master/api/peer_connection_interface.h const std::string ICE_SERVER_URI = "stun://ice.vircadia.com:7337"; +const int MAX_WEBRTC_BUFFER_SIZE = 16777216; // 16MB // #define WEBRTC_DEBUG @@ -128,9 +129,9 @@ void WDCDataChannelObserver::OnMessage(const DataBuffer& buffer) { } -WDCConnection::WDCConnection(quint16 webSocketID, WebRTCDataChannels* parent) : - _webSocketID(webSocketID), - _parent(parent) +WDCConnection::WDCConnection(WebRTCDataChannels* parent, quint16 webSocketID) : + _parent(parent), + _webSocketID(webSocketID) { #ifdef WEBRTC_DEBUG qCDebug(networking_webrtc) << "WDCConnection::WDCConnection() :" << webSocketID; @@ -333,10 +334,11 @@ bool WDCConnection::sendDataMessage(const DataBuffer& buffer) { #ifdef WEBRTC_DEBUG qCDebug(networking_webrtc) << "WDCConnection::sendDataMessage()"; #endif - const int MAX_WEBRTC_BUFFER_SIZE = 16 * 1024 * 1024; // 16MB if (_dataChannel->buffered_amount() + buffer.size() > MAX_WEBRTC_BUFFER_SIZE) { // Don't send, otherwise the data channel will be closed. return false; + } else { + qCDebug(networking_webrtc) << "WebRTC send buffer overflow"; } return _dataChannel->Send(buffer); } @@ -440,7 +442,7 @@ void WebRTCDataChannels::onSignalingMessage(const QJsonObject& message) { if (_connectionsByWebSocket.contains(from)) { connection = _connectionsByWebSocket.value(from); } else { - connection = new WDCConnection(from, this); + connection = new WDCConnection(this, from); _connectionsByWebSocket.insert(from, connection); } diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.h b/libraries/networking/src/webrtc/WebRTCDataChannels.h index 3edf9f9b9aa..75325781edc 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.h +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.h @@ -129,9 +129,9 @@ class WDCConnection { public: /// @brief Constructs a new WDCConnection and opens a WebRTC data connection. - /// @param webSocketID The signaling channel that initiated the opening of the WebRTC data channel. /// @param parent The parent WebRTCDataChannels object. - WDCConnection(quint16 webSocketID, WebRTCDataChannels* parent); + /// @param webSocketID The signaling channel that initiated the opening of the WebRTC data channel. + WDCConnection(WebRTCDataChannels* parent, quint16 webSocketID); /// @brief Gets the WebSocket ID. /// @return The ID of the WebSocket. From 3ebb75acf53c97845b26b7bb7375fb8f66d7839b Mon Sep 17 00:00:00 2001 From: David Rowe Date: Mon, 5 Jul 2021 10:24:59 +1200 Subject: [PATCH 27/82] Undo documentation improvements --- libraries/networking/src/AddressManager.h | 10 +++++----- tools/doxygen/Doxyfile | 4 ++-- tools/doxygen/README.md | 2 ++ 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/libraries/networking/src/AddressManager.h b/libraries/networking/src/AddressManager.h index 74ec1c42667..e8793edc1dc 100644 --- a/libraries/networking/src/AddressManager.h +++ b/libraries/networking/src/AddressManager.h @@ -252,8 +252,8 @@ public slots: * "127.0.0.1" or "localhost"), a file:/// address, a domain name, a named path * on a domain (starts with "/"), a position or position and orientation, or a user (starts with * "@"). - * @param {boolean} [fromSuggestions=false] - Set to true if the address is obtained from the "Explore" app. - * Helps ensure that the user's location history is correctly maintained. + * @param {boolean} [fromSuggestions=false] - Set to true if the address is obtained from the "Goto" dialog. + * Helps ensure that user's location history is correctly maintained. */ void handleLookupString(const QString& lookupString, bool fromSuggestions = false); @@ -390,10 +390,10 @@ public slots: void lookupResultIsNotFound(); /*@jsdoc - * Triggered when a request is made to go to a URL or IP address. + * Triggered when a request is made to go to an IP address. * @function location.possibleDomainChangeRequired - * @param {string} domainURL - The URL of the domain. - * @param {Uuid} domainID - The UUID of the domain to go to. May be "{@link Uuid|Uuid.NULL} if not yet known. + * @param {Url} domainURL - URL for domain + * @param {Uuid} domainID - The UUID of the domain to go to. * @returns {Signal} */ // No example because this function isn't typically used in scripts. diff --git a/tools/doxygen/Doxyfile b/tools/doxygen/Doxyfile index ee7ac1e2e15..ac8136a0c53 100644 --- a/tools/doxygen/Doxyfile +++ b/tools/doxygen/Doxyfile @@ -509,7 +509,7 @@ EXTRACT_PACKAGE = NO # included in the documentation. # The default value is: NO. -EXTRACT_STATIC = YES +EXTRACT_STATIC = NO # If the EXTRACT_LOCAL_CLASSES tag is set to YES, classes (and structs) defined # locally in source files will be included in the documentation. If set to NO, @@ -794,7 +794,7 @@ CITE_BIB_FILES = # messages are off. # The default value is: NO. -QUIET = YES +QUIET = NO # The WARNINGS tag can be used to turn on/off the warning messages that are # generated to standard error (stderr) by doxygen. If WARNINGS is set to YES diff --git a/tools/doxygen/README.md b/tools/doxygen/README.md index 8b8fdc5c264..658d8814aec 100644 --- a/tools/doxygen/README.md +++ b/tools/doxygen/README.md @@ -7,6 +7,8 @@ **Doxygen** ≥ 1.9.1 - https://www.doxygen.nl/ +Make a `/build/doxygen/` directory. + If you want to run Doxygen from a command prompt, add the Doxygen install's `/bin` directory to your system PATH. From 28b46ed8617093acb16b3f16f54ba4d97a937a90 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Mon, 5 Jul 2021 10:59:19 +1200 Subject: [PATCH 28/82] Log warning if try to set local port on WebRTC LimitedNodeList --- libraries/networking/src/LimitedNodeList.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index 275324b865b..0899d25099c 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -218,6 +218,7 @@ void LimitedNodeList::setSocketLocalPort(SocketType socketType, quint16 socketLo LIMITED_NODELIST_LOCAL_PORT.set(socketLocalPort); } else { // WEBRTC TODO: Add WebRTC equivalent? + qCWarning(networking_webrtc) << "LIMITED_NODELIST_LOCAL_PORT not set for WebRTC socket"; } } } From 4dac6110369e334f678ad860460c72ee7be7a5d8 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Thu, 8 Jul 2021 15:30:51 +1200 Subject: [PATCH 29/82] Fix build error --- libraries/networking/src/webrtc/WebRTCDataChannels.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.h b/libraries/networking/src/webrtc/WebRTCDataChannels.h index 24936bd885c..17e7e525bbd 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.h +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.h @@ -165,7 +165,7 @@ class WDCConnection { /// @brief Monitors the peer connection state. /// @param state The new peer connection state. - void onPeerConnectionStateChanged(PeerConnectionInterface::PeerConnectionState state); + void onPeerConnectionStateChanged(webrtc::PeerConnectionInterface::PeerConnectionState state); /// @brief Handles the WebRTC data channel being opened. /// @param dataChannel The WebRTC data channel. From 837f2e06e12f031ada2107237dad727cbed41577 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Thu, 8 Jul 2021 15:43:31 +1200 Subject: [PATCH 30/82] Update to latest Vircadia Web --- vircadia-web | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vircadia-web b/vircadia-web index ac1f13a39c7..0afaa769d46 160000 --- a/vircadia-web +++ b/vircadia-web @@ -1 +1 @@ -Subproject commit ac1f13a39c702ee54bf2cda8bc35e5d34f7f0756 +Subproject commit 0afaa769d46683d461c9288aa31468f64cba0233 From 320b5a2d22f7476c82c5da1cb33a994bbf27502c Mon Sep 17 00:00:00 2001 From: David Rowe Date: Wed, 21 Jul 2021 19:54:40 +1200 Subject: [PATCH 31/82] Fix HifiSockAddr references --- libraries/networking/src/NodeList.cpp | 2 +- libraries/networking/src/udt/NetworkSocket.cpp | 4 ++-- libraries/networking/src/udt/NetworkSocket.h | 6 +++--- libraries/networking/src/webrtc/WebRTCSignalingServer.h | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index 364bcc5b402..97080a349a7 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -894,7 +894,7 @@ void NodeList::parseNodeFromPacketStream(QDataStream& packetStream) { info.publicSocket.setAddress(_domainHandler.getIP()); } - // WEBRTC TODO: Handle WebRTC-connected nodes. Probably need to include SocketType in HifiSockAddr << and >> + // WEBRTC TODO: Handle WebRTC-connected nodes. Probably need to include SocketType in SockAddr << and >> info.publicSocket.setSocketType(SocketType::UDP); info.localSocket.setSocketType(SocketType::UDP); diff --git a/libraries/networking/src/udt/NetworkSocket.cpp b/libraries/networking/src/udt/NetworkSocket.cpp index 16f847a0681..4fdb4e83ff1 100644 --- a/libraries/networking/src/udt/NetworkSocket.cpp +++ b/libraries/networking/src/udt/NetworkSocket.cpp @@ -125,7 +125,7 @@ qintptr NetworkSocket::socketDescriptor(SocketType socketType) const { } -qint64 NetworkSocket::writeDatagram(const QByteArray& datagram, const HifiSockAddr& sockAddr) { +qint64 NetworkSocket::writeDatagram(const QByteArray& datagram, const SockAddr& sockAddr) { switch (sockAddr.getSocketType()) { case SocketType::UDP: // WEBRTC TODO: The Qt documentation says that the following call shouldn't be used if the UDP socket is connected!!! @@ -189,7 +189,7 @@ qint64 NetworkSocket::pendingDatagramSize() { #endif } -qint64 NetworkSocket::readDatagram(char* data, qint64 maxSize, HifiSockAddr* sockAddr) { +qint64 NetworkSocket::readDatagram(char* data, qint64 maxSize, SockAddr* sockAddr) { #if defined(WEBRTC_DATA_CHANNELS) // Read per preceding pendingDatagramSize() if any, otherwise alternate socket types. if (_pendingDatagramSizeSocketType == SocketType::UDP diff --git a/libraries/networking/src/udt/NetworkSocket.h b/libraries/networking/src/udt/NetworkSocket.h index dfd9fe5e728..64805282ec4 100644 --- a/libraries/networking/src/udt/NetworkSocket.h +++ b/libraries/networking/src/udt/NetworkSocket.h @@ -14,7 +14,7 @@ #include -#include "../HifiSockAddr.h" +#include "../SockAddr.h" #include "../NodeType.h" #include "../SocketType.h" #if defined(WEBRTC_DATA_CHANNELS) @@ -76,7 +76,7 @@ class NetworkSocket : public QObject { /// @param datagram The datagram to send. /// @param sockAddr The address to send to. /// @return The number of bytes if successfully sent, otherwise -1. - qint64 writeDatagram(const QByteArray& datagram, const HifiSockAddr& sockAddr); + qint64 writeDatagram(const QByteArray& datagram, const SockAddr& sockAddr); /// @brief Gets the number of bytes waiting to be written. /// @details For UDP, there's a single buffer used for all destinations. For WebRTC, each destination has its own buffer. @@ -100,7 +100,7 @@ class NetworkSocket : public QObject { /// @param maxSize The maximum number of bytes to read. /// @param sockAddr The destination to write the source network address into. /// @return The number of bytes if successfully read, otherwise -1. - qint64 readDatagram(char* data, qint64 maxSize, HifiSockAddr* sockAddr = nullptr); + qint64 readDatagram(char* data, qint64 maxSize, SockAddr* sockAddr = nullptr); /// @brief Gets the state of the UDP or WebRTC socket. diff --git a/libraries/networking/src/webrtc/WebRTCSignalingServer.h b/libraries/networking/src/webrtc/WebRTCSignalingServer.h index e32133dd17c..418becd8ebf 100644 --- a/libraries/networking/src/webrtc/WebRTCSignalingServer.h +++ b/libraries/networking/src/webrtc/WebRTCSignalingServer.h @@ -17,7 +17,7 @@ #include #include -#include "../HifiSockAddr.h" +#include "../SockAddr.h" /// @addtogroup Networking /// @{ From 38f7506c5e7993559d1c2cfb1bacc9cf068dc55f Mon Sep 17 00:00:00 2001 From: David Rowe Date: Wed, 21 Jul 2021 20:06:18 +1200 Subject: [PATCH 32/82] Fix erroneous domain server log message --- libraries/networking/src/webrtc/WebRTCDataChannels.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp index 2f5dde686e6..265b28f126f 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp @@ -345,9 +345,8 @@ bool WDCConnection::sendDataMessage(const DataBuffer& buffer) { #endif if (_dataChannel->buffered_amount() + buffer.size() > MAX_WEBRTC_BUFFER_SIZE) { // Don't send, otherwise the data channel will be closed. - return false; - } else { qCDebug(networking_webrtc) << "WebRTC send buffer overflow"; + return false; } return _dataChannel->Send(buffer); } From ed36321e25d8f6892d36c366b49cec31dd1e0536 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Fri, 30 Jul 2021 14:12:38 +1200 Subject: [PATCH 33/82] Fill in connected client's ports with WebRTC data channel IDs --- domain-server/src/NodeConnectionData.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/domain-server/src/NodeConnectionData.cpp b/domain-server/src/NodeConnectionData.cpp index 1ef8ebf6a9c..60a8d6878ce 100644 --- a/domain-server/src/NodeConnectionData.cpp +++ b/domain-server/src/NodeConnectionData.cpp @@ -55,6 +55,21 @@ NodeConnectionData NodeConnectionData::fromDataStream(QDataStream& dataStream, c >> newHeader.publicSockAddr >> newHeader.localSockAddr >> newHeader.interestList >> newHeader.placeName; + // A WebRTC web client doesn't necessarily know it's public Internet or local network addresses, and for WebRTC they aren't + // needed: for WebRTC, the data channel ID is the important thing. The client's public Internet IP address still needs to + // be known for domain access permissions, though, and this can be obtained from the WebSocket signaling connection. + if (senderSockAddr.getSocketType() == SocketType::WebRTC) { + // WEBRTC TODO: Rather than setting the SocketType here, serialize and deserialize the SocketType in the leading byte of + // the 5 bytes used to encode the IP address. + newHeader.publicSockAddr.setSocketType(SocketType::WebRTC); + newHeader.localSockAddr.setSocketType(SocketType::WebRTC); + + // WEBRTC TODO: Set the public Internet address obtained from the WebSocket used in WebRTC signaling. + + newHeader.publicSockAddr.setPort(senderSockAddr.getPort()); // We don't know whether it's a public or local connection + newHeader.localSockAddr.setPort(senderSockAddr.getPort()); // so set both ports. + } + newHeader.senderSockAddr = senderSockAddr; if (newHeader.publicSockAddr.getAddress().isNull()) { From fa01ead130e47d0adde292fed7d524fe9a068eb8 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Fri, 30 Jul 2021 14:13:15 +1200 Subject: [PATCH 34/82] Add extra WebRTC debug --- libraries/networking/src/webrtc/WebRTCDataChannels.cpp | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp index 265b28f126f..1fd9ae42e5f 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp @@ -291,6 +291,9 @@ void WDCConnection::onDataChannelOpened(rtc::scoped_refptr _dataChannelID = _parent->getNewDataChannelID(); // Not dataChannel->id() because it's only unique per peer connection. _dataChannel->RegisterObserver(_dataChannelObserver.get()); +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WDCConnection::onDataChannelOpened() : channel ID:" << _dataChannelID; +#endif _parent->onDataChannelOpened(this, _dataChannelID); } @@ -486,7 +489,8 @@ void WebRTCDataChannels::sendSignalingMessage(const QJsonObject& message) { void WebRTCDataChannels::emitDataMessage(int dataChannelID, const QByteArray& byteArray) { #ifdef WEBRTC_DEBUG - qCDebug(networking_webrtc) << "WebRTCDataChannels::emitDataMessage() :" << dataChannelID << byteArray; + qCDebug(networking_webrtc) << "WebRTCDataChannels::emitDataMessage() :" << dataChannelID << byteArray.toHex() + << byteArray.length(); #endif emit dataMessage(dataChannelID, byteArray); } @@ -557,6 +561,9 @@ void WebRTCDataChannels::closePeerConnectionNow(WDCConnection* connection) { connection->closePeerConnection(); // Delete the WDCConnection. +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "Dispose of connection for channel ID:" << connection->getDataChannelID(); +#endif _connectionsByWebSocket.remove(connection->getWebSocketID()); _connectionsByDataChannel.remove(connection->getDataChannelID()); delete connection; From 3603520c946a5165878079d6eb60a57766cf2527 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Fri, 30 Jul 2021 14:13:44 +1200 Subject: [PATCH 35/82] fillPacketHeader() can be private --- libraries/networking/src/LimitedNodeList.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/LimitedNodeList.h b/libraries/networking/src/LimitedNodeList.h index 98494aa00a0..86c9d90a3ba 100644 --- a/libraries/networking/src/LimitedNodeList.h +++ b/libraries/networking/src/LimitedNodeList.h @@ -420,7 +420,6 @@ protected slots: qint64 sendPacket(std::unique_ptr packet, const Node& destinationNode, const SockAddr& overridenSockAddr); - void fillPacketHeader(const NLPacket& packet, HMACAuth* hmacAuth = nullptr); void setLocalSocket(const SockAddr& sockAddr); @@ -487,6 +486,8 @@ private slots: void addSTUNHandlerToUnfiltered(); // called once STUN socket known private: + void fillPacketHeader(const NLPacket& packet, HMACAuth* hmacAuth = nullptr); + mutable QReadWriteLock _sessionUUIDLock; QUuid _sessionUUID; using LocalIDMapping = tbb::concurrent_unordered_map; From 0ea6d88bf5982525c7aacff2093dd12cb621e68c Mon Sep 17 00:00:00 2001 From: David Rowe Date: Fri, 30 Jul 2021 14:14:36 +1200 Subject: [PATCH 36/82] Fix function parameter ID --- libraries/networking/src/LimitedNodeList.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/LimitedNodeList.h b/libraries/networking/src/LimitedNodeList.h index 86c9d90a3ba..9e6458225fc 100644 --- a/libraries/networking/src/LimitedNodeList.h +++ b/libraries/networking/src/LimitedNodeList.h @@ -121,7 +121,7 @@ class LimitedNodeList : public QObject, public Dependency { QUuid getSessionUUID() const; void setSessionUUID(const QUuid& sessionUUID); Node::LocalID getSessionLocalID() const; - void setSessionLocalID(Node::LocalID localID); + void setSessionLocalID(Node::LocalID sessionLocalID); void setPermissions(const NodePermissions& newPermissions); bool isAllowedEditor() const { return _permissions.can(NodePermissions::Permission::canAdjustLocks); } From 85284228343cfc4334ca89469abf189d44077399 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Fri, 30 Jul 2021 14:18:34 +1200 Subject: [PATCH 37/82] Fix WebRTC Doxygen typos --- libraries/networking/src/webrtc/WebRTCDataChannels.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.h b/libraries/networking/src/webrtc/WebRTCDataChannels.h index 17e7e525bbd..ec6f67b3c7a 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.h +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.h @@ -52,10 +52,10 @@ class WDCCreateSessionDescriptionObserver : public webrtc::CreateSessionDescript WDCCreateSessionDescriptionObserver(WDCConnection* parent); /// @brief The call to CreateAnswer succeeded. - /// @param The session description. + /// @param desc The session description. void OnSuccess(webrtc::SessionDescriptionInterface* desc) override; - //@ @brief The call to CreateAnswer failed. + /// @brief The call to CreateAnswer failed. /// @param error Error information. void OnFailure(webrtc::RTCError error) override; From 9be1c08c8cd714e3e3fc523bb3e60bd07ca9de09 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sat, 31 Jul 2021 23:03:48 +1200 Subject: [PATCH 38/82] Fix domain server crash --- libraries/networking/src/webrtc/WebRTCDataChannels.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp index 1fd9ae42e5f..974cef29974 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp @@ -339,14 +339,20 @@ qint64 WDCConnection::getBufferedAmount() const { #ifdef WEBRTC_DEBUG qCDebug(networking_webrtc) << "WDCConnection::getBufferedAmount()"; #endif - return _dataChannel->buffered_amount(); + return _dataChannel ? _dataChannel->buffered_amount() : 0; } bool WDCConnection::sendDataMessage(const DataBuffer& buffer) { #ifdef WEBRTC_DEBUG qCDebug(networking_webrtc) << "WDCConnection::sendDataMessage()"; + if (!_dataChannel) { + qCDebug(networking_webrtc) << "No data channel to send on"; + } #endif - if (_dataChannel->buffered_amount() + buffer.size() > MAX_WEBRTC_BUFFER_SIZE) { + if (!_dataChannel) { + // Data channel may have been closed while message to send was being prepared. + return false; + } else if (_dataChannel->buffered_amount() + buffer.size() > MAX_WEBRTC_BUFFER_SIZE) { // Don't send, otherwise the data channel will be closed. qCDebug(networking_webrtc) << "WebRTC send buffer overflow"; return false; From 500da963ade0c62bed25c52ab85d73f6ef51110c Mon Sep 17 00:00:00 2001 From: David Rowe Date: Wed, 4 Aug 2021 16:11:04 +1200 Subject: [PATCH 39/82] Make parameter names consistent across usages --- libraries/networking/src/DomainHandler.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libraries/networking/src/DomainHandler.h b/libraries/networking/src/DomainHandler.h index 954fd35a67a..c76cadeb3e9 100644 --- a/libraries/networking/src/DomainHandler.h +++ b/libraries/networking/src/DomainHandler.h @@ -242,7 +242,7 @@ class DomainHandler : public QObject { }; public slots: - void setURLAndID(QUrl domainURL, QUuid id); + void setURLAndID(QUrl domainURL, QUuid domainID); void setIceServerHostnameAndID(const QString& iceServerHostname, const QUuid& id); void processSettingsPacketList(QSharedPointer packetList); @@ -252,7 +252,7 @@ public slots: void processDomainServerConnectionDeniedPacket(QSharedPointer message); // sets domain handler in error state. - void setRedirectErrorState(QUrl errorUrl, QString reasonMessage = "", int reason = -1, const QString& extraInfo = ""); + void setRedirectErrorState(QUrl errorUrl, QString reasonMessage = "", int reasonCode = -1, const QString& extraInfo = ""); bool isInErrorState() { return _isInErrorState; } @@ -278,7 +278,7 @@ private slots: void settingsReceived(const QJsonObject& domainSettingsObject); void settingsReceiveFail(); - void domainConnectionRefused(QString reasonMessage, int reason, const QString& extraInfo); + void domainConnectionRefused(QString reasonMessage, int reasonCode, const QString& extraInfo); void redirectToErrorDomainURL(QUrl errorDomainURL); void redirectErrorStateChanged(bool isInErrorState); From 9b2c77380599bb1619b996b4b3a3da423917ceeb Mon Sep 17 00:00:00 2001 From: David Rowe Date: Tue, 17 Aug 2021 22:37:43 +1200 Subject: [PATCH 40/82] Move WebRTCSignalingServer into Domain Server --- domain-server/src/DomainServer.cpp | 40 +++++++++++++++++++ domain-server/src/DomainServer.h | 16 ++++++++ ice-server/src/IceServer.cpp | 1 + .../src/BaseAssetScriptingInterface.cpp | 3 +- libraries/networking/src/LimitedNodeList.cpp | 8 +++- libraries/networking/src/LimitedNodeList.h | 5 ++- .../networking/src/udt/NetworkSocket.cpp | 7 ++++ libraries/networking/src/udt/NetworkSocket.h | 11 ++++- libraries/networking/src/udt/Socket.cpp | 6 +++ libraries/networking/src/udt/Socket.h | 4 ++ .../networking/src/webrtc/WebRTCSocket.cpp | 13 +++--- .../networking/src/webrtc/WebRTCSocket.h | 25 ++++++++---- 12 files changed, 119 insertions(+), 20 deletions(-) diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index c49f1758664..341c658ce7d 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -167,6 +167,10 @@ DomainServer::DomainServer(int argc, char* argv[]) : _gatekeeper(this), _httpManager(QHostAddress::AnyIPv4, DOMAIN_SERVER_HTTP_PORT, QString("%1/resources/web/").arg(QCoreApplication::applicationDirPath()), this) +#if defined(WEBRTC_DATA_CHANNELS) + , + _webrtcSignalingServer(this) +#endif { if (_parentPID != -1) { watchParentProcess(_parentPID); @@ -248,6 +252,10 @@ DomainServer::DomainServer(int argc, char* argv[]) : updateDownstreamNodes(); updateUpstreamNodes(); +#if defined(WEBRTC_DATA_CHANNELS) + setUpWebRTCSignalingServer(); +#endif + if (_type != NonMetaverse) { // if we have a metaverse domain, we'll use an access token for API calls resetAccountManagerAccessToken(); @@ -846,6 +854,38 @@ void DomainServer::setupNodeListAndAssignments() { addStaticAssignmentsToQueue(); } + +#if defined(WEBRTC_DATA_CHANNELS) + +void DomainServer::setUpWebRTCSignalingServer() { + // Bind the WebRTC signaling server's WebSocket to its port. + bool isBound = _webrtcSignalingServer.bind(QHostAddress::AnyIPv4, DEFAULT_DOMAIN_SERVER_WS_PORT); + if (!isBound) { + qWarning() << "WebRTC signaling server not bound to port. WebRTC connections are not supported."; + return; + } + + auto limitedNodeList = DependencyManager::get(); + + // Route inbound WebRTC signaling messages received from user clients. + connect(&_webrtcSignalingServer, &WebRTCSignalingServer::messageReceived, + this, &DomainServer::routeWebRTCSignalingMessage); + + // Route domain server signaling messages. + auto webrtcSocket = limitedNodeList->getWebRTCSocket(); + connect(this, &DomainServer::webrtcSignalingMessageForDomainServer, webrtcSocket, &WebRTCSocket::onSignalingMessage); + connect(webrtcSocket, &WebRTCSocket::sendSignalingMessage, &_webrtcSignalingServer, &WebRTCSignalingServer::sendMessage); +} + +void DomainServer::routeWebRTCSignalingMessage(const QJsonObject& json) { + if (json.value("to").toString() == NodeType::DomainServer) { + emit webrtcSignalingMessageForDomainServer(json); + } +} + +#endif + + bool DomainServer::resetAccountManagerAccessToken() { if (!_oauthProviderURL.isEmpty()) { // check for an access-token in our settings, can optionally be overidden by env value diff --git a/domain-server/src/DomainServer.h b/domain-server/src/DomainServer.h index bfcc867630a..3f509b232dd 100644 --- a/domain-server/src/DomainServer.h +++ b/domain-server/src/DomainServer.h @@ -27,6 +27,8 @@ #include #include #include +#include +#include #include "AssetsBackupHandler.h" #include "DomainGatekeeper.h" @@ -155,6 +157,11 @@ private slots: void userConnected(); void userDisconnected(); +#if defined(WEBRTC_DATA_CHANNELS) + void webrtcSignalingMessageForDomainServer(const QJsonObject& json); +#endif + + private: QUuid getID(); @@ -235,6 +242,11 @@ private slots: std::initializer_list optionalData = { }, bool requireAccessToken = true); +#if defined(WEBRTC_DATA_CHANNELS) + void setUpWebRTCSignalingServer(); + void routeWebRTCSignalingMessage(const QJsonObject& json); +#endif + QString operationToString(const QNetworkAccessManager::Operation &op); SubnetList _acSubnetWhitelist; @@ -312,6 +324,10 @@ private slots: std::unordered_map> _pendingContentFiles; QThread _assetClientThread; + +#if defined(WEBRTC_DATA_CHANNELS) + WebRTCSignalingServer _webrtcSignalingServer; +#endif }; diff --git a/ice-server/src/IceServer.cpp b/ice-server/src/IceServer.cpp index 6a6290f7b26..b4542e12a0c 100644 --- a/ice-server/src/IceServer.cpp +++ b/ice-server/src/IceServer.cpp @@ -16,6 +16,7 @@ #include #include +#include #include #include #include diff --git a/libraries/networking/src/BaseAssetScriptingInterface.cpp b/libraries/networking/src/BaseAssetScriptingInterface.cpp index d7d14496baf..f9dd12b279a 100644 --- a/libraries/networking/src/BaseAssetScriptingInterface.cpp +++ b/libraries/networking/src/BaseAssetScriptingInterface.cpp @@ -9,8 +9,9 @@ #include "BaseAssetScriptingInterface.h" -#include #include +#include +#include #include #include diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index 32e63d4b9aa..e7423633738 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -74,7 +74,7 @@ LimitedNodeList::LimitedNodeList(char ownerType, int socketListenPort, int dtlsL qCDebug(networking) << "NodeList DTLS socket is listening on" << _dtlsSocket->localPort(); } - _nodeSocket.bind(SocketType::WebRTC, QHostAddress::AnyIPv4, DEFAULT_DOMAIN_SERVER_WS_PORT); + _nodeSocket.bind(SocketType::WebRTC, QHostAddress::AnyIPv4); // check for local socket updates every so often const int LOCAL_SOCKET_UPDATE_INTERVAL_MSECS = 5 * 1000; @@ -241,6 +241,12 @@ QUdpSocket& LimitedNodeList::getDTLSSocket() { return *_dtlsSocket; } +#if defined(WEBRTC_DATA_CHANNELS) +const WebRTCSocket* LimitedNodeList::getWebRTCSocket() { + return _nodeSocket.getWebRTCSocket(); +} +#endif + bool LimitedNodeList::isPacketVerifiedWithSource(const udt::Packet& packet, Node* sourceNode) { // We track bandwidth when doing packet verification to avoid needing to do a node lookup // later when we already do it in packetSourceAndHashMatchAndTrackBandwidth. A node lookup diff --git a/libraries/networking/src/LimitedNodeList.h b/libraries/networking/src/LimitedNodeList.h index 9e6458225fc..81d93bf9357 100644 --- a/libraries/networking/src/LimitedNodeList.h +++ b/libraries/networking/src/LimitedNodeList.h @@ -37,7 +37,6 @@ #include #include -#include "DomainHandler.h" #include "NetworkingConstants.h" #include "Node.h" #include "NLPacket.h" @@ -139,6 +138,10 @@ class LimitedNodeList : public QObject, public Dependency { Q_INVOKABLE void setSocketLocalPort(SocketType socketType, quint16 socketLocalPort); QUdpSocket& getDTLSSocket(); +#if defined(WEBRTC_DATA_CHANNELS) + const WebRTCSocket* getWebRTCSocket(); +#endif + PacketReceiver& getPacketReceiver() { return *_packetReceiver; } diff --git a/libraries/networking/src/udt/NetworkSocket.cpp b/libraries/networking/src/udt/NetworkSocket.cpp index 4fdb4e83ff1..c882f6dbe39 100644 --- a/libraries/networking/src/udt/NetworkSocket.cpp +++ b/libraries/networking/src/udt/NetworkSocket.cpp @@ -267,6 +267,13 @@ QString NetworkSocket::errorString(SocketType socketType) const { } +#if defined(WEBRTC_DATA_CHANNELS) +const WebRTCSocket* NetworkSocket::getWebRTCSocket() { + return &_webrtcSocket; +} +#endif + + void NetworkSocket::onUDPStateChanged(QAbstractSocket::SocketState socketState) { emit stateChanged(SocketType::UDP, socketState); } diff --git a/libraries/networking/src/udt/NetworkSocket.h b/libraries/networking/src/udt/NetworkSocket.h index 64805282ec4..d5891cbde71 100644 --- a/libraries/networking/src/udt/NetworkSocket.h +++ b/libraries/networking/src/udt/NetworkSocket.h @@ -91,7 +91,7 @@ class NetworkSocket : public QObject { bool hasPendingDatagrams() const; /// @brief Gets the size of the next pending datagram, alternating between socket types if both have datagrams to read. - /// @return The size of the next pendign datagram. + /// @return The size of the next pending datagram. qint64 pendingDatagramSize(); /// @brief Reads the next datagram per the most recent pendingDatagramSize call if made, otherwise alternating between @@ -111,7 +111,7 @@ class NetworkSocket : public QObject { /// @brief Gets the type of error that last occurred. /// @param socketType The type of socket for which to get the last error. - /// @return The type of error that last occurred + /// @return The type of error that last occurred. QAbstractSocket::SocketError error(SocketType socketType) const; /// @brief Gets the description of the error that last occurred. @@ -119,6 +119,13 @@ class NetworkSocket : public QObject { /// @return The description of the error that last occurred. QString errorString(SocketType socketType) const; + +#if defined(WEBRTC_DATA_CHANNELS) + /// @brief @brief Gets a pointer to the WebRTC socket object. + /// @return A pointer to the WebRTC socket object. + const WebRTCSocket* getWebRTCSocket(); +#endif + signals: /// @brief Emitted each time new data becomes available for reading. diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 8cac4b44c65..810948d742a 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -91,6 +91,12 @@ void Socket::rebind(SocketType socketType, quint16 localPort) { bind(socketType, QHostAddress::AnyIPv4, localPort); } +#if defined(WEBRTC_DATA_CHANNELS) +const WebRTCSocket* Socket::getWebRTCSocket() { + return _networkSocket.getWebRTCSocket(); +} +#endif + void Socket::setSystemBufferSizes(SocketType socketType) { for (int i = 0; i < 2; i++) { QAbstractSocket::SocketOption bufferOpt; diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index b92f3fe48d4..5c93d966769 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -90,6 +90,10 @@ class Socket : public QObject { StatsVector sampleStatsForAllConnections(); +#if defined(WEBRTC_DATA_CHANNELS) + const WebRTCSocket* getWebRTCSocket(); +#endif + #if (PR_BUILD || DEV_BUILD) void sendFakedHandshakeRequest(const SockAddr& sockAddr); #endif diff --git a/libraries/networking/src/webrtc/WebRTCSocket.cpp b/libraries/networking/src/webrtc/WebRTCSocket.cpp index b9eee027a17..2247ad86159 100644 --- a/libraries/networking/src/webrtc/WebRTCSocket.cpp +++ b/libraries/networking/src/webrtc/WebRTCSocket.cpp @@ -10,20 +10,19 @@ #if defined(WEBRTC_DATA_CHANNELS) +#include + #include "../NetworkLogging.h" #include "../udt/Constants.h" WebRTCSocket::WebRTCSocket(QObject* parent, NodeType_t nodeType) : QObject(parent), - _signalingServer(this /*, QHostAddress::AnyIPv4, DEFAULT_DOMAIN_SERVER_WS_PORT*/), _dataChannels(this, nodeType) { - // Connect WebRTC signaling server and data channels. - connect(&_signalingServer, &WebRTCSignalingServer::messageReceived, - &_dataChannels, &WebRTCDataChannels::onSignalingMessage); - connect(&_dataChannels, &WebRTCDataChannels::signalingMessage, - &_signalingServer, &WebRTCSignalingServer::sendMessage); + // Route signaling messages. + connect(this, &WebRTCSocket::onSignalingMessage, &_dataChannels, &WebRTCDataChannels::onSignalingMessage); + connect(&_dataChannels, &WebRTCDataChannels::signalingMessage, this, &WebRTCSocket::sendSignalingMessage); // Route received data channel messages. connect(&_dataChannels, &WebRTCDataChannels::dataMessage, this, &WebRTCSocket::onDataChannelReceivedMessage); @@ -63,7 +62,7 @@ QVariant WebRTCSocket::socketOption(QAbstractSocket::SocketOption option) { bool WebRTCSocket::bind(const QHostAddress& address, quint16 port, QAbstractSocket::BindMode mode) { // WebRTC data channels aren't bound to ports so just treat this as a successful operation. auto wasBound = _isBound; - _isBound = _signalingServer.bind(address, port); + _isBound = true; if (_isBound != wasBound) { emit stateChanged(_isBound ? QAbstractSocket::BoundState : QAbstractSocket::UnconnectedState); } diff --git a/libraries/networking/src/webrtc/WebRTCSocket.h b/libraries/networking/src/webrtc/WebRTCSocket.h index 8d5d5fc3476..9a1d6442d7f 100644 --- a/libraries/networking/src/webrtc/WebRTCSocket.h +++ b/libraries/networking/src/webrtc/WebRTCSocket.h @@ -18,7 +18,6 @@ #include #include "WebRTCDataChannels.h" -#include "WebRTCSignalingServer.h" /// @addtogroup Networking /// @{ @@ -52,13 +51,14 @@ class WebRTCSocket : public QObject { /// @return The value of the socket option. QVariant socketOption(QAbstractSocket::SocketOption option); - /// @brief Binds the WebRTC socket's signaling server to an address and port. - /// @details Note: WebRTC data connections aren't bound to an address or port. Their ports are negotiated as part of the + /// @brief Nominally binds the WebRTC socket to an address and port. + /// @details WebRTC data connections aren't actually bound to an address or port. Their ports are negotiated as part of the /// WebRTC peer connection process. - /// @param address The address to use for the signaling server. - /// @param port The port to use for the signaling server. - /// @param mode The bind mode. (Not used: included for compatibility with the QUdpSocket interface.) - /// @return true if the signaling server was successfully bound, false if it wasn't. + /// Included for compatibility with the QUdpSocket interface. + /// @param address The address. + /// @param port The port. + /// @param mode The bind mode. + /// @return true. bool bind(const QHostAddress& address, quint16 port = 0, QAbstractSocket::BindMode mode = QAbstractSocket::DefaultForPlatform); @@ -132,17 +132,26 @@ public slots: signals: /// @brief Emitted when the state of the socket changes. + /// @param socketState The new state of the socket. void stateChanged(QAbstractSocket::SocketState socketState); /// @brief Emitted each time new data becomes available for reading. void readyRead(); + /// @brief Emitted when a WebRTC signaling message has been received from the signaling server for this WebRTCSocket. + /// @param json The signaling message. + void onSignalingMessage(const QJsonObject& json); + + /// @brief Emitted when there's a WebRTC signaling message to send via the signaling server. + /// @param json The signaling message. + void sendSignalingMessage(const QJsonObject& message); + + private: void setError(QAbstractSocket::SocketError errorType, QString errorString); void clearError(); - WebRTCSignalingServer _signalingServer; WebRTCDataChannels _dataChannels; bool _isBound { false }; From a3c1d504784f1b5fbda1b32a8ab2afb1bfd84940 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Wed, 18 Aug 2021 20:10:11 +1200 Subject: [PATCH 41/82] WebRTC signaling with assignment clients via domain server --- assignment-client/src/AssignmentClient.cpp | 54 ++++++++++++++++++++ assignment-client/src/AssignmentClient.h | 19 +++++-- domain-server/src/DomainServer.cpp | 33 ++++++++++++ domain-server/src/DomainServer.h | 6 +++ libraries/networking/src/Node.cpp | 19 +++++++ libraries/networking/src/NodeType.h | 1 + libraries/networking/src/udt/PacketHeaders.h | 3 +- 7 files changed, 129 insertions(+), 6 deletions(-) diff --git a/assignment-client/src/AssignmentClient.cpp b/assignment-client/src/AssignmentClient.cpp index d1c3efc4750..cfe940133f0 100644 --- a/assignment-client/src/AssignmentClient.cpp +++ b/assignment-client/src/AssignmentClient.cpp @@ -14,6 +14,7 @@ #include +#include #include #include #include @@ -125,6 +126,18 @@ AssignmentClient::AssignmentClient(Assignment::Type requestAssignmentType, QStri PacketReceiver::makeUnsourcedListenerReference(this, &AssignmentClient::handleCreateAssignmentPacket)); packetReceiver.registerListener(PacketType::StopNode, PacketReceiver::makeUnsourcedListenerReference(this, &AssignmentClient::handleStopNodePacket)); + +#if defined(WEBRTC_DATA_CHANNELS) + auto webrtcSocket = nodeList->getWebRTCSocket(); + + // Route inbound WebRTC signaling messages from the Domain Server. + packetReceiver.registerListener(PacketType::WebRTCSignaling, + PacketReceiver::makeUnsourcedListenerReference(this, &AssignmentClient::handleWebRTCSignalingPacket)); + connect(this, &AssignmentClient::webrtcSignalingMessageFromUserClient, webrtcSocket, &WebRTCSocket::onSignalingMessage); + + // Route outbound WebRTC signaling messages via the Domain Server to the user client. + connect(webrtcSocket, &WebRTCSocket::sendSignalingMessage, this, &AssignmentClient::sendSignalingMessageToUserClient); +#endif } void AssignmentClient::stopAssignmentClient() { @@ -333,3 +346,44 @@ void AssignmentClient::assignmentCompleted() { _isAssigned = false; } + +#if defined(WEBRTC_DATA_CHANNELS) + +void AssignmentClient::handleWebRTCSignalingPacket(QSharedPointer message) { + auto messageString = message->readString(); + auto json = QJsonDocument::fromJson(messageString.toUtf8()).object(); + if (json.keys().contains("echo")) { + // Echo message back to sender. + + if (!json.keys().contains("to") || !json.keys().contains("from")) { + return; + } + + // Swap to/from. + auto to = json.value("to"); + json.insert("to", json.value("from")); + json.insert("from", to); + + // Send back to sender via the Domain Server. + auto packetList = NLPacketList::create(PacketType::WebRTCSignaling, QByteArray(), true, true); + packetList->writeString(QJsonDocument(json).toJson(QJsonDocument::Compact)); + auto nodeList = DependencyManager::get(); + auto domainServerAddress = nodeList->getDomainHandler().getSockAddr(); + nodeList->sendPacketList(std::move(packetList), domainServerAddress); + + } else { + // WebRTC signaling message. + emit webrtcSignalingMessageFromUserClient(json); + } +} + +// Sends a signaling message from the assignment client to the user client via the Domain Server. +void AssignmentClient::sendSignalingMessageToUserClient(const QJsonObject& json) { + auto packetList = NLPacketList::create(PacketType::WebRTCSignaling, QByteArray(), true, true); + packetList->writeString(QJsonDocument(json).toJson(QJsonDocument::Compact)); + auto nodeList = DependencyManager::get(); + auto domainServerAddress = nodeList->getDomainHandler().getSockAddr(); + nodeList->sendPacketList(std::move(packetList), domainServerAddress); +} + +#endif diff --git a/assignment-client/src/AssignmentClient.h b/assignment-client/src/AssignmentClient.h index c70baf11fdf..58e0e8cda11 100644 --- a/assignment-client/src/AssignmentClient.h +++ b/assignment-client/src/AssignmentClient.h @@ -16,6 +16,8 @@ #include #include +#include + #include "ThreadedAssignment.h" class QSharedMemory; @@ -29,19 +31,26 @@ class AssignmentClient : public QObject { quint16 assignmentMonitorPort); ~AssignmentClient(); +public slots: + void aboutToQuit(); + private slots: void sendAssignmentRequest(); void assignmentCompleted(); void handleAuthenticationRequest(); void sendStatusPacketToACM(); void stopAssignmentClient(); - -public slots: - void aboutToQuit(); - -private slots: void handleCreateAssignmentPacket(QSharedPointer message); void handleStopNodePacket(QSharedPointer message); +#if defined(WEBRTC_DATA_CHANNELS) + void handleWebRTCSignalingPacket(QSharedPointer message); + void sendSignalingMessageToUserClient(const QJsonObject& json); +#endif + +signals: +#if defined(WEBRTC_DATA_CHANNELS) + void webrtcSignalingMessageFromUserClient(const QJsonObject& json); +#endif private: void setUpStatusToMonitor(); diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 341c658ce7d..185250e8099 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -857,6 +857,7 @@ void DomainServer::setupNodeListAndAssignments() { #if defined(WEBRTC_DATA_CHANNELS) +// Sets up the WebRTC signaling server that's hosted by the domain server. void DomainServer::setUpWebRTCSignalingServer() { // Bind the WebRTC signaling server's WebSocket to its port. bool isBound = _webrtcSignalingServer.bind(QHostAddress::AnyIPv4, DEFAULT_DOMAIN_SERVER_WS_PORT); @@ -875,14 +876,46 @@ void DomainServer::setUpWebRTCSignalingServer() { auto webrtcSocket = limitedNodeList->getWebRTCSocket(); connect(this, &DomainServer::webrtcSignalingMessageForDomainServer, webrtcSocket, &WebRTCSocket::onSignalingMessage); connect(webrtcSocket, &WebRTCSocket::sendSignalingMessage, &_webrtcSignalingServer, &WebRTCSignalingServer::sendMessage); + + // Forward signaling messages received from assignment clients to user client. + PacketReceiver& packetReceiver = limitedNodeList->getPacketReceiver(); + packetReceiver.registerListener(PacketType::WebRTCSignaling, + PacketReceiver::makeUnsourcedListenerReference(this, + &DomainServer::forwardAssignmentClientSignalingMessageToUserClient)); + connect(this, &DomainServer::webrtcSignalingMessageForUserClient, + &_webrtcSignalingServer, &WebRTCSignalingServer::sendMessage); } +// Routes an inbound WebRTC signaling message received from a client app to the appropriate recipient. void DomainServer::routeWebRTCSignalingMessage(const QJsonObject& json) { if (json.value("to").toString() == NodeType::DomainServer) { emit webrtcSignalingMessageForDomainServer(json); + } else { + sendWebRTCSignalingMessageToAssignmentClient(json); } } +// Sends a WebRTC signaling message to the target AC contained in the message. +void DomainServer::sendWebRTCSignalingMessageToAssignmentClient(const QJsonObject& json) { + NodeType_t destinationNodeType = NodeType::fromChar(json.value("to").toString().at(0)); + auto limitedNodeList = DependencyManager::get(); + auto destinationNode = limitedNodeList->soloNodeOfType(destinationNodeType); + if (!destinationNode) { + return; + } + // Use an NLPacketList because the signaling message is not necessarily small. + auto packetList = NLPacketList::create(PacketType::WebRTCSignaling, QByteArray(), true, true); + packetList->writeString(QJsonDocument(json).toJson(QJsonDocument::Compact)); + limitedNodeList->sendPacketList(std::move(packetList), *destinationNode); +} + +// Forwards a WebRTC signaling message received from an assignment client to the relevant user client. +void DomainServer::forwardAssignmentClientSignalingMessageToUserClient(QSharedPointer message) { + auto messageString = message->readString(); + auto json = QJsonDocument::fromJson(messageString.toUtf8()).object(); + emit webrtcSignalingMessageForUserClient(json); +} + #endif diff --git a/domain-server/src/DomainServer.h b/domain-server/src/DomainServer.h index 3f509b232dd..53f2aec85bf 100644 --- a/domain-server/src/DomainServer.h +++ b/domain-server/src/DomainServer.h @@ -150,6 +150,10 @@ private slots: void tokenGrantFinished(); void profileRequestFinished(); +#if defined(WEBRTC_DATA_CHANNELS) + void forwardAssignmentClientSignalingMessageToUserClient(QSharedPointer message); +#endif + void aboutToQuit(); signals: @@ -159,6 +163,7 @@ private slots: #if defined(WEBRTC_DATA_CHANNELS) void webrtcSignalingMessageForDomainServer(const QJsonObject& json); + void webrtcSignalingMessageForUserClient(const QJsonObject& json); #endif @@ -245,6 +250,7 @@ private slots: #if defined(WEBRTC_DATA_CHANNELS) void setUpWebRTCSignalingServer(); void routeWebRTCSignalingMessage(const QJsonObject& json); + void sendWebRTCSignalingMessageToAssignmentClient(const QJsonObject& json); #endif QString operationToString(const QNetworkAccessManager::Operation &op); diff --git a/libraries/networking/src/Node.cpp b/libraries/networking/src/Node.cpp index aa13dd65657..a90da4a9292 100644 --- a/libraries/networking/src/Node.cpp +++ b/libraries/networking/src/Node.cpp @@ -46,6 +46,22 @@ static const QHash TYPE_NAME_HASH { { NodeType::Unassigned, "Unassigned" } }; +static const QHash TYPE_CHAR_HASH { + { NodeType::DomainServer, "D" }, + { NodeType::EntityServer, "o" }, + { NodeType::Agent, "I" }, + { NodeType::AudioMixer, "M" }, + { NodeType::AvatarMixer, "W" }, + { NodeType::AssetServer, "A" }, + { NodeType::MessagesMixer, "m" }, + { NodeType::EntityScriptServer, "S" }, + { NodeType::UpstreamAudioMixer, "B" }, + { NodeType::UpstreamAvatarMixer, "C" }, + { NodeType::DownstreamAudioMixer, "a" }, + { NodeType::DownstreamAvatarMixer, "w" }, + { NodeType::Unassigned, QChar(1) } +}; + const QString& NodeType::getNodeTypeName(NodeType_t nodeType) { const auto matchedTypeName = TYPE_NAME_HASH.find(nodeType); return matchedTypeName != TYPE_NAME_HASH.end() ? matchedTypeName.value() : UNKNOWN_NodeType_t_NAME; @@ -85,6 +101,9 @@ NodeType_t NodeType::fromString(QString type) { return TYPE_NAME_HASH.key(type, NodeType::Unassigned); } +NodeType_t NodeType::fromChar(QChar type) { + return TYPE_CHAR_HASH.key(type, NodeType::Unassigned); +} Node::Node(const QUuid& uuid, NodeType_t type, const SockAddr& publicSocket, const SockAddr& localSocket, QObject* parent) : diff --git a/libraries/networking/src/NodeType.h b/libraries/networking/src/NodeType.h index 8539ce8fb3a..55754888c45 100644 --- a/libraries/networking/src/NodeType.h +++ b/libraries/networking/src/NodeType.h @@ -43,6 +43,7 @@ namespace NodeType { NodeType_t downstreamType(NodeType_t primaryType); NodeType_t fromString(QString type); + NodeType_t fromChar(QChar type); } typedef QSet NodeSet; diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index d57fa9f663d..6fc1b6c157e 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -139,6 +139,7 @@ class PacketTypeEnum { BulkAvatarTraitsAck, StopInjector, AvatarZonePresence, + WebRTCSignaling, NUM_PACKET_TYPE }; @@ -190,7 +191,7 @@ class PacketTypeEnum { << PacketTypeEnum::Value::ReplicatedMicrophoneAudioWithEcho << PacketTypeEnum::Value::ReplicatedInjectAudio << PacketTypeEnum::Value::ReplicatedSilentAudioFrame << PacketTypeEnum::Value::ReplicatedAvatarIdentity << PacketTypeEnum::Value::ReplicatedKillAvatar << PacketTypeEnum::Value::ReplicatedBulkAvatarData - << PacketTypeEnum::Value::AvatarZonePresence; + << PacketTypeEnum::Value::AvatarZonePresence << PacketTypeEnum::Value::WebRTCSignaling; return NON_SOURCED_PACKETS; } From 5d15ebb7237d37fdc49e8e19dacb4574b03f157d Mon Sep 17 00:00:00 2001 From: David Rowe Date: Wed, 18 Aug 2021 20:21:56 +1200 Subject: [PATCH 42/82] WebRTC data channel with assignment clients --- libraries/networking/src/webrtc/WebRTCDataChannels.cpp | 8 +++++++- libraries/networking/src/webrtc/WebRTCDataChannels.h | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp index 974cef29974..c62bca72c4c 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp @@ -452,12 +452,18 @@ void WebRTCDataChannels::onSignalingMessage(const QJsonObject& message) { const int MAX_DEBUG_DETAIL_LENGTH = 64; auto data = message.value("data").isObject() ? message.value("data").toObject() : QJsonObject(); int from = message.value("from").isDouble() ? (quint16)(message.value("from").toInt()) : 0; - if (from <= 0 || from > MAXUINT16 || !data.contains("description") && !data.contains("candidate")) { + auto to = NodeType::fromChar(message.value("to").toString().at(0)); + + if (from <= 0 || from > MAXUINT16 || to == NodeType::Unassigned + || !data.contains("description") && !data.contains("candidate")) { qCWarning(networking_webrtc) << "Unexpected signaling message:" << QJsonDocument(message).toJson(QJsonDocument::Compact).left(MAX_DEBUG_DETAIL_LENGTH); return; } + // Remember this node's type for the reply. + _nodeType = to; + // Find or create a connection. WDCConnection* connection; if (_connectionsByWebSocket.contains(from)) { diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.h b/libraries/networking/src/webrtc/WebRTCDataChannels.h index ec6f67b3c7a..f6287c3717b 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.h +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.h @@ -319,7 +319,7 @@ public slots: QObject* _parent; - NodeType_t _nodeType; + NodeType_t _nodeType { NodeType::Unassigned }; std::unique_ptr _rtcNetworkThread { nullptr }; std::unique_ptr _rtcWorkerThread { nullptr }; From ace612d038d4a613089e2c8c0dc1a25aab58117a Mon Sep 17 00:00:00 2001 From: David Rowe Date: Wed, 18 Aug 2021 21:27:35 +1200 Subject: [PATCH 43/82] Assignment client type is not known at construction --- assignment-client/src/AssignmentClientMonitor.cpp | 2 +- domain-server/src/DomainServer.cpp | 2 +- libraries/networking/src/LimitedNodeList.cpp | 4 ++-- libraries/networking/src/LimitedNodeList.h | 3 +-- libraries/networking/src/NodeList.cpp | 2 +- libraries/networking/src/NodeList.h | 2 +- libraries/networking/src/udt/NetworkSocket.cpp | 4 ++-- libraries/networking/src/udt/NetworkSocket.h | 3 +-- libraries/networking/src/udt/Socket.cpp | 4 ++-- libraries/networking/src/udt/Socket.h | 4 ++-- libraries/networking/src/webrtc/WebRTCDataChannels.cpp | 7 +++---- libraries/networking/src/webrtc/WebRTCDataChannels.h | 3 +-- libraries/networking/src/webrtc/WebRTCSocket.cpp | 4 ++-- libraries/networking/src/webrtc/WebRTCSocket.h | 3 +-- 14 files changed, 21 insertions(+), 26 deletions(-) diff --git a/assignment-client/src/AssignmentClientMonitor.cpp b/assignment-client/src/AssignmentClientMonitor.cpp index a2e4d4a697e..221a922a4b3 100644 --- a/assignment-client/src/AssignmentClientMonitor.cpp +++ b/assignment-client/src/AssignmentClientMonitor.cpp @@ -70,7 +70,7 @@ AssignmentClientMonitor::AssignmentClientMonitor(const unsigned int numAssignmen // create a NodeList so we can receive stats from children DependencyManager::registerInheritance(); auto addressManager = DependencyManager::set(); - auto nodeList = DependencyManager::set(NodeType::Unassigned, listenPort); + auto nodeList = DependencyManager::set(listenPort); auto& packetReceiver = DependencyManager::get()->getPacketReceiver(); packetReceiver.registerListener(PacketType::AssignmentClientStatus, diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 185250e8099..569e662ae0f 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -739,7 +739,7 @@ void DomainServer::setupNodeListAndAssignments() { // check for scripts the user wants to persist from their domain-server config populateStaticScriptedAssignmentsFromSettings(); - auto nodeList = DependencyManager::set(NodeType::DomainServer, domainServerPort, domainServerDTLSPort); + auto nodeList = DependencyManager::set(domainServerPort, domainServerDTLSPort); // no matter the local port, save it to shared mem so that local assignment clients can ask what it is nodeList->putLocalPortIntoSharedMemory(DOMAIN_SERVER_LOCAL_PORT_SMEM_KEY, this, diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index e7423633738..8a551f55cb0 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -50,8 +50,8 @@ static Setting::Handle LIMITED_NODELIST_LOCAL_PORT("LimitedNodeList.Loc using namespace std::chrono_literals; static const std::chrono::milliseconds CONNECTION_RATE_INTERVAL_MS = 1s; -LimitedNodeList::LimitedNodeList(char ownerType, int socketListenPort, int dtlsListenPort) : - _nodeSocket(this, true, ownerType), +LimitedNodeList::LimitedNodeList(int socketListenPort, int dtlsListenPort) : + _nodeSocket(this, true), _packetReceiver(new PacketReceiver(this)) { qRegisterMetaType("ConnectionStep"); diff --git a/libraries/networking/src/LimitedNodeList.h b/libraries/networking/src/LimitedNodeList.h index 81d93bf9357..2ee863da07b 100644 --- a/libraries/networking/src/LimitedNodeList.h +++ b/libraries/networking/src/LimitedNodeList.h @@ -416,8 +416,7 @@ protected slots: QUuid connectionSecretUUID; }; - LimitedNodeList(char ownerType = NodeType::DomainServer, int socketListenPort = INVALID_PORT, - int dtlsListenPort = INVALID_PORT); + LimitedNodeList(int socketListenPort = INVALID_PORT, int dtlsListenPort = INVALID_PORT); LimitedNodeList(LimitedNodeList const&) = delete; // Don't implement, needed to avoid copies of singleton void operator=(LimitedNodeList const&) = delete; // Don't implement, needed to avoid copies of singleton diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index 97080a349a7..b8e6cdf80a3 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -50,7 +50,7 @@ const int KEEPALIVE_PING_INTERVAL_MS = 1000; const int MAX_SYSTEM_INFO_SIZE = 1000; NodeList::NodeList(char newOwnerType, int socketListenPort, int dtlsListenPort) : - LimitedNodeList(newOwnerType, socketListenPort, dtlsListenPort), + LimitedNodeList(socketListenPort, dtlsListenPort), _ownerType(newOwnerType), _nodeTypesOfInterest(), _domainHandler(this), diff --git a/libraries/networking/src/NodeList.h b/libraries/networking/src/NodeList.h index d3999c31f62..7af0dc405fc 100644 --- a/libraries/networking/src/NodeList.h +++ b/libraries/networking/src/NodeList.h @@ -154,7 +154,7 @@ private slots: void maybeSendIgnoreSetToNode(SharedNodePointer node); private: - NodeList() : LimitedNodeList(NodeType::Unassigned, INVALID_PORT, INVALID_PORT) { + NodeList() : LimitedNodeList(INVALID_PORT, INVALID_PORT) { assert(false); // Not implemented, needed for DependencyManager templates compile } NodeList(char ownerType, int socketListenPort = INVALID_PORT, int dtlsListenPort = INVALID_PORT); diff --git a/libraries/networking/src/udt/NetworkSocket.cpp b/libraries/networking/src/udt/NetworkSocket.cpp index c882f6dbe39..fd646fe3172 100644 --- a/libraries/networking/src/udt/NetworkSocket.cpp +++ b/libraries/networking/src/udt/NetworkSocket.cpp @@ -11,13 +11,13 @@ #include "../NetworkLogging.h" -NetworkSocket::NetworkSocket(QObject* parent, NodeType_t nodeType) : +NetworkSocket::NetworkSocket(QObject* parent) : QObject(parent), _parent(parent), _udpSocket(this) #if defined(WEBRTC_DATA_CHANNELS) , - _webrtcSocket(this, nodeType) + _webrtcSocket(this) #endif { connect(&_udpSocket, &QUdpSocket::readyRead, this, &NetworkSocket::readyRead); diff --git a/libraries/networking/src/udt/NetworkSocket.h b/libraries/networking/src/udt/NetworkSocket.h index d5891cbde71..2670fa40b8d 100644 --- a/libraries/networking/src/udt/NetworkSocket.h +++ b/libraries/networking/src/udt/NetworkSocket.h @@ -33,8 +33,7 @@ class NetworkSocket : public QObject { /// @brief Constructs a new NetworkSocket object. /// @param parent Qt parent object. - /// @param nodeType The type of node that the NetworkSocket object is being used in. - NetworkSocket(QObject* parent, NodeType_t nodeType); + NetworkSocket(QObject* parent); /// @brief Set the value of a UDP or WebRTC socket option. diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 810948d742a..8313a87bbf9 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -40,9 +40,9 @@ using namespace udt; #endif -Socket::Socket(QObject* parent, bool shouldChangeSocketOptions, NodeType_t nodeType) : +Socket::Socket(QObject* parent, bool shouldChangeSocketOptions) : QObject(parent), - _networkSocket(parent, nodeType), + _networkSocket(parent), _readyReadBackupTimer(new QTimer(this)), _shouldChangeSocketOptions(shouldChangeSocketOptions) { diff --git a/libraries/networking/src/udt/Socket.h b/libraries/networking/src/udt/Socket.h index 5c93d966769..ab9699bb8f4 100644 --- a/libraries/networking/src/udt/Socket.h +++ b/libraries/networking/src/udt/Socket.h @@ -55,8 +55,8 @@ class Socket : public QObject { public: using StatsVector = std::vector>; - - Socket(QObject* object = 0, bool shouldChangeSocketOptions = true, NodeType_t nodeType = NodeType::Unassigned); + + Socket(QObject* object = 0, bool shouldChangeSocketOptions = true); quint16 localPort(SocketType socketType) const { return _networkSocket.localPort(socketType); } diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp index c62bca72c4c..3f7b84086d5 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp @@ -373,13 +373,12 @@ void WDCConnection::closePeerConnection() { } -WebRTCDataChannels::WebRTCDataChannels(QObject* parent, NodeType_t nodeType) : +WebRTCDataChannels::WebRTCDataChannels(QObject* parent) : QObject(parent), - _parent(parent), - _nodeType(nodeType) + _parent(parent) { #ifdef WEBRTC_DEBUG - qCDebug(networking_webrtc) << "WebRTCDataChannels::WebRTCDataChannels()" << nodeType << NodeType::getNodeTypeName(nodeType); + qCDebug(networking_webrtc) << "WebRTCDataChannels::WebRTCDataChannels()"; #endif // Create a peer connection factory. diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.h b/libraries/networking/src/webrtc/WebRTCDataChannels.h index f6287c3717b..b1751093d5a 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.h +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.h @@ -227,8 +227,7 @@ class WebRTCDataChannels : public QObject { /// @brief Constructs a new WebRTCDataChannels object. /// @param parent The parent Qt object. - /// @param nodeType The type of node that the WebRTCDataChannels object is being used in. - WebRTCDataChannels(QObject* parent, NodeType_t nodeType); + WebRTCDataChannels(QObject* parent); /// @brief Destroys a WebRTCDataChannels object. ~WebRTCDataChannels(); diff --git a/libraries/networking/src/webrtc/WebRTCSocket.cpp b/libraries/networking/src/webrtc/WebRTCSocket.cpp index 2247ad86159..69b97ebf94c 100644 --- a/libraries/networking/src/webrtc/WebRTCSocket.cpp +++ b/libraries/networking/src/webrtc/WebRTCSocket.cpp @@ -16,9 +16,9 @@ #include "../udt/Constants.h" -WebRTCSocket::WebRTCSocket(QObject* parent, NodeType_t nodeType) : +WebRTCSocket::WebRTCSocket(QObject* parent) : QObject(parent), - _dataChannels(this, nodeType) + _dataChannels(this) { // Route signaling messages. connect(this, &WebRTCSocket::onSignalingMessage, &_dataChannels, &WebRTCDataChannels::onSignalingMessage); diff --git a/libraries/networking/src/webrtc/WebRTCSocket.h b/libraries/networking/src/webrtc/WebRTCSocket.h index 9a1d6442d7f..8e429673e14 100644 --- a/libraries/networking/src/webrtc/WebRTCSocket.h +++ b/libraries/networking/src/webrtc/WebRTCSocket.h @@ -31,8 +31,7 @@ class WebRTCSocket : public QObject { /// @brief Constructs a new WebRTCSocket object. /// @param parent Qt parent object. - /// @param nodeType The type of node that the WebRTCsocket object is being used in. - WebRTCSocket(QObject* parent, NodeType_t nodeType); + WebRTCSocket(QObject* parent); /// @brief Nominally sets the value of a socket option. From 1605039929c6d88b616700c9e643d4ec5d05658b Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sat, 21 Aug 2021 21:11:42 +1200 Subject: [PATCH 44/82] Tidying --- libraries/networking/src/BaseAssetScriptingInterface.cpp | 1 + libraries/networking/src/NetworkPeer.cpp | 2 +- libraries/networking/src/udt/NetworkSocket.h | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/BaseAssetScriptingInterface.cpp b/libraries/networking/src/BaseAssetScriptingInterface.cpp index f9dd12b279a..872913082ac 100644 --- a/libraries/networking/src/BaseAssetScriptingInterface.cpp +++ b/libraries/networking/src/BaseAssetScriptingInterface.cpp @@ -3,6 +3,7 @@ // libraries/networking/src // // Copyright 2017 High Fidelity, Inc. +// Copyright 2021 Vircadia contributors. // // Distributed under the Apache License, Version 2.0. // See the accompanying file LICENSE or http://www.apache.org/licenses/LICENSE-2.0.html diff --git a/libraries/networking/src/NetworkPeer.cpp b/libraries/networking/src/NetworkPeer.cpp index e76b8c31a64..0b3b82ca65b 100644 --- a/libraries/networking/src/NetworkPeer.cpp +++ b/libraries/networking/src/NetworkPeer.cpp @@ -117,7 +117,7 @@ void NetworkPeer::setActiveSocket(SockAddr* discoveredSocket) { // we have an active socket, stop our ping timer stopPingTimer(); - // we're now considered connected to this peer - reset the number of connection attemps + // we're now considered connected to this peer - reset the number of connection attempts resetConnectionAttempts(); if (_activeSocket) { diff --git a/libraries/networking/src/udt/NetworkSocket.h b/libraries/networking/src/udt/NetworkSocket.h index 2670fa40b8d..887e73b32ba 100644 --- a/libraries/networking/src/udt/NetworkSocket.h +++ b/libraries/networking/src/udt/NetworkSocket.h @@ -120,7 +120,7 @@ class NetworkSocket : public QObject { #if defined(WEBRTC_DATA_CHANNELS) - /// @brief @brief Gets a pointer to the WebRTC socket object. + /// @brief Gets a pointer to the WebRTC socket object. /// @return A pointer to the WebRTC socket object. const WebRTCSocket* getWebRTCSocket(); #endif From adc94a280ec7a021d424f4475a71a37fe4835e1f Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sun, 29 Aug 2021 14:19:39 +1200 Subject: [PATCH 45/82] Log some warnings --- assignment-client/src/AssignmentClient.cpp | 1 + domain-server/src/DomainServer.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/assignment-client/src/AssignmentClient.cpp b/assignment-client/src/AssignmentClient.cpp index cfe940133f0..bd35c20df8d 100644 --- a/assignment-client/src/AssignmentClient.cpp +++ b/assignment-client/src/AssignmentClient.cpp @@ -356,6 +356,7 @@ void AssignmentClient::handleWebRTCSignalingPacket(QSharedPointer(); auto destinationNode = limitedNodeList->soloNodeOfType(destinationNodeType); if (!destinationNode) { + qWarning() << NodeType::getNodeTypeName(destinationNodeType) << "not found for WebRTC signaling message."; return; } // Use an NLPacketList because the signaling message is not necessarily small. From 65dec6dfd8f15d9bd0e96d7401d253c5658fceed Mon Sep 17 00:00:00 2001 From: David Rowe Date: Thu, 2 Sep 2021 15:00:26 +1200 Subject: [PATCH 46/82] Socket type isn't used in all code --- libraries/networking/src/SockAddr.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/SockAddr.cpp b/libraries/networking/src/SockAddr.cpp index 58dc9714bd6..2760290ea6a 100644 --- a/libraries/networking/src/SockAddr.cpp +++ b/libraries/networking/src/SockAddr.cpp @@ -129,7 +129,9 @@ bool SockAddr::hasPrivateAddress() const { } QDebug operator<<(QDebug debug, const SockAddr& sockAddr) { - debug.nospace() << socketTypeToString(sockAddr._socketType).toLocal8Bit().constData() << " " + debug.nospace() + << (sockAddr._socketType != SocketType::Unknown + ? (socketTypeToString(sockAddr._socketType) + " ").toLocal8Bit().constData() : "") << sockAddr._address.toString().toLocal8Bit().constData() << ":" << sockAddr._port; return debug.space(); } From ff72b422b30a2b6d4b19cfc289eced6b3fe1b122 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Fri, 3 Sep 2021 19:04:24 +1200 Subject: [PATCH 47/82] Fix WebRTCDataChannels crash --- libraries/networking/src/webrtc/WebRTCDataChannels.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp index 3f7b84086d5..c2b7ac6f5b1 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp @@ -524,8 +524,14 @@ bool WebRTCDataChannels::sendDataMessage(int dataChannelID, const QByteArray& by /// @brief Gets the number of bytes waiting to be written on a data channel. /// @param port The data channel ID. -/// @return The number of bytes waiting to be written on the data channel. +/// @return The number of bytes waiting to be written on the data channel; 0 if the channel doesn't exist. qint64 WebRTCDataChannels::getBufferedAmount(int dataChannelID) const { + if (!_connectionsByDataChannel.contains(dataChannelID)) { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WebRTCDataChannels::getBufferedAmount() : Channel doesn't exist:" << dataChannelID; +#endif + return 0; + } auto connection = _connectionsByDataChannel.value(dataChannelID); return connection->getBufferedAmount(); } From 911dc2aff377519bfcc630e198ae40e781727320 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sat, 4 Sep 2021 10:36:13 +1200 Subject: [PATCH 48/82] Include socket type in domain packets Required for user client info sent to assignment clients from the domain server. --- domain-server/src/NodeConnectionData.cpp | 24 +++++++++---------- libraries/networking/src/Node.cpp | 7 ++++++ libraries/networking/src/NodeList.cpp | 12 ++++++---- libraries/networking/src/SockAddr.cpp | 11 ++------- libraries/networking/src/SockAddr.h | 4 ++-- libraries/networking/src/SocketType.h | 4 ++-- .../networking/src/udt/NetworkSocket.cpp | 8 +++---- .../networking/src/udt/PacketHeaders.cpp | 8 ++++--- libraries/networking/src/udt/PacketHeaders.h | 14 ++++++++--- libraries/networking/src/udt/Socket.cpp | 2 +- 10 files changed, 53 insertions(+), 41 deletions(-) diff --git a/domain-server/src/NodeConnectionData.cpp b/domain-server/src/NodeConnectionData.cpp index 60a8d6878ce..6e0c8427ba1 100644 --- a/domain-server/src/NodeConnectionData.cpp +++ b/domain-server/src/NodeConnectionData.cpp @@ -51,20 +51,20 @@ NodeConnectionData NodeConnectionData::fromDataStream(QDataStream& dataStream, c dataStream >> newHeader.lastPingTimestamp; + SocketType publicSocketType, localSocketType; dataStream >> newHeader.nodeType - >> newHeader.publicSockAddr >> newHeader.localSockAddr + >> publicSocketType >> newHeader.publicSockAddr >> localSocketType >> newHeader.localSockAddr >> newHeader.interestList >> newHeader.placeName; - - // A WebRTC web client doesn't necessarily know it's public Internet or local network addresses, and for WebRTC they aren't - // needed: for WebRTC, the data channel ID is the important thing. The client's public Internet IP address still needs to - // be known for domain access permissions, though, and this can be obtained from the WebSocket signaling connection. - if (senderSockAddr.getSocketType() == SocketType::WebRTC) { - // WEBRTC TODO: Rather than setting the SocketType here, serialize and deserialize the SocketType in the leading byte of - // the 5 bytes used to encode the IP address. - newHeader.publicSockAddr.setSocketType(SocketType::WebRTC); - newHeader.localSockAddr.setSocketType(SocketType::WebRTC); - - // WEBRTC TODO: Set the public Internet address obtained from the WebSocket used in WebRTC signaling. + newHeader.publicSockAddr.setType(publicSocketType); + newHeader.localSockAddr.setType(localSocketType); + + // For WebRTC connections, the user client doesn't know the WebRTC data channel ID that the domain server is using as its + // SockAddr port, so set the port values here. + if (senderSockAddr.getType() == SocketType::WebRTC) { + if (newHeader.publicSockAddr.getType() != SocketType::WebRTC + || newHeader.localSockAddr.getType() != SocketType::WebRTC) { + qDebug() << "Inconsistent WebRTC socket types!"; + } newHeader.publicSockAddr.setPort(senderSockAddr.getPort()); // We don't know whether it's a public or local connection newHeader.localSockAddr.setPort(senderSockAddr.getPort()); // so set both ports. diff --git a/libraries/networking/src/Node.cpp b/libraries/networking/src/Node.cpp index a90da4a9292..de9fb6bddf1 100644 --- a/libraries/networking/src/Node.cpp +++ b/libraries/networking/src/Node.cpp @@ -196,7 +196,9 @@ bool Node::isIgnoringNodeWithID(const QUuid& nodeID) const { QDataStream& operator<<(QDataStream& out, const Node& node) { out << node._type; out << node._uuid; + out << node._publicSocket.getType(); out << node._publicSocket; + out << node._localSocket.getType(); out << node._localSocket; out << node._permissions; out << node._isReplicated; @@ -205,10 +207,15 @@ QDataStream& operator<<(QDataStream& out, const Node& node) { } QDataStream& operator>>(QDataStream& in, Node& node) { + SocketType publicSocketType, localSocketType; in >> node._type; in >> node._uuid; + in >> publicSocketType; in >> node._publicSocket; + node._publicSocket.setType(publicSocketType); + in >> localSocketType; in >> node._localSocket; + node._localSocket.setType(localSocketType); in >> node._permissions; in >> node._isReplicated; in >> node._localID; diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index 1957b09e82b..9225d25ddf5 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -494,7 +494,8 @@ void NodeList::sendDomainServerCheckIn() { // pack our data to send to the domain-server including // the hostname information (so the domain-server can see which place name we came in on) - packetStream << _ownerType.load() << publicSockAddr << localSockAddr << _nodeTypesOfInterest.toList(); + packetStream << _ownerType.load() << publicSockAddr.getType() << publicSockAddr << localSockAddr.getType() + << localSockAddr << _nodeTypesOfInterest.toList(); packetStream << DependencyManager::get()->getPlaceName(); if (!domainIsConnected) { @@ -879,14 +880,19 @@ void NodeList::processDomainServerRemovedNode(QSharedPointer me void NodeList::parseNodeFromPacketStream(QDataStream& packetStream) { NewNodeInfo info; + SocketType publicSocketType, localSocketType; packetStream >> info.type >> info.uuid + >> publicSocketType >> info.publicSocket + >> localSocketType >> info.localSocket >> info.permissions >> info.isReplicated >> info.sessionLocalID >> info.connectionSecretUUID; + info.publicSocket.setType(publicSocketType); + info.localSocket.setType(localSocketType); // if the public socket address is 0 then it's reachable at the same IP // as the domain server @@ -894,10 +900,6 @@ void NodeList::parseNodeFromPacketStream(QDataStream& packetStream) { info.publicSocket.setAddress(_domainHandler.getIP()); } - // WEBRTC TODO: Handle WebRTC-connected nodes. Probably need to include SocketType in SockAddr << and >> - info.publicSocket.setSocketType(SocketType::UDP); - info.localSocket.setSocketType(SocketType::UDP); - addNewNode(info); } diff --git a/libraries/networking/src/SockAddr.cpp b/libraries/networking/src/SockAddr.cpp index 2760290ea6a..61be5219f24 100644 --- a/libraries/networking/src/SockAddr.cpp +++ b/libraries/networking/src/SockAddr.cpp @@ -137,21 +137,14 @@ QDebug operator<<(QDebug debug, const SockAddr& sockAddr) { } QDataStream& operator<<(QDataStream& dataStream, const SockAddr& sockAddr) { - // Don't include socketType because it can be implied from the type of connection used. - // WEBRTC TODO: Reconsider this. + // Don't include socket type because ICE packets must not have it. dataStream << sockAddr._address << sockAddr._port; return dataStream; } QDataStream& operator>>(QDataStream& dataStream, SockAddr& sockAddr) { - // Don't include socketType because it can be implied from the type of connection used. - // WEBRTC TODO: Reconsider this. + // Don't include socket type because ICE packets must not have it. dataStream >> sockAddr._address >> sockAddr._port; - - // Set default for non-WebRTC code. - // WEBRTC TODO: Reconsider this. - sockAddr.setSocketType(SocketType::UDP); - return dataStream; } diff --git a/libraries/networking/src/SockAddr.h b/libraries/networking/src/SockAddr.h index ab70325ed4e..4e2ec89a504 100644 --- a/libraries/networking/src/SockAddr.h +++ b/libraries/networking/src/SockAddr.h @@ -40,9 +40,9 @@ class SockAddr : public QObject { bool operator==(const SockAddr& rhsSockAddr) const; bool operator!=(const SockAddr& rhsSockAddr) const { return !(*this == rhsSockAddr); } - SocketType getSocketType() const { return _socketType; } + SocketType getType() const { return _socketType; } SocketType* getSocketTypePointer() { return &_socketType; } - void setSocketType(const SocketType socketType) { _socketType = socketType; } + void setType(const SocketType socketType) { _socketType = socketType; } const QHostAddress& getAddress() const { return _address; } QHostAddress* getAddressPointer() { return &_address; } diff --git a/libraries/networking/src/SocketType.h b/libraries/networking/src/SocketType.h index a798fb59c6f..d9b613a7e1e 100644 --- a/libraries/networking/src/SocketType.h +++ b/libraries/networking/src/SocketType.h @@ -19,8 +19,8 @@ /// @brief The types of network socket. -enum class SocketType { - Unknown, ///< Unknown socket type. +enum class SocketType : uint8_t { + Unknown, ///< Socket type unknown or not set. UDP, ///< UDP socket. WebRTC ///< WebRTC socket. A WebRTC data channel presented as a UDP-style socket. }; diff --git a/libraries/networking/src/udt/NetworkSocket.cpp b/libraries/networking/src/udt/NetworkSocket.cpp index fd646fe3172..639e4bbb88a 100644 --- a/libraries/networking/src/udt/NetworkSocket.cpp +++ b/libraries/networking/src/udt/NetworkSocket.cpp @@ -126,7 +126,7 @@ qintptr NetworkSocket::socketDescriptor(SocketType socketType) const { qint64 NetworkSocket::writeDatagram(const QByteArray& datagram, const SockAddr& sockAddr) { - switch (sockAddr.getSocketType()) { + switch (sockAddr.getType()) { case SocketType::UDP: // WEBRTC TODO: The Qt documentation says that the following call shouldn't be used if the UDP socket is connected!!! // https://doc.qt.io/qt-5/qudpsocket.html#writeDatagram @@ -197,7 +197,7 @@ qint64 NetworkSocket::readDatagram(char* data, qint64 maxSize, SockAddr* sockAdd _lastSocketTypeRead = SocketType::UDP; _pendingDatagramSizeSocketType = SocketType::Unknown; if (sockAddr) { - sockAddr->setSocketType(SocketType::UDP); + sockAddr->setType(SocketType::UDP); return _udpSocket.readDatagram(data, maxSize, sockAddr->getAddressPointer(), sockAddr->getPortPointer()); } else { return _udpSocket.readDatagram(data, maxSize); @@ -206,7 +206,7 @@ qint64 NetworkSocket::readDatagram(char* data, qint64 maxSize, SockAddr* sockAdd _lastSocketTypeRead = SocketType::WebRTC; _pendingDatagramSizeSocketType = SocketType::Unknown; if (sockAddr) { - sockAddr->setSocketType(SocketType::WebRTC); + sockAddr->setType(SocketType::WebRTC); return _webrtcSocket.readDatagram(data, maxSize, sockAddr->getAddressPointer(), sockAddr->getPortPointer()); } else { return _webrtcSocket.readDatagram(data, maxSize); @@ -214,7 +214,7 @@ qint64 NetworkSocket::readDatagram(char* data, qint64 maxSize, SockAddr* sockAdd } #else if (sockAddr) { - sockAddr->setSocketType(SocketType::UDP); + sockAddr->setType(SocketType::UDP); return _udpSocket.readDatagram(data, maxSize, sockAddr->getAddressPointer(), sockAddr->getPortPointer()); } else { return _udpSocket.readDatagram(data, maxSize); diff --git a/libraries/networking/src/udt/PacketHeaders.cpp b/libraries/networking/src/udt/PacketHeaders.cpp index f7c1192886b..e561bfe21ec 100644 --- a/libraries/networking/src/udt/PacketHeaders.cpp +++ b/libraries/networking/src/udt/PacketHeaders.cpp @@ -27,7 +27,7 @@ PacketVersion versionForPacketType(PacketType packetType) { case PacketType::DomainConnectRequestPending: // keeping the old version to maintain the protocol hash return 17; case PacketType::DomainList: - return static_cast(DomainListVersion::HasConnectReason); + return static_cast(DomainListVersion::SocketTypes); case PacketType::EntityAdd: case PacketType::EntityClone: case PacketType::EntityEdit: @@ -72,10 +72,12 @@ PacketVersion versionForPacketType(PacketType packetType) { return static_cast(DomainConnectionDeniedVersion::IncludesExtraInfo); case PacketType::DomainConnectRequest: - return static_cast(DomainConnectRequestVersion::HasCompressedSystemInfo); + return static_cast(DomainConnectRequestVersion::SocketTypes); + case PacketType::DomainListRequest: + return static_cast(DomainListRequestVersion::SocketTypes); case PacketType::DomainServerAddedNode: - return static_cast(DomainServerAddedNodeVersion::PermissionsGrid); + return static_cast(DomainServerAddedNodeVersion::SocketTypes); case PacketType::EntityScriptCallMethod: return static_cast(EntityScriptCallMethodVersion::ClientCallable); diff --git a/libraries/networking/src/udt/PacketHeaders.h b/libraries/networking/src/udt/PacketHeaders.h index 6fc1b6c157e..64b2e481c52 100644 --- a/libraries/networking/src/udt/PacketHeaders.h +++ b/libraries/networking/src/udt/PacketHeaders.h @@ -368,7 +368,13 @@ enum class DomainConnectRequestVersion : PacketVersion { HasTimestamp, HasReason, HasSystemInfo, - HasCompressedSystemInfo + HasCompressedSystemInfo, + SocketTypes +}; + +enum class DomainListRequestVersion : PacketVersion { + PreSocketTypes = 22, + SocketTypes }; enum class DomainConnectionDeniedVersion : PacketVersion { @@ -379,7 +385,8 @@ enum class DomainConnectionDeniedVersion : PacketVersion { enum class DomainServerAddedNodeVersion : PacketVersion { PrePermissionsGrid = 17, - PermissionsGrid + PermissionsGrid, + SocketTypes }; enum class DomainListVersion : PacketVersion { @@ -389,7 +396,8 @@ enum class DomainListVersion : PacketVersion { GetMachineFingerprintFromUUIDSupport, AuthenticationOptional, HasTimestamp, - HasConnectReason + HasConnectReason, + SocketTypes }; enum class AudioVersion : PacketVersion { diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 8313a87bbf9..2aa5542651b 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -245,7 +245,7 @@ qint64 Socket::writeDatagram(const char* data, qint64 size, const SockAddr& sock } qint64 Socket::writeDatagram(const QByteArray& datagram, const SockAddr& sockAddr) { - auto socketType = sockAddr.getSocketType(); + auto socketType = sockAddr.getType(); // don't attempt to write the datagram if we're unbound. Just drop it. // _networkSocket.writeDatagram will return an error anyway, but there are From dddabecc8449d6ba715fe57c55a781d5c0bef51c Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sat, 4 Sep 2021 14:25:21 +1200 Subject: [PATCH 49/82] Use domain server-assigned WebRTC data channel ID in assignment clients --- domain-server/src/DomainServer.cpp | 11 +++++- .../src/webrtc/WebRTCDataChannels.cpp | 34 +++++++++++++++---- .../src/webrtc/WebRTCDataChannels.h | 10 ++++-- .../networking/src/webrtc/WebRTCSocket.cpp | 5 +++ .../networking/src/webrtc/WebRTCSocket.h | 6 ++++ 5 files changed, 56 insertions(+), 10 deletions(-) diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index b844615c6d1..93abd8c7592 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -891,7 +891,16 @@ void DomainServer::routeWebRTCSignalingMessage(const QJsonObject& json) { if (json.value("to").toString() == NodeType::DomainServer) { emit webrtcSignalingMessageForDomainServer(json); } else { - sendWebRTCSignalingMessageToAssignmentClient(json); + // Insert the WebRTC data channel ID for the assignment client to use. + auto webrtcSocket = DependencyManager::get()->getWebRTCSocket(); + auto channelID = webrtcSocket->getDataChannelIDForWebSocket((quint16)json.value("from").toInt()); + if (channelID == 0 && json.value("echo").isUndefined()) { // Let echo messages through without a domain connection. + qCritical() << "WebRTC data channel ID not found for assignment client signaling!"; + return; + } + QJsonObject jsonModified = json; + jsonModified.insert("channel", QJsonValue(channelID)); + sendWebRTCSignalingMessageToAssignmentClient(jsonModified); } } diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp index c2b7ac6f5b1..fd7ae8a4548 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp @@ -131,12 +131,13 @@ void WDCDataChannelObserver::OnMessage(const DataBuffer& buffer) { } -WDCConnection::WDCConnection(WebRTCDataChannels* parent, quint16 webSocketID) : +WDCConnection::WDCConnection(WebRTCDataChannels* parent, quint16 webSocketID, int dataChannelID) : _parent(parent), - _webSocketID(webSocketID) + _webSocketID(webSocketID), + _dataChannelID(dataChannelID) { #ifdef WEBRTC_DEBUG - qCDebug(networking_webrtc) << "WDCConnection::WDCConnection() :" << webSocketID; + qCDebug(networking_webrtc) << "WDCConnection::WDCConnection() :" << webSocketID << dataChannelID; #endif // Create observers. @@ -288,7 +289,6 @@ void WDCConnection::onDataChannelOpened(rtc::scoped_refptr #endif _dataChannel = dataChannel; - _dataChannelID = _parent->getNewDataChannelID(); // Not dataChannel->id() because it's only unique per peer connection. _dataChannel->RegisterObserver(_dataChannelObserver.get()); #ifdef WEBRTC_DEBUG @@ -432,9 +432,23 @@ void WebRTCDataChannels::reset() { quint16 WebRTCDataChannels::getNewDataChannelID() { static const int QUINT16_LIMIT = std::numeric_limits::max() + 1; _lastDataChannelID = std::max((_lastDataChannelID + 1) % QUINT16_LIMIT, 1); +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WebRTCDataChannels::getNewDataChannelID() :" << _lastDataChannelID; +#endif return _lastDataChannelID; } +int WebRTCDataChannels::getDataChannelIDForWebSocket(quint16 webSocketID) const { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WebRTCDataChannels::getDataChannelIDForWebSocket() :" << webSocketID; +#endif + auto connection = _connectionsByWebSocket.value(webSocketID); + if (!connection) { + return 0; + } + return connection->getDataChannelID(); +} + void WebRTCDataChannels::onDataChannelOpened(WDCConnection* connection, quint16 dataChannelID) { #ifdef WEBRTC_DEBUG qCDebug(networking_webrtc) << "WebRTCDataChannels::onDataChannelOpened() :" << dataChannelID; @@ -444,7 +458,7 @@ void WebRTCDataChannels::onDataChannelOpened(WDCConnection* connection, quint16 void WebRTCDataChannels::onSignalingMessage(const QJsonObject& message) { #ifdef WEBRTC_DEBUG - qCDebug(networking_webrtc) << "WebRTCDataChannel::onSignalingMessage()" << message; + qCDebug(networking_webrtc) << "WebRTCDataChannel::onSignalingMessage()" << message << message.value("channel"); #endif // Validate message. @@ -452,8 +466,9 @@ void WebRTCDataChannels::onSignalingMessage(const QJsonObject& message) { auto data = message.value("data").isObject() ? message.value("data").toObject() : QJsonObject(); int from = message.value("from").isDouble() ? (quint16)(message.value("from").toInt()) : 0; auto to = NodeType::fromChar(message.value("to").toString().at(0)); + int channel = message.value("channel").isDouble() ? (int)(message.value("channel").toInt()) : 0; - if (from <= 0 || from > MAXUINT16 || to == NodeType::Unassigned + if (from <= 0 || from > MAXUINT16 || to == NodeType::Unassigned || channel < 0 || channel > MAXUINT16 || !data.contains("description") && !data.contains("candidate")) { qCWarning(networking_webrtc) << "Unexpected signaling message:" << QJsonDocument(message).toJson(QJsonDocument::Compact).left(MAX_DEBUG_DETAIL_LENGTH); @@ -468,7 +483,12 @@ void WebRTCDataChannels::onSignalingMessage(const QJsonObject& message) { if (_connectionsByWebSocket.contains(from)) { connection = _connectionsByWebSocket.value(from); } else { - connection = new WDCConnection(this, from); + // Assignment clients use the same data channel ID as the domain server, which is provided in the "channel" property. + // The domain server creates a new data channel ID. + if (channel == 0) { + channel = getNewDataChannelID(); + } + connection = new WDCConnection(this, from, channel); _connectionsByWebSocket.insert(from, connection); } diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.h b/libraries/networking/src/webrtc/WebRTCDataChannels.h index b1751093d5a..2e2827122b1 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.h +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.h @@ -129,7 +129,8 @@ class WDCConnection { /// @brief Constructs a new WDCConnection and opens a WebRTC data connection. /// @param parent The parent WebRTCDataChannels object. /// @param webSocketID The signaling channel that initiated the opening of the WebRTC data channel. - WDCConnection(WebRTCDataChannels* parent, quint16 webSocketID); + /// @param dataChannelID - The WebRTC data channel ID to assign to this connection. + WDCConnection(WebRTCDataChannels* parent, quint16 webSocketID, int dataChannelID); /// @brief Gets the WebSocket ID. /// @return The ID of the WebSocket. @@ -241,12 +242,17 @@ class WebRTCDataChannels : public QObject { /// @brief Immediately closes all connections and resets the socket. void reset(); - /// @brief Get a new data channel ID to uniquely identify a WDCConnection. + /// @brief Gets a new data channel ID to uniquely identify a WDCConnection. /// @details This ID is assigned by WebRTCDataChannels; it is not the WebRTC data channel ID because that is only /// unique within a peer connection. /// @return A new data channel ID. quint16 getNewDataChannelID(); + /// @brief Gets the data channel ID associated with a WebSocket. + /// @param webSocketID The WebSocket. + /// @return The data channel ID associated with the WebSocket if found, `0` if the WebSocket was not found. + int getDataChannelIDForWebSocket(quint16 webSocketID) const; + /// @brief Handles a WebRTC data channel opening. /// @param connection The WebRTC data channel connection. /// @param dataChannelID The WebRTC data channel ID. diff --git a/libraries/networking/src/webrtc/WebRTCSocket.cpp b/libraries/networking/src/webrtc/WebRTCSocket.cpp index 69b97ebf94c..589d46a76ea 100644 --- a/libraries/networking/src/webrtc/WebRTCSocket.cpp +++ b/libraries/networking/src/webrtc/WebRTCSocket.cpp @@ -153,4 +153,9 @@ void WebRTCSocket::onDataChannelReceivedMessage(int dataChannelID, const QByteAr emit readyRead(); } + +int WebRTCSocket::getDataChannelIDForWebSocket(quint16 webSocketID) const { + return _dataChannels.getDataChannelIDForWebSocket(webSocketID); +} + #endif // WEBRTC_DATA_CHANNELS diff --git a/libraries/networking/src/webrtc/WebRTCSocket.h b/libraries/networking/src/webrtc/WebRTCSocket.h index 8e429673e14..f0a6a99d0c6 100644 --- a/libraries/networking/src/webrtc/WebRTCSocket.h +++ b/libraries/networking/src/webrtc/WebRTCSocket.h @@ -120,6 +120,12 @@ class WebRTCSocket : public QObject { /// @return The description of the error that last occurred. QString errorString() const; + + /// @brief Gets the data channel ID associated with a WebSocket. + /// @param webSocketID + /// @return The data channel ID associated with the WebSocket if found, `0` if the WebSocket was not found. + int getDataChannelIDForWebSocket(quint16 webSocketID) const; + public slots: /// @brief Handles the WebRTC data channel receiving a message. From b5867fef48d0c2e43952957dcaa0a40678c9a784 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sat, 4 Sep 2021 14:31:20 +1200 Subject: [PATCH 50/82] Tidying --- libraries/networking/src/NodeList.cpp | 2 ++ libraries/networking/src/webrtc/WebRTCDataChannels.cpp | 3 ++- libraries/networking/src/webrtc/WebRTCDataChannels.h | 2 +- libraries/networking/src/webrtc/WebRTCSocket.h | 2 -- 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/libraries/networking/src/NodeList.cpp b/libraries/networking/src/NodeList.cpp index 9225d25ddf5..58d161a7975 100644 --- a/libraries/networking/src/NodeList.cpp +++ b/libraries/networking/src/NodeList.cpp @@ -470,10 +470,12 @@ void NodeList::sendDomainServerCheckIn() { QByteArray compressedSystemInfo = qCompress(systemInfo); if (compressedSystemInfo.size() > MAX_SYSTEM_INFO_SIZE) { + // FIXME // Highly unlikely, as not even unreasonable machines will // overflow the max size, but prevent MTU overflow anyway. // We could do something sophisticated like clearing specific // values if they're too big, but we'll save that for later. + // Alternative solution would be to write system info at the end of the packet, only if there is space. compressedSystemInfo.clear(); } diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp index fd7ae8a4548..d3646508c89 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp @@ -430,6 +430,7 @@ void WebRTCDataChannels::reset() { } quint16 WebRTCDataChannels::getNewDataChannelID() { + // The first data channel ID is 1. static const int QUINT16_LIMIT = std::numeric_limits::max() + 1; _lastDataChannelID = std::max((_lastDataChannelID + 1) % QUINT16_LIMIT, 1); #ifdef WEBRTC_DEBUG @@ -533,7 +534,7 @@ bool WebRTCDataChannels::sendDataMessage(int dataChannelID, const QByteArray& by // Find connection. if (!_connectionsByDataChannel.contains(dataChannelID)) { - qCWarning(networking_webrtc) << "Could not find data channel to send message on!"; + qCWarning(networking_webrtc) << "Could not find WebRTC data channel to send message on!"; return false; } diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.h b/libraries/networking/src/webrtc/WebRTCDataChannels.h index 2e2827122b1..dc8152ee348 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.h +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.h @@ -332,7 +332,7 @@ public slots: rtc::scoped_refptr _peerConnectionFactory { nullptr }; - quint16 _lastDataChannelID { 0 }; // First data channel ID is 1. + quint16 _lastDataChannelID { 0 }; QHash _connectionsByWebSocket; QHash _connectionsByDataChannel; diff --git a/libraries/networking/src/webrtc/WebRTCSocket.h b/libraries/networking/src/webrtc/WebRTCSocket.h index f0a6a99d0c6..53c8921f023 100644 --- a/libraries/networking/src/webrtc/WebRTCSocket.h +++ b/libraries/networking/src/webrtc/WebRTCSocket.h @@ -71,13 +71,11 @@ class WebRTCSocket : public QObject { void abort(); /// @brief Nominally gets the host port number. - /// @details /// Included for compatibility with the QUdpSocket interface. /// @return 0 quint16 localPort() const { return 0; } /// @brief Nominally gets the socket descriptor. - /// @details /// Included for compatibility with the QUdpSocket interface. /// @return -1 qintptr socketDescriptor() const { return -1; } From e9776541bd896ec16b1fcd24960d0b1e07b634a0 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sun, 5 Sep 2021 22:54:58 +1200 Subject: [PATCH 51/82] Use WebSocket address and port as WebRTC data channel ID --- domain-server/src/DomainServer.cpp | 11 +-- libraries/networking/src/SockAddr.cpp | 4 + libraries/networking/src/SockAddr.h | 1 + .../networking/src/udt/NetworkSocket.cpp | 6 +- libraries/networking/src/udt/NetworkSocket.h | 5 +- libraries/networking/src/udt/Socket.cpp | 2 +- .../src/webrtc/WebRTCDataChannels.cpp | 96 +++++++------------ .../src/webrtc/WebRTCDataChannels.h | 62 +++++------- .../src/webrtc/WebRTCSignalingServer.cpp | 16 ++-- .../src/webrtc/WebRTCSignalingServer.h | 27 +++--- .../networking/src/webrtc/WebRTCSocket.cpp | 22 ++--- .../networking/src/webrtc/WebRTCSocket.h | 30 +++--- 12 files changed, 118 insertions(+), 164 deletions(-) diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 93abd8c7592..b844615c6d1 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -891,16 +891,7 @@ void DomainServer::routeWebRTCSignalingMessage(const QJsonObject& json) { if (json.value("to").toString() == NodeType::DomainServer) { emit webrtcSignalingMessageForDomainServer(json); } else { - // Insert the WebRTC data channel ID for the assignment client to use. - auto webrtcSocket = DependencyManager::get()->getWebRTCSocket(); - auto channelID = webrtcSocket->getDataChannelIDForWebSocket((quint16)json.value("from").toInt()); - if (channelID == 0 && json.value("echo").isUndefined()) { // Let echo messages through without a domain connection. - qCritical() << "WebRTC data channel ID not found for assignment client signaling!"; - return; - } - QJsonObject jsonModified = json; - jsonModified.insert("channel", QJsonValue(channelID)); - sendWebRTCSignalingMessageToAssignmentClient(jsonModified); + sendWebRTCSignalingMessageToAssignmentClient(json); } } diff --git a/libraries/networking/src/SockAddr.cpp b/libraries/networking/src/SockAddr.cpp index 61be5219f24..e8eb6c4b863 100644 --- a/libraries/networking/src/SockAddr.cpp +++ b/libraries/networking/src/SockAddr.cpp @@ -116,6 +116,10 @@ QString SockAddr::toString() const { return socketTypeToString(_socketType) + " " + _address.toString() + ":" + QString::number(_port); } +QString SockAddr::toShortString() const { + return _address.toString() + ":" + QString::number(_port); +} + bool SockAddr::hasPrivateAddress() const { // an address is private if it is loopback or falls in any of the RFC1918 address spaces const QPair TWENTY_FOUR_BIT_BLOCK = { QHostAddress("10.0.0.0"), 8 }; diff --git a/libraries/networking/src/SockAddr.h b/libraries/networking/src/SockAddr.h index 4e2ec89a504..877bae4ee47 100644 --- a/libraries/networking/src/SockAddr.h +++ b/libraries/networking/src/SockAddr.h @@ -56,6 +56,7 @@ class SockAddr : public QObject { static int unpackSockAddr(const unsigned char* packetData, SockAddr& unpackDestSockAddr); QString toString() const; + QString toShortString() const; bool hasPrivateAddress() const; // checks if the address behind this sock addr is private per RFC 1918 diff --git a/libraries/networking/src/udt/NetworkSocket.cpp b/libraries/networking/src/udt/NetworkSocket.cpp index 639e4bbb88a..cc28cbfc734 100644 --- a/libraries/networking/src/udt/NetworkSocket.cpp +++ b/libraries/networking/src/udt/NetworkSocket.cpp @@ -133,7 +133,7 @@ qint64 NetworkSocket::writeDatagram(const QByteArray& datagram, const SockAddr& return _udpSocket.writeDatagram(datagram, sockAddr.getAddress(), sockAddr.getPort()); #if defined(WEBRTC_DATA_CHANNELS) case SocketType::WebRTC: - return _webrtcSocket.writeDatagram(datagram, sockAddr.getPort()); + return _webrtcSocket.writeDatagram(datagram, sockAddr); #endif default: qCCritical(networking) << "Socket type not specified in writeDatagram() address"; @@ -141,13 +141,13 @@ qint64 NetworkSocket::writeDatagram(const QByteArray& datagram, const SockAddr& } } -qint64 NetworkSocket::bytesToWrite(SocketType socketType, quint16 port) const { +qint64 NetworkSocket::bytesToWrite(SocketType socketType, const SockAddr& address) const { switch (socketType) { case SocketType::UDP: return _udpSocket.bytesToWrite(); #if defined(WEBRTC_DATA_CHANNELS) case SocketType::WebRTC: - return _webrtcSocket.bytesToWrite(port); + return _webrtcSocket.bytesToWrite(address); #endif default: qCCritical(networking) << "Socket type not specified in bytesToWrite()"; diff --git a/libraries/networking/src/udt/NetworkSocket.h b/libraries/networking/src/udt/NetworkSocket.h index 887e73b32ba..030f27e1197 100644 --- a/libraries/networking/src/udt/NetworkSocket.h +++ b/libraries/networking/src/udt/NetworkSocket.h @@ -80,9 +80,10 @@ class NetworkSocket : public QObject { /// @brief Gets the number of bytes waiting to be written. /// @details For UDP, there's a single buffer used for all destinations. For WebRTC, each destination has its own buffer. /// @param socketType The type of socket for which to get the number of bytes waiting to be written. - /// @param port If a WebRTC socket, the data channel for which to get the number of bytes waiting. + /// @param address If a WebRTCSocket, the destination address for which to get the number of bytes waiting. + /// @param port If a WebRTC socket, the destination port for which to get the number of bytes waiting. /// @return The number of bytes waiting to be written. - qint64 bytesToWrite(SocketType socketType, quint16 port = 0) const; + qint64 bytesToWrite(SocketType socketType, const SockAddr& address = SockAddr()) const; /// @brief Gets whether there is a pending datagram waiting to be read. diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 2aa5542651b..721eafd7e78 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -256,7 +256,7 @@ qint64 Socket::writeDatagram(const QByteArray& datagram, const SockAddr& sockAdd } qint64 bytesWritten = _networkSocket.writeDatagram(datagram, sockAddr); - int pending = _networkSocket.bytesToWrite(socketType, sockAddr.getPort()); + int pending = _networkSocket.bytesToWrite(socketType, sockAddr); if (bytesWritten < 0 || pending) { int wsaError = 0; static std::atomic previousWsaError (0); diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp index d3646508c89..769b1720dfa 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp @@ -131,13 +131,12 @@ void WDCDataChannelObserver::OnMessage(const DataBuffer& buffer) { } -WDCConnection::WDCConnection(WebRTCDataChannels* parent, quint16 webSocketID, int dataChannelID) : +WDCConnection::WDCConnection(WebRTCDataChannels* parent, const QString& dataChannelID) : _parent(parent), - _webSocketID(webSocketID), _dataChannelID(dataChannelID) { #ifdef WEBRTC_DEBUG - qCDebug(networking_webrtc) << "WDCConnection::WDCConnection() :" << webSocketID << dataChannelID; + qCDebug(networking_webrtc) << "WDCConnection::WDCConnection() :" << dataChannelID; #endif // Create observers. @@ -197,7 +196,7 @@ void WDCConnection::sendAnswer(SessionDescriptionInterface* description) { QJsonObject jsonObject; jsonObject.insert("from", QString(_parent->getNodeType())); - jsonObject.insert("to", _webSocketID); + jsonObject.insert("to", _dataChannelID); jsonObject.insert("data", jsonWebRTCPayload); _parent->sendSignalingMessage(jsonObject); @@ -251,7 +250,7 @@ void WDCConnection::sendIceCandidate(const IceCandidateInterface* candidate) { QJsonObject jsonObject; jsonObject.insert("from", QString(_parent->getNodeType())); - jsonObject.insert("to", _webSocketID); + jsonObject.insert("to", _dataChannelID); jsonObject.insert("data", jsonWebRTCData); QJsonDocument jsonDocument = QJsonDocument(jsonObject); @@ -328,7 +327,9 @@ void WDCConnection::onDataChannelMessageReceived(const DataBuffer& buffer) { #ifdef WEBRTC_DEBUG qCDebug(networking_webrtc) << "Echo message back"; #endif - _parent->sendDataMessage(_dataChannelID, byteArray); // Use parent method to exercise the code stack. + auto addressParts = _dataChannelID.split(":"); + auto address = SockAddr(SocketType::WebRTC, QHostAddress(addressParts[0]), addressParts[1].toInt()); + _parent->sendDataMessage(address, byteArray); // Use parent method to exercise the code stack. return; } @@ -420,58 +421,35 @@ WebRTCDataChannels::~WebRTCDataChannels() { } void WebRTCDataChannels::reset() { - QHashIterator i(_connectionsByDataChannel); + QHashIterator i(_connectionsByID); while (i.hasNext()) { i.next(); delete i.value(); } - _connectionsByWebSocket.clear(); - _connectionsByDataChannel.clear(); + _connectionsByID.clear(); } -quint16 WebRTCDataChannels::getNewDataChannelID() { - // The first data channel ID is 1. - static const int QUINT16_LIMIT = std::numeric_limits::max() + 1; - _lastDataChannelID = std::max((_lastDataChannelID + 1) % QUINT16_LIMIT, 1); -#ifdef WEBRTC_DEBUG - qCDebug(networking_webrtc) << "WebRTCDataChannels::getNewDataChannelID() :" << _lastDataChannelID; -#endif - return _lastDataChannelID; -} - -int WebRTCDataChannels::getDataChannelIDForWebSocket(quint16 webSocketID) const { -#ifdef WEBRTC_DEBUG - qCDebug(networking_webrtc) << "WebRTCDataChannels::getDataChannelIDForWebSocket() :" << webSocketID; -#endif - auto connection = _connectionsByWebSocket.value(webSocketID); - if (!connection) { - return 0; - } - return connection->getDataChannelID(); -} - -void WebRTCDataChannels::onDataChannelOpened(WDCConnection* connection, quint16 dataChannelID) { +void WebRTCDataChannels::onDataChannelOpened(WDCConnection* connection, const QString& dataChannelID) { #ifdef WEBRTC_DEBUG qCDebug(networking_webrtc) << "WebRTCDataChannels::onDataChannelOpened() :" << dataChannelID; #endif - _connectionsByDataChannel.insert(dataChannelID, connection); + _connectionsByID.insert(dataChannelID, connection); } void WebRTCDataChannels::onSignalingMessage(const QJsonObject& message) { #ifdef WEBRTC_DEBUG - qCDebug(networking_webrtc) << "WebRTCDataChannel::onSignalingMessage()" << message << message.value("channel"); + qCDebug(networking_webrtc) << "WebRTCDataChannel::onSignalingMessage()" << message; #endif // Validate message. const int MAX_DEBUG_DETAIL_LENGTH = 64; + const QRegularExpression DATA_CHANNEL_ID_REGEX{ "^[1-9]\\d*\\.\\d+\\.\\d+\\.\\d+:\\d+$" }; auto data = message.value("data").isObject() ? message.value("data").toObject() : QJsonObject(); - int from = message.value("from").isDouble() ? (quint16)(message.value("from").toInt()) : 0; + auto from = message.value("from").toString(); auto to = NodeType::fromChar(message.value("to").toString().at(0)); - int channel = message.value("channel").isDouble() ? (int)(message.value("channel").toInt()) : 0; - - if (from <= 0 || from > MAXUINT16 || to == NodeType::Unassigned || channel < 0 || channel > MAXUINT16 + if (!DATA_CHANNEL_ID_REGEX.match(from).hasMatch() || to == NodeType::Unassigned || !data.contains("description") && !data.contains("candidate")) { - qCWarning(networking_webrtc) << "Unexpected signaling message:" + qCWarning(networking_webrtc) << "Invalid or unexpected signaling message:" << QJsonDocument(message).toJson(QJsonDocument::Compact).left(MAX_DEBUG_DETAIL_LENGTH); return; } @@ -481,16 +459,11 @@ void WebRTCDataChannels::onSignalingMessage(const QJsonObject& message) { // Find or create a connection. WDCConnection* connection; - if (_connectionsByWebSocket.contains(from)) { - connection = _connectionsByWebSocket.value(from); + if (_connectionsByID.contains(from)) { + connection = _connectionsByID.value(from); } else { - // Assignment clients use the same data channel ID as the domain server, which is provided in the "channel" property. - // The domain server creates a new data channel ID. - if (channel == 0) { - channel = getNewDataChannelID(); - } - connection = new WDCConnection(this, from, channel); - _connectionsByWebSocket.insert(from, connection); + connection = new WDCConnection(this, from); + _connectionsByID.insert(from, connection); } // Set the remote description and reply with an answer. @@ -519,41 +492,41 @@ void WebRTCDataChannels::sendSignalingMessage(const QJsonObject& message) { emit signalingMessage(message); } -void WebRTCDataChannels::emitDataMessage(int dataChannelID, const QByteArray& byteArray) { +void WebRTCDataChannels::emitDataMessage(const QString& dataChannelID, const QByteArray& byteArray) { #ifdef WEBRTC_DEBUG qCDebug(networking_webrtc) << "WebRTCDataChannels::emitDataMessage() :" << dataChannelID << byteArray.toHex() << byteArray.length(); #endif - emit dataMessage(dataChannelID, byteArray); + auto addressParts = dataChannelID.split(":"); + auto address = SockAddr(SocketType::WebRTC, QHostAddress(addressParts[0]), addressParts[1].toInt()); + emit dataMessage(address, byteArray); } -bool WebRTCDataChannels::sendDataMessage(int dataChannelID, const QByteArray& byteArray) { +bool WebRTCDataChannels::sendDataMessage(const SockAddr& destination, const QByteArray& byteArray) { + auto dataChannelID = destination.toShortString(); #ifdef WEBRTC_DEBUG qCDebug(networking_webrtc) << "WebRTCDataChannels::sendDataMessage() :" << dataChannelID; #endif - // Find connection. - if (!_connectionsByDataChannel.contains(dataChannelID)) { + if (!_connectionsByID.contains(dataChannelID)) { qCWarning(networking_webrtc) << "Could not find WebRTC data channel to send message on!"; return false; } - auto connection = _connectionsByDataChannel.value(dataChannelID); + auto connection = _connectionsByID.value(dataChannelID); DataBuffer buffer(byteArray.toStdString(), true); return connection->sendDataMessage(buffer); } -/// @brief Gets the number of bytes waiting to be written on a data channel. -/// @param port The data channel ID. -/// @return The number of bytes waiting to be written on the data channel; 0 if the channel doesn't exist. -qint64 WebRTCDataChannels::getBufferedAmount(int dataChannelID) const { - if (!_connectionsByDataChannel.contains(dataChannelID)) { +qint64 WebRTCDataChannels::getBufferedAmount(const SockAddr& address) const { + auto dataChannelID = address.toShortString(); + if (!_connectionsByID.contains(dataChannelID)) { #ifdef WEBRTC_DEBUG qCDebug(networking_webrtc) << "WebRTCDataChannels::getBufferedAmount() : Channel doesn't exist:" << dataChannelID; #endif return 0; } - auto connection = _connectionsByDataChannel.value(dataChannelID); + auto connection = _connectionsByID.value(dataChannelID); return connection->getBufferedAmount(); } @@ -600,10 +573,9 @@ void WebRTCDataChannels::closePeerConnectionNow(WDCConnection* connection) { // Delete the WDCConnection. #ifdef WEBRTC_DEBUG - qCDebug(networking_webrtc) << "Dispose of connection for channel ID:" << connection->getDataChannelID(); + qCDebug(networking_webrtc) << "Dispose of connection for channel:" << connection->getDataChannelID(); #endif - _connectionsByWebSocket.remove(connection->getWebSocketID()); - _connectionsByDataChannel.remove(connection->getDataChannelID()); + _connectionsByID.remove(connection->getDataChannelID()); delete connection; #ifdef WEBRTC_DEBUG qCDebug(networking_webrtc) << "Disposed of connection"; diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.h b/libraries/networking/src/webrtc/WebRTCDataChannels.h index dc8152ee348..fe8af770789 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.h +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.h @@ -22,6 +22,7 @@ #define emit #include "../NodeType.h" +#include "../SockAddr.h" class WebRTCDataChannels; class WDCConnection; @@ -128,17 +129,12 @@ class WDCConnection { /// @brief Constructs a new WDCConnection and opens a WebRTC data connection. /// @param parent The parent WebRTCDataChannels object. - /// @param webSocketID The signaling channel that initiated the opening of the WebRTC data channel. - /// @param dataChannelID - The WebRTC data channel ID to assign to this connection. - WDCConnection(WebRTCDataChannels* parent, quint16 webSocketID, int dataChannelID); - - /// @brief Gets the WebSocket ID. - /// @return The ID of the WebSocket. - quint16 getWebSocketID() const { return _webSocketID; } + /// @param dataChannelID The data channel ID. + WDCConnection(WebRTCDataChannels* parent, const QString& dataChannelID); - /// @brief Gets the WebRTC data channel ID. - /// @return The WebRTC data channel ID. `-1` if not open yet. - int getDataChannelID() const { return _dataChannelID; } + /// @brief Gets the data channel ID. + /// @return The data channel ID. + QString getDataChannelID() const { return _dataChannelID; } /// @brief Sets the remote session description received from the remote client via the signaling channel. @@ -160,7 +156,7 @@ class WDCConnection { /// @param data The ICE candidate. void addIceCandidate(QJsonObject& data); - /// @brief Sends an ICE candidate to the remote vlient via the signaling channel. + /// @brief Sends an ICE candidate to the remote client via the signaling channel. /// @param candidate The ICE candidate. void sendIceCandidate(const webrtc::IceCandidateInterface* candidate); @@ -195,8 +191,7 @@ class WDCConnection { private: WebRTCDataChannels* _parent; - quint16 _webSocketID { 0 }; - int _dataChannelID { -1 }; + QString _dataChannelID; rtc::scoped_refptr _setSessionDescriptionObserver { nullptr }; rtc::scoped_refptr _createSessionDescriptionObserver { nullptr }; @@ -221,6 +216,9 @@ class WDCConnection { /// Additionally, for debugging purposes, instead of containing a Vircadia protocol payload, a WebRTC message may be an echo /// request. This is bounced back to the client. /// +/// A WebRTC data channel is identified by the IP address and port of the client WebSocket that was used when opening the data +/// channel - this is considered to be the WebRTC data channel's address. The IP address and port of the actual WebRTC +/// connection is not used. class WebRTCDataChannels : public QObject { Q_OBJECT @@ -242,41 +240,30 @@ class WebRTCDataChannels : public QObject { /// @brief Immediately closes all connections and resets the socket. void reset(); - /// @brief Gets a new data channel ID to uniquely identify a WDCConnection. - /// @details This ID is assigned by WebRTCDataChannels; it is not the WebRTC data channel ID because that is only - /// unique within a peer connection. - /// @return A new data channel ID. - quint16 getNewDataChannelID(); - - /// @brief Gets the data channel ID associated with a WebSocket. - /// @param webSocketID The WebSocket. - /// @return The data channel ID associated with the WebSocket if found, `0` if the WebSocket was not found. - int getDataChannelIDForWebSocket(quint16 webSocketID) const; - /// @brief Handles a WebRTC data channel opening. /// @param connection The WebRTC data channel connection. - /// @param dataChannelID The WebRTC data channel ID. - void onDataChannelOpened(WDCConnection* connection, quint16 dataChannelID); + /// @param dataChannelID The IP address and port of the signaling WebSocket that the client used to connect, `"n.n.n.n:n"`. + void onDataChannelOpened(WDCConnection* connection, const QString& dataChannelID); /// @brief Emits a signalingMessage to be sent to the Interface client. /// @param message The WebRTC signaling message to send. void sendSignalingMessage(const QJsonObject& message); /// @brief Emits a dataMessage received from the Interface client. - /// @param dataChannelID The WebRTC data channel the message was received on. + /// @param dataChannelID The IP address and port of the signaling WebSocket that the client used to connect, `"n.n.n.n:n"`. /// @param byteArray The data message received. - void emitDataMessage(int dataChannelID, const QByteArray& byteArray); + void emitDataMessage(const QString& dataChannelID, const QByteArray& byteArray); /// @brief Sends a data message to an Interface client. - /// @param dataChannelID The WebRTC channel ID of the Interface client. + /// @param dataChannelID The IP address and port of the signaling WebSocket that the client used to connect, `"n.n.n.n:n"`. /// @param message The data message to send. /// @return `true` if the data message was sent, otherwise `false`. - bool sendDataMessage(int dataChannelID, const QByteArray& message); + bool sendDataMessage(const SockAddr& destination, const QByteArray& message); /// @brief Gets the number of bytes waiting to be sent on a data channel. - /// @param dataChannelID The data channel ID. + /// @param address The address of the signaling WebSocket that the client used to connect. /// @return The number of bytes waiting to be sent on the data channel. - qint64 getBufferedAmount(int dataChannelID) const; + qint64 getBufferedAmount(const SockAddr& address) const; /// @brief Creates a new WebRTC peer connection for connecting to an Interface client. /// @param peerConnectionObserver An observer to monitor the WebRTC peer connection. @@ -311,9 +298,9 @@ public slots: /// @brief A WebRTC data message received from the Interface client. /// @details This message is for handling at a higher level in the Vircadia protocol. - /// @param dataChannelID The WebRTC data channel ID. + /// @param address The address of the signaling WebSocket that the client used to connect. /// @param byteArray The Vircadia protocol message. - void dataMessage(int dataChannelID, const QByteArray& byteArray); + void dataMessage(const SockAddr& address, const QByteArray& byteArray); /// @brief Signals that the peer connection for a WebRTC data channel should be closed. /// @details Used by {@link WebRTCDataChannels.closePeerConnection}. @@ -332,10 +319,9 @@ public slots: rtc::scoped_refptr _peerConnectionFactory { nullptr }; - quint16 _lastDataChannelID { 0 }; - - QHash _connectionsByWebSocket; - QHash _connectionsByDataChannel; + QHash _connectionsByID; // + // The client's WebSocket IP and port is used as the data channel ID to uniquely identify each. + // The WebSocket IP address and port is formatted as "n.n.n.n:n", the same as used in WebRTCSignalingServer. }; diff --git a/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp b/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp index a11d09ea41e..b35cd5158b5 100644 --- a/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp +++ b/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp @@ -62,7 +62,8 @@ void WebRTCSignalingServer::webSocketTextMessageReceived(const QString& message) source->sendTextMessage(echo); } else { // WebRTC message or assignment client echo request. (Send both to target.) - json.insert("from", source->peerPort()); + auto from = source->peerAddress().toString() + ":" + QString::number(source->peerPort()); + json.insert("from", from); emit messageReceived(json); } } else { @@ -71,9 +72,9 @@ void WebRTCSignalingServer::webSocketTextMessageReceived(const QString& message) } void WebRTCSignalingServer::sendMessage(const QJsonObject& message) { - quint16 destinationPort = message.value("to").toInt(); - if (_webSockets.contains(destinationPort)) { - _webSockets.value(destinationPort)->sendTextMessage(QString(QJsonDocument(message).toJson())); + auto destinationAddress = message.value("to").toString(); + if (_webSockets.contains(destinationAddress)) { + _webSockets.value(destinationAddress)->sendTextMessage(QString(QJsonDocument(message).toJson())); } else { qCWarning(networking_webrtc) << "Failed to find WebSocket for outgoing WebRTC signaling message."; } @@ -82,7 +83,9 @@ void WebRTCSignalingServer::sendMessage(const QJsonObject& message) { void WebRTCSignalingServer::webSocketDisconnected() { auto source = qobject_cast(sender()); if (source) { - _webSockets.remove(source->peerPort()); + auto address = source->peerAddress().toString() + ":" + QString::number(source->peerPort()); + delete _webSockets.value(address); + _webSockets.remove(address); } } @@ -90,7 +93,8 @@ void WebRTCSignalingServer::newWebSocketConnection() { auto webSocket = _webSocketServer->nextPendingConnection(); connect(webSocket, &QWebSocket::textMessageReceived, this, &WebRTCSignalingServer::webSocketTextMessageReceived); connect(webSocket, &QWebSocket::disconnected, this, &WebRTCSignalingServer::webSocketDisconnected); - _webSockets.insert(webSocket->peerPort(), webSocket); + auto webSocketAddress = webSocket->peerAddress().toString() + ":" + QString::number(webSocket->peerPort()); + _webSockets.insert(webSocketAddress, webSocket); } #endif // WEBRTC_DATA_CHANNELS diff --git a/libraries/networking/src/webrtc/WebRTCSignalingServer.h b/libraries/networking/src/webrtc/WebRTCSignalingServer.h index 418becd8ebf..fb695cc6fcc 100644 --- a/libraries/networking/src/webrtc/WebRTCSignalingServer.h +++ b/libraries/networking/src/webrtc/WebRTCSignalingServer.h @@ -39,19 +39,20 @@ /// signaling `data` payload or an `echo` request: /// /// | Interface -> Server || -/// | -------- | -----------------------| -/// | `to` | NodeType | -/// | `from` | WebSocket port number* | -/// | [`data`] | WebRTC payload | -/// | [`echo`] | Echo request | -/// * The `from` field is filled in by the WebRTCSignalingServer. +/// | -------- | ---------------------------------------- | +/// | `to` | NodeType | +/// | `from` | WebSocket IP address & port, "n.n.n.n:n" | +/// | [`data`] | WebRTC signaling payload | +/// | [`echo`] | Echo request | +/// +/// `*` The `from` field is filled in upon receipt by the WebRTCSignalingServer. /// /// | Server -> Interface || -/// | -------- | --------------------- | -/// | `to` | WebSocket port number | -/// | `from` | NodeType | -/// | [`data`] | WebRTC payload | -/// | [`echo`] | Echo response | +/// | -------- | ---------------------------------------- | +/// | `to` | WebSocket IP address & port, "n.n.n.n:n" | +/// | `from` | NodeType | +/// | [`data`] | WebRTC signaling payload | +/// | [`echo`] | Echo response | /// class WebRTCSignalingServer : public QObject { Q_OBJECT @@ -97,7 +98,9 @@ private slots: QHostAddress _address; quint16 _port { 0 }; - QHash _webSockets; // client WebSocket port, client WebSocket object + QHash _webSockets; // + // The WebSocket IP address and port is formatted as "n.n.n.n:n". + // A QString is used rather than a SockAddr, to make signaling easier. QTimer* _isWebSocketServerListeningTimer; }; diff --git a/libraries/networking/src/webrtc/WebRTCSocket.cpp b/libraries/networking/src/webrtc/WebRTCSocket.cpp index 589d46a76ea..e34dc5d5a24 100644 --- a/libraries/networking/src/webrtc/WebRTCSocket.cpp +++ b/libraries/networking/src/webrtc/WebRTCSocket.cpp @@ -78,17 +78,17 @@ void WebRTCSocket::abort() { } -qint64 WebRTCSocket::writeDatagram(const QByteArray& datagram, quint16 port) { +qint64 WebRTCSocket::writeDatagram(const QByteArray& datagram, const SockAddr& destination) { clearError(); - if (_dataChannels.sendDataMessage(port, datagram)) { + if (_dataChannels.sendDataMessage(destination, datagram)) { return datagram.length(); } setError(QAbstractSocket::SocketError::UnknownSocketError, "Failed to write datagram"); return -1; } -qint64 WebRTCSocket::bytesToWrite(quint16 port) const { - return _dataChannels.getBufferedAmount(port); +qint64 WebRTCSocket::bytesToWrite(const SockAddr& destination) const { + return _dataChannels.getBufferedAmount(destination); } @@ -114,12 +114,11 @@ qint64 WebRTCSocket::readDatagram(char* data, qint64 maxSize, QHostAddress* addr } if (address) { - // WEBRTC TODO: Use signaling channel's remote WebSocket address? Or remote data channel address? - *address = QHostAddress::AnyIPv4; + *address = datagram.first.getAddress(); } if (port) { - *port = datagram.first; + *port = datagram.first.getPort(); } return length; @@ -148,14 +147,9 @@ void WebRTCSocket::clearError() { } -void WebRTCSocket::onDataChannelReceivedMessage(int dataChannelID, const QByteArray& message) { - _receivedQueue.enqueue(QPair(dataChannelID, message)); +void WebRTCSocket::onDataChannelReceivedMessage(const SockAddr& source, const QByteArray& message) { + _receivedQueue.enqueue(QPair(source, message)); emit readyRead(); } - -int WebRTCSocket::getDataChannelIDForWebSocket(quint16 webSocketID) const { - return _dataChannels.getDataChannelIDForWebSocket(webSocketID); -} - #endif // WEBRTC_DATA_CHANNELS diff --git a/libraries/networking/src/webrtc/WebRTCSocket.h b/libraries/networking/src/webrtc/WebRTCSocket.h index 53c8921f023..04856f50f1f 100644 --- a/libraries/networking/src/webrtc/WebRTCSocket.h +++ b/libraries/networking/src/webrtc/WebRTCSocket.h @@ -24,6 +24,10 @@ /// @brief Provides a QUdpSocket-style interface for using WebRTCDataChannels. +/// +/// @details A WebRTC data channel is identified by the IP address and port of the client WebSocket that was used when opening +/// the data channel - this is considered to be the WebRTC data channel's address. The IP address and port of the actual WebRTC +/// connection is not used. class WebRTCSocket : public QObject { Q_OBJECT @@ -81,16 +85,16 @@ class WebRTCSocket : public QObject { qintptr socketDescriptor() const { return -1; } - /// @brief Sends a datagram to the host on a data channel. + /// @brief Sends a datagram. /// @param datagram The datagram to send. - /// @param port The data channel ID. + /// @param destination The destination WebRTC data channel address. /// @return The number of bytes if successfully sent, otherwise -1. - qint64 writeDatagram(const QByteArray& datagram, quint16 port); + qint64 writeDatagram(const QByteArray& datagram, const SockAddr& destination); /// @brief Gets the number of bytes waiting to be written. - /// @param port The data channel ID. + /// @param destination The destination WebRTC data channel address. /// @return The number of bytes waiting to be written. - qint64 bytesToWrite(quint16 port) const; + qint64 bytesToWrite(const SockAddr& destination) const; /// @brief Gets whether there's a datagram waiting to be read. /// @return true if there's a datagram waiting to be read, false if there isn't. @@ -104,8 +108,8 @@ class WebRTCSocket : public QObject { /// @details Any remaining data in the datagram is lost. /// @param data The destination to read the datagram into. /// @param maxSize The maximum number of bytes to read. - /// @param address The destination to put the IP address that the datagram was read from. (Not currently set.) - /// @param port The destination to put the data channel ID that the datagram was read from. + /// @param address The destination to put the WebRTC data channel's IP address. + /// @param port The destination to put the WebRTC data channel's port. /// @return The number of bytes read on success; -1 if reading unsuccessful. qint64 readDatagram(char* data, qint64 maxSize, QHostAddress* address = nullptr, quint16* port = nullptr); @@ -118,19 +122,13 @@ class WebRTCSocket : public QObject { /// @return The description of the error that last occurred. QString errorString() const; - - /// @brief Gets the data channel ID associated with a WebSocket. - /// @param webSocketID - /// @return The data channel ID associated with the WebSocket if found, `0` if the WebSocket was not found. - int getDataChannelIDForWebSocket(quint16 webSocketID) const; - public slots: /// @brief Handles the WebRTC data channel receiving a message. /// @details Queues the message to be read via readDatagram. - /// @param dataChannelID The data channel that the message was received on. + /// @param source The WebRTC data channel that the message was received on. /// @param message The message that was received. - void onDataChannelReceivedMessage(int dataChannelID, const QByteArray& message); + void onDataChannelReceivedMessage(const SockAddr& source, const QByteArray& message); signals: @@ -159,7 +157,7 @@ public slots: bool _isBound { false }; - QQueue> _receivedQueue; // Messages received are queued for reading from the "socket". + QQueue> _receivedQueue; // Messages received are queued for reading from the "socket". QAbstractSocket::SocketError _lastErrorType { QAbstractSocket::UnknownSocketError }; QString _lastErrorString; From 87803e512414efcd19983a9ae381f46c45ce5301 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sun, 5 Sep 2021 22:55:24 +1200 Subject: [PATCH 52/82] Use client WebSocket address as WebRTC address --- domain-server/src/NodeConnectionData.cpp | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/domain-server/src/NodeConnectionData.cpp b/domain-server/src/NodeConnectionData.cpp index 6e0c8427ba1..d7d813e3df3 100644 --- a/domain-server/src/NodeConnectionData.cpp +++ b/domain-server/src/NodeConnectionData.cpp @@ -17,7 +17,7 @@ NodeConnectionData NodeConnectionData::fromDataStream(QDataStream& dataStream, const SockAddr& senderSockAddr, bool isConnectRequest) { NodeConnectionData newHeader; - + if (isConnectRequest) { dataStream >> newHeader.connectUUID; @@ -58,16 +58,21 @@ NodeConnectionData NodeConnectionData::fromDataStream(QDataStream& dataStream, c newHeader.publicSockAddr.setType(publicSocketType); newHeader.localSockAddr.setType(localSocketType); - // For WebRTC connections, the user client doesn't know the WebRTC data channel ID that the domain server is using as its - // SockAddr port, so set the port values here. + // For WebRTC connections, the user client's signaling channel WebSocket address is used instead of the actual data + // channel's address. if (senderSockAddr.getType() == SocketType::WebRTC) { if (newHeader.publicSockAddr.getType() != SocketType::WebRTC || newHeader.localSockAddr.getType() != SocketType::WebRTC) { qDebug() << "Inconsistent WebRTC socket types!"; } - newHeader.publicSockAddr.setPort(senderSockAddr.getPort()); // We don't know whether it's a public or local connection - newHeader.localSockAddr.setPort(senderSockAddr.getPort()); // so set both ports. + // We don't know whether it's a public or local connection so set both the same. + auto address = senderSockAddr.getAddress(); + auto port = senderSockAddr.getPort(); + newHeader.publicSockAddr.setAddress(address); + newHeader.publicSockAddr.setPort(port); + newHeader.localSockAddr.setAddress(address); + newHeader.localSockAddr.setPort(port); } newHeader.senderSockAddr = senderSockAddr; From 6c4e105c068ad39dd1c8d0c4ec71ed5dbdf28da7 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sat, 18 Sep 2021 10:01:21 +1200 Subject: [PATCH 53/82] Remove some unused audio stream method parameters --- libraries/audio/src/InboundAudioStream.cpp | 4 ++-- libraries/audio/src/InboundAudioStream.h | 2 +- libraries/audio/src/MixedProcessedAudioStream.cpp | 2 +- libraries/audio/src/MixedProcessedAudioStream.h | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/libraries/audio/src/InboundAudioStream.cpp b/libraries/audio/src/InboundAudioStream.cpp index 7a81b8a67ae..e946c6afb7e 100644 --- a/libraries/audio/src/InboundAudioStream.cpp +++ b/libraries/audio/src/InboundAudioStream.cpp @@ -167,7 +167,7 @@ int InboundAudioStream::parseData(ReceivedMessage& message) { bool packetPCM = codecInPacket == "pcm" || codecInPacket == ""; if (codecInPacket == _selectedCodecName || (packetPCM && selectedPCM)) { auto afterProperties = message.readWithoutCopy(message.getBytesLeftToRead()); - parseAudioData(message.getType(), afterProperties); + parseAudioData(afterProperties); _mismatchedAudioCodecCount = 0; } else { @@ -267,7 +267,7 @@ int InboundAudioStream::lostAudioData(int numPackets) { return 0; } -int InboundAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) { +int InboundAudioStream::parseAudioData(const QByteArray& packetAfterStreamProperties) { QByteArray decodedBuffer; // may block on the real-time thread, which is acceptible as diff --git a/libraries/audio/src/InboundAudioStream.h b/libraries/audio/src/InboundAudioStream.h index c10a86cb693..b42609d5769 100644 --- a/libraries/audio/src/InboundAudioStream.h +++ b/libraries/audio/src/InboundAudioStream.h @@ -132,7 +132,7 @@ public slots: /// parses the audio data in the network packet. /// default implementation assumes packet contains raw audio samples after stream properties - virtual int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties); + virtual int parseAudioData(const QByteArray& packetAfterStreamProperties); /// produces audio data for lost network packets. virtual int lostAudioData(int numPackets); diff --git a/libraries/audio/src/MixedProcessedAudioStream.cpp b/libraries/audio/src/MixedProcessedAudioStream.cpp index 6510f0bfc98..d1312edb956 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.cpp +++ b/libraries/audio/src/MixedProcessedAudioStream.cpp @@ -61,7 +61,7 @@ int MixedProcessedAudioStream::lostAudioData(int numPackets) { return 0; } -int MixedProcessedAudioStream::parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) { +int MixedProcessedAudioStream::parseAudioData(const QByteArray& packetAfterStreamProperties) { QByteArray decodedBuffer; // may block on the real-time thread, which is acceptible as diff --git a/libraries/audio/src/MixedProcessedAudioStream.h b/libraries/audio/src/MixedProcessedAudioStream.h index 14da1d45af0..5732f32e902 100644 --- a/libraries/audio/src/MixedProcessedAudioStream.h +++ b/libraries/audio/src/MixedProcessedAudioStream.h @@ -34,7 +34,7 @@ class MixedProcessedAudioStream : public InboundAudioStream { protected: int writeDroppableSilentFrames(int silentFrames) override; - int parseAudioData(PacketType type, const QByteArray& packetAfterStreamProperties) override; + int parseAudioData(const QByteArray& packetAfterStreamProperties) override; int lostAudioData(int numPackets) override; private: From de37734e18356d75a015822b6bf595470bf8d7d8 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Fri, 1 Oct 2021 09:09:07 +1300 Subject: [PATCH 54/82] Tidying to match the Web SDK --- libraries/audio-client/src/AudioClient.cpp | 21 ++++++++++----------- libraries/audio-client/src/AudioClient.h | 4 ++-- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index c25dd61f688..285549734c4 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -942,7 +942,7 @@ void AudioClient::Gate::flush() { void AudioClient::handleNoisyMutePacket(QSharedPointer message) { - if (!_muted) { + if (!_isMuted) { setMuted(true); // have the audio scripting interface emit a signal to say we were muted by the mixer @@ -989,7 +989,7 @@ void AudioClient::selectAudioFormat(const QString& selectedCodecName) { _selectedCodecName = selectedCodecName; - qCDebug(audioclient) << "Selected Codec:" << _selectedCodecName << "isStereoInput:" << _isStereoInput; + qCDebug(audioclient) << "Selected codec:" << _selectedCodecName << "; Is stereo input:" << _isStereoInput; // release any old codec encoder/decoder first... if (_codec && _encoder) { @@ -1005,7 +1005,7 @@ void AudioClient::selectAudioFormat(const QString& selectedCodecName) { _codec = plugin; _receivedAudioStream.setupCodec(plugin, _selectedCodecName, AudioConstants::STEREO); _encoder = plugin->createEncoder(AudioConstants::SAMPLE_RATE, _isStereoInput ? AudioConstants::STEREO : AudioConstants::MONO); - qCDebug(audioclient) << "Selected Codec Plugin:" << _codec.get(); + qCDebug(audioclient) << "Selected codec plugin:" << _codec.get(); break; } } @@ -1269,7 +1269,7 @@ void AudioClient::processWebrtcNearEnd(int16_t* samples, int numFrames, int numC void AudioClient::handleLocalEchoAndReverb(QByteArray& inputByteArray) { // If there is server echo, reverb will be applied to the recieved audio stream so no need to have it here. bool hasReverb = _reverb || _receivedAudioStream.hasReverb(); - if ((_muted && !_shouldEchoLocally) || !_audioOutput || (!_shouldEchoLocally && !hasReverb) || !_audioGateOpen) { + if ((_isMuted && !_shouldEchoLocally) || !_audioOutput || (!_shouldEchoLocally && !hasReverb) || !_audioGateOpen) { return; } @@ -1357,7 +1357,7 @@ void AudioClient::handleAudioInput(QByteArray& audioBuffer) { bool audioGateOpen = false; - if (!_muted) { + if (!_isMuted) { int16_t* samples = reinterpret_cast(audioBuffer.data()); int numSamples = audioBuffer.size() / AudioConstants::SAMPLE_SIZE; int numFrames = numSamples / (_isStereoInput ? AudioConstants::STEREO : AudioConstants::MONO); @@ -1378,7 +1378,7 @@ void AudioClient::handleAudioInput(QByteArray& audioBuffer) { } // loudness after mute/gate - _lastInputLoudness = (_muted || !audioGateOpen) ? 0.0f : _lastRawInputLoudness; + _lastInputLoudness = (_isMuted || !audioGateOpen) ? 0.0f : _lastRawInputLoudness; // detect gate opening and closing bool openedInLastBlock = !_audioGateOpen && audioGateOpen; // the gate just opened @@ -1482,7 +1482,7 @@ void AudioClient::handleMicAudioInput() { emit inputLoudnessChanged(_lastSmoothedRawInputLoudness, isClipping); - if (!_muted) { + if (!_isMuted) { possibleResampling(_inputToNetworkResampler, inputAudioSamples.get(), networkAudioSamples, inputSamplesRequired, numNetworkSamples, @@ -1748,10 +1748,10 @@ void AudioClient::sendMuteEnvironmentPacket() { } void AudioClient::setMuted(bool muted, bool emitSignal) { - if (_muted != muted) { - _muted = muted; + if (_isMuted != muted) { + _isMuted = muted; if (emitSignal) { - emit muteToggled(_muted); + emit muteToggled(_isMuted); } } } @@ -1896,7 +1896,6 @@ bool AudioClient::switchInputToAudioDevice(const HifiAudioDeviceInfo inputDevice if (_dummyAudioInput) { _dummyAudioInput->stop(); - _dummyAudioInput->deleteLater(); _dummyAudioInput = NULL; } diff --git a/libraries/audio-client/src/AudioClient.h b/libraries/audio-client/src/AudioClient.h index c414d74b6a9..51e0ecd2ee8 100644 --- a/libraries/audio-client/src/AudioClient.h +++ b/libraries/audio-client/src/AudioClient.h @@ -217,7 +217,7 @@ public slots: void audioMixerKilled(); void setMuted(bool muted, bool emitSignal = true); - bool isMuted() { return _muted; } + bool isMuted() { return _isMuted; } virtual bool setIsStereoInput(bool stereo) override; virtual bool isStereoInput() override { return _isStereoInput; } @@ -410,7 +410,7 @@ public slots: float _timeSinceLastClip{ -1.0f }; int _totalInputAudioSamples; - bool _muted{ false }; + bool _isMuted{ false }; bool _shouldEchoLocally{ false }; bool _shouldEchoToServer{ false }; bool _isNoiseGateEnabled{ true }; From ada8dace5dc71e7c831a3b7fac0a3eab595ece49 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sun, 3 Oct 2021 12:52:11 +1300 Subject: [PATCH 55/82] Fix typo noticed in passing --- cmake/ports/webrtc/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/ports/webrtc/README.md b/cmake/ports/webrtc/README.md index 3db73dce96f..a4053b679e4 100644 --- a/cmake/ports/webrtc/README.md +++ b/cmake/ports/webrtc/README.md @@ -17,7 +17,7 @@ WebRTC Information: ## Windows - M84 -WebRTC's M84 release is currently used because it corresponded to Microsoft's latest WinRTC release at the time of develeopment, +WebRTC's M84 release is currently used because it corresponded to Microsoft's latest WinRTC release at the time of development, and WinRTC is a source of potentially useful patches. The following notes document how the M84-based Windows VCPKG was created, using Visual Studio 2019. From f5452a43245547856796309da194dab1f0290845 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Thu, 7 Oct 2021 21:03:23 +1300 Subject: [PATCH 56/82] Guard against invalid data channel address --- libraries/networking/src/webrtc/WebRTCDataChannels.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp index 769b1720dfa..cbc5a8ff6e8 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp @@ -328,6 +328,10 @@ void WDCConnection::onDataChannelMessageReceived(const DataBuffer& buffer) { qCDebug(networking_webrtc) << "Echo message back"; #endif auto addressParts = _dataChannelID.split(":"); + if (addressParts.length() != 2) { + qCWarning(networking_webrtc) << "Invalid dataChannelID:" << _dataChannelID; + return; + } auto address = SockAddr(SocketType::WebRTC, QHostAddress(addressParts[0]), addressParts[1].toInt()); _parent->sendDataMessage(address, byteArray); // Use parent method to exercise the code stack. return; @@ -498,6 +502,10 @@ void WebRTCDataChannels::emitDataMessage(const QString& dataChannelID, const QBy << byteArray.length(); #endif auto addressParts = dataChannelID.split(":"); + if (addressParts.length() != 2) { + qCWarning(networking_webrtc) << "Invalid dataChannelID:" << dataChannelID; + return; + } auto address = SockAddr(SocketType::WebRTC, QHostAddress(addressParts[0]), addressParts[1].toInt()); emit dataMessage(address, byteArray); } From f15d87fff1527b7e21d7833722fe1063a08a24cd Mon Sep 17 00:00:00 2001 From: David Rowe Date: Thu, 14 Oct 2021 13:38:30 +1300 Subject: [PATCH 57/82] Remove unused code --- libraries/networking/src/udt/Connection.cpp | 9 --------- libraries/networking/src/udt/Connection.h | 3 --- 2 files changed, 12 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 66bf3f66134..4c05e8af91c 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -417,9 +417,6 @@ void Connection::resetReceiveState() { // clear the loss list _lossList.clear(); - // clear sync variables - _connectionStart = p_high_resolution_clock::now(); - // clear any pending received messages for (auto& pendingMessage : _pendingReceivedMessages) { _parentSocket->messageFailed(this, pendingMessage.first); @@ -451,12 +448,6 @@ void PendingReceivedMessage::enqueuePacket(std::unique_ptr packet) { "PendingReceivedMessage::enqueuePacket", "called with a packet that is not part of a message"); - if (packet->getPacketPosition() == Packet::PacketPosition::LAST || - packet->getPacketPosition() == Packet::PacketPosition::ONLY) { - _hasLastPacket = true; - _numPackets = packet->getMessagePartNumber() + 1; - } - // Insert into the packets list in sorted order. Because we generally expect to receive packets in order, begin // searching from the end of the list. auto messagePartNumber = packet->getMessagePartNumber(); diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index 2869b7f57de..a7904cba233 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -43,9 +43,7 @@ class PendingReceivedMessage { std::list> _packets; private: - bool _hasLastPacket { false }; Packet::MessagePartNumber _nextPartNumber = 0; - unsigned int _numPackets { 0 }; }; class Connection : public QObject { @@ -112,7 +110,6 @@ private slots: bool _hasReceivedHandshakeACK { false }; // flag for receipt of handshake ACK from client bool _didRequestHandshake { false }; // flag for request of handshake from server - p_high_resolution_clock::time_point _connectionStart = p_high_resolution_clock::now(); // holds the time_point for creation of this connection p_high_resolution_clock::time_point _lastReceiveTime; // holds the last time we received anything from sender SequenceNumber _initialSequenceNumber; // Randomized on Connection creation, identifies connection during re-connect requests From 8e653c3a65ce9c5b213cfc323a33b36c9b279482 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Thu, 14 Oct 2021 21:59:46 +1300 Subject: [PATCH 58/82] Fix assertion message --- libraries/networking/src/udt/ControlPacket.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/udt/ControlPacket.cpp b/libraries/networking/src/udt/ControlPacket.cpp index b56f455030a..4dced78d0e2 100644 --- a/libraries/networking/src/udt/ControlPacket.cpp +++ b/libraries/networking/src/udt/ControlPacket.cpp @@ -98,10 +98,11 @@ void ControlPacket::writeType() { void ControlPacket::readType() { ControlBitAndType bitAndType = *reinterpret_cast(_packet.get()); - Q_ASSERT_X(bitAndType & CONTROL_BIT_MASK, "ControlPacket::readHeader()", "This should be a control packet"); + Q_ASSERT_X(bitAndType & CONTROL_BIT_MASK, "ControlPacket::readType()", "This should be a control packet"); uint16_t packetType = (bitAndType & ~CONTROL_BIT_MASK) >> (8 * sizeof(Type)); - Q_ASSERT_X(packetType <= ControlPacket::Type::HandshakeRequest, "ControlPacket::readType()", "Received a control packet with wrong type"); + Q_ASSERT_X(packetType <= ControlPacket::Type::HandshakeRequest, "ControlPacket::readType()", + "Received a control packet with invalid type"); // read the type _type = (Type) packetType; From 4b871d1073861148d916cf4e26cec9699cb85b75 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Fri, 15 Oct 2021 09:37:37 +1300 Subject: [PATCH 59/82] Tidying --- libraries/networking/src/LimitedNodeList.cpp | 4 ++-- libraries/networking/src/NLPacketList.h | 1 - libraries/networking/src/udt/Connection.cpp | 2 +- libraries/networking/src/udt/SequenceNumber.h | 4 ++-- libraries/networking/src/udt/Socket.cpp | 2 +- 5 files changed, 6 insertions(+), 7 deletions(-) diff --git a/libraries/networking/src/LimitedNodeList.cpp b/libraries/networking/src/LimitedNodeList.cpp index 1559a863555..2b487b8b2c0 100644 --- a/libraries/networking/src/LimitedNodeList.cpp +++ b/libraries/networking/src/LimitedNodeList.cpp @@ -505,8 +505,8 @@ qint64 LimitedNodeList::sendUnreliableUnorderedPacketList(NLPacketList& packetLi } return bytesSent; } else { - qCDebug(networking) << "LimitedNodeList::sendPacketList called without active socket for node" << destinationNode - << " - not sending."; + qCDebug(networking) << "LimitedNodeList::sendUnreliableUnorderedPacketList called without active socket for node" + << destinationNode << " - not sending."; return ERROR_SENDING_PACKET_BYTES; } } diff --git a/libraries/networking/src/NLPacketList.h b/libraries/networking/src/NLPacketList.h index 9c50033ca77..358dea42679 100644 --- a/libraries/networking/src/NLPacketList.h +++ b/libraries/networking/src/NLPacketList.h @@ -35,7 +35,6 @@ class NLPacketList : public udt::PacketList { virtual std::unique_ptr createPacket() override; - PacketVersion _packetVersion; NLPacket::LocalID _sourceID; }; diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 4c05e8af91c..7afeb149999 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -54,7 +54,7 @@ Connection::Connection(Socket* parentSocket, SockAddr destination, std::unique_p static std::mt19937 generator(rd()); static std::uniform_int_distribution<> distribution(0, SequenceNumber::MAX); - // randomize the intial sequence number + // randomize the initial sequence number _initialSequenceNumber = SequenceNumber(distribution(generator)); } diff --git a/libraries/networking/src/udt/SequenceNumber.h b/libraries/networking/src/udt/SequenceNumber.h index 13bd83d4f2c..beefaa50760 100644 --- a/libraries/networking/src/udt/SequenceNumber.h +++ b/libraries/networking/src/udt/SequenceNumber.h @@ -25,8 +25,8 @@ class SequenceNumber { using UType = uint32_t; // Values are for 27 bit SequenceNumber - static const Type THRESHOLD = 0x03FFFFFF; // threshold for comparing sequence numbers - static const Type MAX = 0x07FFFFFF; // maximum sequence number used in UDT + static const Type THRESHOLD = 0x03FFFFFF; // Threshold for comparing sequence numbers. + static const Type MAX = 0x07FFFFFF; // Maximum sequence number used in UDT. SequenceNumber() = default; SequenceNumber(const SequenceNumber& other) : _value(other._value) {} diff --git a/libraries/networking/src/udt/Socket.cpp b/libraries/networking/src/udt/Socket.cpp index 721eafd7e78..7beeedb1eba 100644 --- a/libraries/networking/src/udt/Socket.cpp +++ b/libraries/networking/src/udt/Socket.cpp @@ -207,7 +207,7 @@ qint64 Socket::writePacketList(std::unique_ptr packetList, const Soc return 0; } - // Unerliable and Unordered + // Unreliable and Unordered qint64 totalBytesSent = 0; while (!packetList->_packets.empty()) { totalBytesSent += writePacket(packetList->takeFront(), sockAddr); From cd56af6e7e47a26892bc27c1b2728ecbe8ed6c8c Mon Sep 17 00:00:00 2001 From: Dale Glass Date: Sat, 25 Sep 2021 19:15:28 +0200 Subject: [PATCH 60/82] Partial changes --- cmake/ports/webrtc/portfile.cmake | 6 +++--- libraries/audio-client/src/AudioClient.cpp | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/ports/webrtc/portfile.cmake b/cmake/ports/webrtc/portfile.cmake index 3e81b9fd9c6..34ea21005c0 100644 --- a/cmake/ports/webrtc/portfile.cmake +++ b/cmake/ports/webrtc/portfile.cmake @@ -24,9 +24,9 @@ else () # else Linux desktop vcpkg_download_distfile( WEBRTC_SOURCE_ARCHIVE - URLS "${EXTERNAL_BUILD_ASSETS}/seth/webrtc-20190626-linux.tar.gz" - SHA512 07d7776551aa78cb09a3ef088a8dee7762735c168c243053b262083d90a1d258cec66dc386f6903da5c4461921a3c2db157a1ee106a2b47e7756cb424b66cc43 - FILENAME webrtc-20190626-linux.tar.gz + URLS "${EXTERNAL_BUILD_ASSETS}/dependencies/vcpkg/webrtc-20210914-linux.tar.xz" + SHA512 763d1b5e15ba65c88d74c0d478f612799f567fe32ac4688e5d529aa4932a6ace89a48a3741cde86c1dee6566fab3b32504656006764882a4c17769114b860f37 + FILENAME webrtc-20210914-linux.tar.xz ) endif () diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 285549734c4..47690925e4b 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1177,7 +1177,7 @@ void AudioClient::configureWebrtc() { config.echo_canceller.enabled = true; config.echo_canceller.mobile_mode = false; #if defined(WEBRTC_LEGACY) - config.echo_canceller.use_legacy_aec = false; +// config.echo_canceller.use_legacy_aec = false; #endif config.noise_suppression.enabled = false; config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kModerate; @@ -2507,4 +2507,4 @@ void AudioClient::setInputVolume(float volume, bool emitSignal) { emit inputVolumeChanged(_audioInput->volume()); } } -} \ No newline at end of file +} From 499010efe1424ca5dabdb917957b90f3ec3d0784 Mon Sep 17 00:00:00 2001 From: Dale Glass Date: Mon, 11 Oct 2021 23:26:50 +0200 Subject: [PATCH 61/82] Update to latest webrtc package for Linux --- cmake/ports/webrtc/portfile.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/ports/webrtc/portfile.cmake b/cmake/ports/webrtc/portfile.cmake index 34ea21005c0..8c1ac9967ad 100644 --- a/cmake/ports/webrtc/portfile.cmake +++ b/cmake/ports/webrtc/portfile.cmake @@ -24,9 +24,9 @@ else () # else Linux desktop vcpkg_download_distfile( WEBRTC_SOURCE_ARCHIVE - URLS "${EXTERNAL_BUILD_ASSETS}/dependencies/vcpkg/webrtc-20210914-linux.tar.xz" - SHA512 763d1b5e15ba65c88d74c0d478f612799f567fe32ac4688e5d529aa4932a6ace89a48a3741cde86c1dee6566fab3b32504656006764882a4c17769114b860f37 - FILENAME webrtc-20210914-linux.tar.xz + URLS "${EXTERNAL_BUILD_ASSETS}/dependencies/vcpkg/webrtc-20211009-gcc-linux.tar.xz" + SHA512 d77f609bae18f1c1cbd4c50459691d1d887c6362d093e9a21614bcd7c0d9aa6364fb3cdedf04d9987f3c5728619f8c27e2c960585455c89e5b0d935595fb0386 + FILENAME webrtc-20211009-gcc-linux.tar.xz ) endif () From b9568e36ecc268bce5970a32d1de49974ff69a4c Mon Sep 17 00:00:00 2001 From: Dale Glass Date: Sat, 16 Oct 2021 18:58:14 +0200 Subject: [PATCH 62/82] Enable data channel --- libraries/shared/src/shared/WebRTC.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/shared/src/shared/WebRTC.h b/libraries/shared/src/shared/WebRTC.h index 4a900dde4ec..3f4c9e156f9 100644 --- a/libraries/shared/src/shared/WebRTC.h +++ b/libraries/shared/src/shared/WebRTC.h @@ -37,7 +37,7 @@ #elif defined(Q_OS_LINUX) && defined(Q_PROCESSOR_X86_64) # define WEBRTC_AUDIO 1 # define WEBRTC_POSIX 1 -# define WEBRTC_LEGACY 1 +# define WEBRTC_DATA_CHANNELS 1 #elif defined(Q_OS_LINUX) && defined(Q_PROCESSOR_ARM) // WebRTC is basically impossible to build on aarch64 Linux. // I am looking at https://gitlab.freedesktop.org/pulseaudio/webrtc-audio-processing for an alternative. From 9cab15a586ac5bdcad5d7dc3a13e06bae3f2553e Mon Sep 17 00:00:00 2001 From: Dale Glass Date: Sat, 16 Oct 2021 18:58:23 +0200 Subject: [PATCH 63/82] Update package --- cmake/ports/webrtc/portfile.cmake | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/ports/webrtc/portfile.cmake b/cmake/ports/webrtc/portfile.cmake index 8c1ac9967ad..bc10e35eac7 100644 --- a/cmake/ports/webrtc/portfile.cmake +++ b/cmake/ports/webrtc/portfile.cmake @@ -24,9 +24,9 @@ else () # else Linux desktop vcpkg_download_distfile( WEBRTC_SOURCE_ARCHIVE - URLS "${EXTERNAL_BUILD_ASSETS}/dependencies/vcpkg/webrtc-20211009-gcc-linux.tar.xz" - SHA512 d77f609bae18f1c1cbd4c50459691d1d887c6362d093e9a21614bcd7c0d9aa6364fb3cdedf04d9987f3c5728619f8c27e2c960585455c89e5b0d935595fb0386 - FILENAME webrtc-20211009-gcc-linux.tar.xz + URLS "${EXTERNAL_BUILD_ASSETS}/dependencies/vcpkg/webrtc-m84-gcc-linux.tar.xz" + SHA512 13f311f0d6b8fc986e1f6eea426030e6ad961c98d012d79a66eb263b923c5a4894553c55f793cf70a3a7af6cd2252e733ead9eb54e24d729e2f73c2426b39a55 + FILENAME webrtc-m84-gcc-linux.tar.xz ) endif () From e8f44fd5548d47321dbf42c765cbab1c1f682613 Mon Sep 17 00:00:00 2001 From: Dale Glass Date: Mon, 18 Oct 2021 20:40:23 +0200 Subject: [PATCH 64/82] Updated package --- cmake/ports/webrtc/portfile.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/ports/webrtc/portfile.cmake b/cmake/ports/webrtc/portfile.cmake index bc10e35eac7..7458d6e8999 100644 --- a/cmake/ports/webrtc/portfile.cmake +++ b/cmake/ports/webrtc/portfile.cmake @@ -25,7 +25,7 @@ else () vcpkg_download_distfile( WEBRTC_SOURCE_ARCHIVE URLS "${EXTERNAL_BUILD_ASSETS}/dependencies/vcpkg/webrtc-m84-gcc-linux.tar.xz" - SHA512 13f311f0d6b8fc986e1f6eea426030e6ad961c98d012d79a66eb263b923c5a4894553c55f793cf70a3a7af6cd2252e733ead9eb54e24d729e2f73c2426b39a55 + SHA512 dd08efb3e46d9e049f15bee8113b6630d969a3f98d757e6afaa62598a64658ae59b58f7b3f03658dc0fd94a010e76de71fe3fa3eed723bd8f7250433acfa936f FILENAME webrtc-m84-gcc-linux.tar.xz ) endif () From e44628f6d651a3badf70c5367b5753711f877d43 Mon Sep 17 00:00:00 2001 From: Dale Glass Date: Mon, 18 Oct 2021 20:40:33 +0200 Subject: [PATCH 65/82] Target webrtc on Linux --- libraries/networking/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/networking/CMakeLists.txt b/libraries/networking/CMakeLists.txt index 7a11329a14e..c635059d1b7 100644 --- a/libraries/networking/CMakeLists.txt +++ b/libraries/networking/CMakeLists.txt @@ -4,8 +4,8 @@ link_hifi_libraries(shared platform) target_openssl() target_tbb() -if (WIN32) - # WEBRTC TODO: Add UNIX. + +if (WIN32 OR (UNIX AND NOT APPLE)) target_webrtc() endif () From afe459b5b33597c36156abd50e566532ad556655 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Tue, 19 Oct 2021 11:40:00 +1300 Subject: [PATCH 66/82] Remove unused code --- libraries/networking/src/udt/Connection.cpp | 3 --- libraries/networking/src/udt/Connection.h | 2 -- 2 files changed, 5 deletions(-) diff --git a/libraries/networking/src/udt/Connection.cpp b/libraries/networking/src/udt/Connection.cpp index 7afeb149999..529f26d2397 100644 --- a/libraries/networking/src/udt/Connection.cpp +++ b/libraries/networking/src/udt/Connection.cpp @@ -255,9 +255,6 @@ bool Connection::processReceivedSequenceNumber(SequenceNumber sequenceNumber, in return false; } - // mark our last receive time as now (to push the potential expiry farther) - _lastReceiveTime = p_high_resolution_clock::now(); - // If this is not the next sequence number, report loss if (sequenceNumber > _lastReceivedSequenceNumber + 1) { if (_lastReceivedSequenceNumber + 1 == sequenceNumber - 1) { diff --git a/libraries/networking/src/udt/Connection.h b/libraries/networking/src/udt/Connection.h index a7904cba233..460238fe6a2 100644 --- a/libraries/networking/src/udt/Connection.h +++ b/libraries/networking/src/udt/Connection.h @@ -110,8 +110,6 @@ private slots: bool _hasReceivedHandshakeACK { false }; // flag for receipt of handshake ACK from client bool _didRequestHandshake { false }; // flag for request of handshake from server - p_high_resolution_clock::time_point _lastReceiveTime; // holds the last time we received anything from sender - SequenceNumber _initialSequenceNumber; // Randomized on Connection creation, identifies connection during re-connect requests SequenceNumber _initialReceiveSequenceNumber; // Randomized by peer Connection on creation, identifies connection during re-connect requests From 7bb74d0103fbc03cfe51028b81084f494826bcd5 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Tue, 19 Oct 2021 11:40:20 +1300 Subject: [PATCH 67/82] Comment code intent --- libraries/networking/src/PacketReceiver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/PacketReceiver.cpp b/libraries/networking/src/PacketReceiver.cpp index 9f6fd4ee4b8..d99b25e6834 100644 --- a/libraries/networking/src/PacketReceiver.cpp +++ b/libraries/networking/src/PacketReceiver.cpp @@ -161,7 +161,7 @@ void PacketReceiver::handleVerifiedMessagePacket(std::unique_ptr pa if (!message->isComplete()) { _pendingMessages[key] = message; } - handleVerifiedMessage(message, true); + handleVerifiedMessage(message, true); // Handler may handle first message packet immediately when it arrives. } else { message = it->second; message->appendPacket(*nlPacket); From 4abe8da23e4494fa139b5f1b320d6ff299909229 Mon Sep 17 00:00:00 2001 From: Dale Glass Date: Wed, 20 Oct 2021 01:16:45 +0200 Subject: [PATCH 68/82] New build, against openssl --- cmake/ports/webrtc/portfile.cmake | 2 +- libraries/audio-client/src/AudioClient.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmake/ports/webrtc/portfile.cmake b/cmake/ports/webrtc/portfile.cmake index 7458d6e8999..0986507e898 100644 --- a/cmake/ports/webrtc/portfile.cmake +++ b/cmake/ports/webrtc/portfile.cmake @@ -25,7 +25,7 @@ else () vcpkg_download_distfile( WEBRTC_SOURCE_ARCHIVE URLS "${EXTERNAL_BUILD_ASSETS}/dependencies/vcpkg/webrtc-m84-gcc-linux.tar.xz" - SHA512 dd08efb3e46d9e049f15bee8113b6630d969a3f98d757e6afaa62598a64658ae59b58f7b3f03658dc0fd94a010e76de71fe3fa3eed723bd8f7250433acfa936f + SHA512 f7c5f93566e2e79241cbb9628ab47302dd48739bb6a022c351be75553060fac4221892d094306a572cb3ec94c5031d7e812f07e7b3c0102be8c01b8c231f8ea0 FILENAME webrtc-m84-gcc-linux.tar.xz ) endif () diff --git a/libraries/audio-client/src/AudioClient.cpp b/libraries/audio-client/src/AudioClient.cpp index 47690925e4b..b77ebbee21f 100644 --- a/libraries/audio-client/src/AudioClient.cpp +++ b/libraries/audio-client/src/AudioClient.cpp @@ -1177,7 +1177,7 @@ void AudioClient::configureWebrtc() { config.echo_canceller.enabled = true; config.echo_canceller.mobile_mode = false; #if defined(WEBRTC_LEGACY) -// config.echo_canceller.use_legacy_aec = false; + config.echo_canceller.use_legacy_aec = false; #endif config.noise_suppression.enabled = false; config.noise_suppression.level = webrtc::AudioProcessing::Config::NoiseSuppression::kModerate; From 0fcb9423bf64d04c572ee9dc81f85713a93f3b2a Mon Sep 17 00:00:00 2001 From: David Rowe Date: Tue, 26 Oct 2021 11:21:23 +1300 Subject: [PATCH 69/82] Fix ICE servers used for STUN --- .../src/webrtc/WebRTCDataChannels.cpp | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp index cbc5a8ff6e8..a44cc65d2b1 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp @@ -20,10 +20,15 @@ // - https://webrtc.github.io/webrtc-org/native-code/native-apis/ // - https://webrtc.googlesource.com/src/+/master/api/peer_connection_interface.h -const std::string ICE_SERVER_URI = "stun://ice.vircadia.com:7337"; +// FIXME: stun:ice.vircadia.com:7337 doesn't work for WebRTC. +const std::list ICE_SERVER_URIS = { + "stun:stun1.l.google.com:19302", + "stun:stun4.l.google.com:19302", + "stun:stun.schlund.de" +}; const int MAX_WEBRTC_BUFFER_SIZE = 16777216; // 16MB -// #define WEBRTC_DEBUG +#define WEBRTC_DEBUG using namespace webrtc; @@ -545,9 +550,11 @@ rtc::scoped_refptr WebRTCDataChannels::createPeerConnec #endif PeerConnectionInterface::RTCConfiguration configuration; - PeerConnectionInterface::IceServer iceServer; - iceServer.uri = ICE_SERVER_URI; - configuration.servers.push_back(iceServer); + for (const auto& uri : ICE_SERVER_URIS) { + PeerConnectionInterface::IceServer iceServer; + iceServer.uri = uri; + configuration.servers.push_back(iceServer); + } #ifdef WEBRTC_DEBUG qCDebug(networking_webrtc) << "2. Create a new peer connection"; From 2971dd61852253cc8ad6507fb252228db0266203 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Tue, 26 Oct 2021 12:13:25 +1300 Subject: [PATCH 70/82] Fix handling of ICE candidate received from client --- libraries/networking/src/webrtc/WebRTCDataChannels.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp index a44cc65d2b1..92b667e18f8 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp @@ -489,7 +489,7 @@ void WebRTCDataChannels::onSignalingMessage(const QJsonObject& message) { // Add a remote ICE candidate. if (data.contains("candidate")) { - connection->addIceCandidate(data); + connection->addIceCandidate(data.value("candidate").toObject()); } } From eb21eb3f386c183b25fe6a9fb812f76d268e639e Mon Sep 17 00:00:00 2001 From: David Rowe Date: Tue, 26 Oct 2021 21:06:23 +1300 Subject: [PATCH 71/82] Add WebRTC on/off switch to domain server settings --- .../resources/describe-settings.json | 16 ++++++++++- domain-server/src/DomainServer.cpp | 28 +++++++++++++------ domain-server/src/DomainServer.h | 2 +- .../src/DomainServerSettingsManager.cpp | 2 ++ 4 files changed, 37 insertions(+), 11 deletions(-) diff --git a/domain-server/resources/describe-settings.json b/domain-server/resources/describe-settings.json index b91a10a17c5..9032d3fbd90 100644 --- a/domain-server/resources/describe-settings.json +++ b/domain-server/resources/describe-settings.json @@ -1,5 +1,5 @@ { - "version": 2.5, + "version": 2.6, "settings": [ { "name": "metaverse", @@ -73,6 +73,20 @@ } ] }, + { + "name": "webrtc", + "label": "Networking / WebRTC", + "settings": [ + { + "name": "enable_webrtc", + "label": "Enable WebRTC Client Connections", + "help": "Allow web clients to connect over WebRTC data channels.", + "type": "checkbox", + "default": false, + "advanced": true + } + ] + }, { "name": "authentication", "label": "Networking / WordPress OAuth2", diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index b295e5eaea8..c93c1f9acf3 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -196,10 +196,6 @@ DomainServer::DomainServer(int argc, char* argv[]) : _gatekeeper(this), _httpManager(QHostAddress::AnyIPv4, DOMAIN_SERVER_HTTP_PORT, QString("%1/resources/web/").arg(QCoreApplication::applicationDirPath()), this) -#if defined(WEBRTC_DATA_CHANNELS) - , - _webrtcSignalingServer(this) -#endif { if (_parentPID != -1) { watchParentProcess(_parentPID); @@ -275,6 +271,17 @@ DomainServer::DomainServer(int argc, char* argv[]) : _settingsManager.apiRefreshGroupInformation(); +#if defined(WEBRTC_DATA_CHANNELS) + const QString WEBRTC_ENABLE = "webrtc.enable_webrtc"; + bool isWebRTCEnabled = _settingsManager.valueForKeyPath(WEBRTC_ENABLE).toBool(); + qDebug() << "WebRTC enabled:" << isWebRTCEnabled; + // The domain server's WebRTC signaling server is used by the domain server and the assignment clients, so disabling it + // disables WebRTC for the server as a whole. + if (isWebRTCEnabled) { + _webrtcSignalingServer.reset(new WebRTCSignalingServer(this)); + } +#endif + setupNodeListAndAssignments(); updateReplicatedNodes(); @@ -282,7 +289,9 @@ DomainServer::DomainServer(int argc, char* argv[]) : updateUpstreamNodes(); #if defined(WEBRTC_DATA_CHANNELS) - setUpWebRTCSignalingServer(); + if (isWebRTCEnabled) { + setUpWebRTCSignalingServer(); + } #endif if (_type != NonMetaverse) { @@ -889,7 +898,7 @@ void DomainServer::setupNodeListAndAssignments() { // Sets up the WebRTC signaling server that's hosted by the domain server. void DomainServer::setUpWebRTCSignalingServer() { // Bind the WebRTC signaling server's WebSocket to its port. - bool isBound = _webrtcSignalingServer.bind(QHostAddress::AnyIPv4, DEFAULT_DOMAIN_SERVER_WS_PORT); + bool isBound = _webrtcSignalingServer->bind(QHostAddress::AnyIPv4, DEFAULT_DOMAIN_SERVER_WS_PORT); if (!isBound) { qWarning() << "WebRTC signaling server not bound to port. WebRTC connections are not supported."; return; @@ -898,13 +907,14 @@ void DomainServer::setUpWebRTCSignalingServer() { auto limitedNodeList = DependencyManager::get(); // Route inbound WebRTC signaling messages received from user clients. - connect(&_webrtcSignalingServer, &WebRTCSignalingServer::messageReceived, + connect(_webrtcSignalingServer.get(), &WebRTCSignalingServer::messageReceived, this, &DomainServer::routeWebRTCSignalingMessage); // Route domain server signaling messages. auto webrtcSocket = limitedNodeList->getWebRTCSocket(); connect(this, &DomainServer::webrtcSignalingMessageForDomainServer, webrtcSocket, &WebRTCSocket::onSignalingMessage); - connect(webrtcSocket, &WebRTCSocket::sendSignalingMessage, &_webrtcSignalingServer, &WebRTCSignalingServer::sendMessage); + connect(webrtcSocket, &WebRTCSocket::sendSignalingMessage, + _webrtcSignalingServer.get(), &WebRTCSignalingServer::sendMessage); // Forward signaling messages received from assignment clients to user client. PacketReceiver& packetReceiver = limitedNodeList->getPacketReceiver(); @@ -912,7 +922,7 @@ void DomainServer::setUpWebRTCSignalingServer() { PacketReceiver::makeUnsourcedListenerReference(this, &DomainServer::forwardAssignmentClientSignalingMessageToUserClient)); connect(this, &DomainServer::webrtcSignalingMessageForUserClient, - &_webrtcSignalingServer, &WebRTCSignalingServer::sendMessage); + _webrtcSignalingServer.get(), &WebRTCSignalingServer::sendMessage); } // Routes an inbound WebRTC signaling message received from a client app to the appropriate recipient. diff --git a/domain-server/src/DomainServer.h b/domain-server/src/DomainServer.h index 83816a132a1..13c715d5e3d 100644 --- a/domain-server/src/DomainServer.h +++ b/domain-server/src/DomainServer.h @@ -333,7 +333,7 @@ private slots: QThread _assetClientThread; #if defined(WEBRTC_DATA_CHANNELS) - WebRTCSignalingServer _webrtcSignalingServer; + std::unique_ptr _webrtcSignalingServer { nullptr }; #endif }; diff --git a/domain-server/src/DomainServerSettingsManager.cpp b/domain-server/src/DomainServerSettingsManager.cpp index 7b04c72845a..912a771ab3c 100644 --- a/domain-server/src/DomainServerSettingsManager.cpp +++ b/domain-server/src/DomainServerSettingsManager.cpp @@ -551,6 +551,8 @@ void DomainServerSettingsManager::setupConfigMap(const QString& userConfigFilena packPermissions(); } + // No migration needed to version 2.6. + // write the current description version to our settings *versionVariant = _descriptionVersion; From 92e3d1465eb17dd892d1b5a933931521cd5b719d Mon Sep 17 00:00:00 2001 From: David Rowe Date: Wed, 27 Oct 2021 21:41:12 +1300 Subject: [PATCH 72/82] Reduce the number of ICE servers used --- libraries/networking/src/webrtc/WebRTCDataChannels.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp index 92b667e18f8..0d069d71714 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp @@ -21,9 +21,9 @@ // - https://webrtc.googlesource.com/src/+/master/api/peer_connection_interface.h // FIXME: stun:ice.vircadia.com:7337 doesn't work for WebRTC. -const std::list ICE_SERVER_URIS = { +// Firefox warns: "WebRTC: Using more than two STUN/TURN servers slows down discovery" +const std::list ICE_SERVER_URIS = { "stun:stun1.l.google.com:19302", - "stun:stun4.l.google.com:19302", "stun:stun.schlund.de" }; const int MAX_WEBRTC_BUFFER_SIZE = 16777216; // 16MB From bb3d0fa5fa13ba4c352613c7fe52b209ca10b5cb Mon Sep 17 00:00:00 2001 From: David Rowe Date: Wed, 27 Oct 2021 21:42:58 +1300 Subject: [PATCH 73/82] Add secure WebSocket support --- .../resources/describe-settings.json | 8 +++ domain-server/src/DomainServer.cpp | 5 +- .../src/webrtc/WebRTCSignalingServer.cpp | 57 ++++++++++++++++++- .../src/webrtc/WebRTCSignalingServer.h | 3 +- 4 files changed, 68 insertions(+), 5 deletions(-) diff --git a/domain-server/resources/describe-settings.json b/domain-server/resources/describe-settings.json index 9032d3fbd90..f83a4c47b4f 100644 --- a/domain-server/resources/describe-settings.json +++ b/domain-server/resources/describe-settings.json @@ -84,6 +84,14 @@ "type": "checkbox", "default": false, "advanced": true + }, + { + "name": "enable_webrtc_websocket_ssl", + "label": "Enable WebRTC WebSocket SSL", + "help": "Use secure WebSocket (wss:// protocol) for WebRTC signaling channel. If \"on\", the key, cert, and CA files are expected to be in the local Vircadia app directory, in a /domain-server/ subdirectory with filenames cert.key, cert.crt, and crt-ca.crt.", + "type": "checkbox", + "default": false, + "advanced": true } ] }, diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index c93c1f9acf3..67f961d36f5 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -278,7 +278,10 @@ DomainServer::DomainServer(int argc, char* argv[]) : // The domain server's WebRTC signaling server is used by the domain server and the assignment clients, so disabling it // disables WebRTC for the server as a whole. if (isWebRTCEnabled) { - _webrtcSignalingServer.reset(new WebRTCSignalingServer(this)); + const QString WEBRTC_WSS_ENABLE = "webrtc.enable_webrtc_websocket_ssl"; + bool isWebRTCEnabled = _settingsManager.valueForKeyPath(WEBRTC_WSS_ENABLE).toBool(); + qDebug() << "WebRTC WSS enabled:" << isWebRTCEnabled; + _webrtcSignalingServer.reset(new WebRTCSignalingServer(this, isWebRTCEnabled)); } #endif diff --git a/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp b/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp index b35cd5158b5..987bf6b02f4 100644 --- a/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp +++ b/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp @@ -10,19 +10,70 @@ #if defined(WEBRTC_DATA_CHANNELS) +#include #include #include +#include +#include + #include "../NetworkLogging.h" #include "../NodeType.h" const int WEBRTC_SOCKET_CHECK_INTERVAL_IN_MS = 30000; -WebRTCSignalingServer::WebRTCSignalingServer(QObject* parent) : - QObject(parent), - _webSocketServer(new QWebSocketServer(QStringLiteral("WebRTC Signaling Server"), QWebSocketServer::NonSecureMode, this)) +WebRTCSignalingServer::WebRTCSignalingServer(QObject* parent, bool isWSSEnabled) : + QObject(parent) { + if (isWSSEnabled) { + _webSocketServer = (new QWebSocketServer(QStringLiteral("WebRTC Signaling Server"), QWebSocketServer::SecureMode, + this)); + + auto dsDirPath = PathUtils::getAppLocalDataPath(); + const QString KEY_FILENAME = "cert.key"; + const QString CRT_FILENAME = "cert.crt"; + const QString CA_CRT_FILENAME = "cert-ca.crt"; + qCDebug(networking_webrtc) << "WebSocket WSS key file:" << dsDirPath + KEY_FILENAME; + qCDebug(networking_webrtc) << "WebSocket WSS cert file:" << dsDirPath + CRT_FILENAME; + qCDebug(networking_webrtc) << "WebSocket WSS CA cert file:" << dsDirPath + CA_CRT_FILENAME; + + QFile sslCaFile(dsDirPath + CA_CRT_FILENAME); + sslCaFile.open(QIODevice::ReadOnly); + QSslCertificate sslCaCertificate(&sslCaFile, QSsl::Pem); + sslCaFile.close(); + + QSslConfiguration sslConfiguration; + QFile sslCrtFile(dsDirPath + CRT_FILENAME); + sslCrtFile.open(QIODevice::ReadOnly); + QSslCertificate sslCertificate(&sslCrtFile, QSsl::Pem); + sslCrtFile.close(); + + QFile sslKeyFile(dsDirPath + KEY_FILENAME); + sslKeyFile.open(QIODevice::ReadOnly); + QSslKey sslKey(&sslKeyFile, QSsl::Rsa, QSsl::Pem); + sslKeyFile.close(); + + if (!sslCaCertificate.isNull() && !sslKey.isNull() && !sslCertificate.isNull()) { + sslConfiguration.setPeerVerifyMode(QSslSocket::VerifyNone); + sslConfiguration.addCaCertificate(sslCaCertificate); + sslConfiguration.setLocalCertificate(sslCertificate); + sslConfiguration.setPrivateKey(sslKey); + _webSocketServer->setSslConfiguration(sslConfiguration); + qCDebug(networking_webrtc) << "WebSocket SSL mode enabled:" + << (_webSocketServer->secureMode() == QWebSocketServer::SecureMode); + } else { + qCWarning(networking_webrtc) << "Error creating WebSocket SSL key."; + } + + } else { + _webSocketServer = (new QWebSocketServer(QStringLiteral("WebRTC Signaling Server"), QWebSocketServer::NonSecureMode, + this)); + } +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WebRTCSignalingServer mode =" << _webSocketServer->secureMode(); +#endif + connect(_webSocketServer, &QWebSocketServer::newConnection, this, &WebRTCSignalingServer::newWebSocketConnection); // Automatically recover from network interruptions. diff --git a/libraries/networking/src/webrtc/WebRTCSignalingServer.h b/libraries/networking/src/webrtc/WebRTCSignalingServer.h index fb695cc6fcc..3a2eebbcefe 100644 --- a/libraries/networking/src/webrtc/WebRTCSignalingServer.h +++ b/libraries/networking/src/webrtc/WebRTCSignalingServer.h @@ -61,7 +61,8 @@ class WebRTCSignalingServer : public QObject { /// @brief Constructs a new WebRTCSignalingServer object. /// @param parent Qt parent object. - WebRTCSignalingServer(QObject* parent); + /// @param isWSSEnabled Whether the WebSocket used for WebRTC signaling should be secure (WSS protocol). + WebRTCSignalingServer(QObject* parent, bool isWSSEnabled); /// @brief Binds the WebRTC signaling server's WebSocket to an address and port. /// @param address The address to use for the WebSocket. From 8e3cfe9805dbc3fa3a29bbbcaf857d9d239e668f Mon Sep 17 00:00:00 2001 From: David Rowe Date: Fri, 29 Oct 2021 23:04:41 +1300 Subject: [PATCH 74/82] Fix non-const lvalue reference --- libraries/networking/src/webrtc/WebRTCDataChannels.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp index 0d069d71714..128d48f42af 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp @@ -489,7 +489,8 @@ void WebRTCDataChannels::onSignalingMessage(const QJsonObject& message) { // Add a remote ICE candidate. if (data.contains("candidate")) { - connection->addIceCandidate(data.value("candidate").toObject()); + auto candidate = data.value("candidate").toObject(); + connection->addIceCandidate(candidate); } } From ccd525a89b4bd8e77c8aacc5fa92544f40e57ff5 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sat, 30 Oct 2021 08:30:05 +1300 Subject: [PATCH 75/82] CR --- domain-server/src/DomainServer.cpp | 4 ++-- libraries/networking/src/webrtc/WebRTCSignalingServer.cpp | 4 ---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/domain-server/src/DomainServer.cpp b/domain-server/src/DomainServer.cpp index 67f961d36f5..d14a909a1a2 100644 --- a/domain-server/src/DomainServer.cpp +++ b/domain-server/src/DomainServer.cpp @@ -274,13 +274,13 @@ DomainServer::DomainServer(int argc, char* argv[]) : #if defined(WEBRTC_DATA_CHANNELS) const QString WEBRTC_ENABLE = "webrtc.enable_webrtc"; bool isWebRTCEnabled = _settingsManager.valueForKeyPath(WEBRTC_ENABLE).toBool(); - qDebug() << "WebRTC enabled:" << isWebRTCEnabled; + qCDebug(domain_server) << "WebRTC enabled:" << isWebRTCEnabled; // The domain server's WebRTC signaling server is used by the domain server and the assignment clients, so disabling it // disables WebRTC for the server as a whole. if (isWebRTCEnabled) { const QString WEBRTC_WSS_ENABLE = "webrtc.enable_webrtc_websocket_ssl"; bool isWebRTCEnabled = _settingsManager.valueForKeyPath(WEBRTC_WSS_ENABLE).toBool(); - qDebug() << "WebRTC WSS enabled:" << isWebRTCEnabled; + qCDebug(domain_server) << "WebRTC WSS enabled:" << isWebRTCEnabled; _webrtcSignalingServer.reset(new WebRTCSignalingServer(this, isWebRTCEnabled)); } #endif diff --git a/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp b/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp index 987bf6b02f4..bd8cd2fef48 100644 --- a/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp +++ b/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp @@ -70,10 +70,6 @@ WebRTCSignalingServer::WebRTCSignalingServer(QObject* parent, bool isWSSEnabled) _webSocketServer = (new QWebSocketServer(QStringLiteral("WebRTC Signaling Server"), QWebSocketServer::NonSecureMode, this)); } -#ifdef WEBRTC_DEBUG - qCDebug(networking_webrtc) << "WebRTCSignalingServer mode =" << _webSocketServer->secureMode(); -#endif - connect(_webSocketServer, &QWebSocketServer::newConnection, this, &WebRTCSignalingServer::newWebSocketConnection); // Automatically recover from network interruptions. From c76dc5358ba0976b416eabe202f03aafc2e74089 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sun, 31 Oct 2021 10:01:04 +1300 Subject: [PATCH 76/82] Rename domain server cert files --- domain-server/resources/describe-settings.json | 2 +- libraries/networking/src/webrtc/WebRTCSignalingServer.cpp | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/domain-server/resources/describe-settings.json b/domain-server/resources/describe-settings.json index f83a4c47b4f..f6882d6a053 100644 --- a/domain-server/resources/describe-settings.json +++ b/domain-server/resources/describe-settings.json @@ -88,7 +88,7 @@ { "name": "enable_webrtc_websocket_ssl", "label": "Enable WebRTC WebSocket SSL", - "help": "Use secure WebSocket (wss:// protocol) for WebRTC signaling channel. If \"on\", the key, cert, and CA files are expected to be in the local Vircadia app directory, in a /domain-server/ subdirectory with filenames cert.key, cert.crt, and crt-ca.crt.", + "help": "Use secure WebSocket (wss:// protocol) for WebRTC signaling channel. If \"on\", the key, cert, and CA files are expected to be in the local Vircadia app directory, in a /domain-server/ subdirectory with filenames vircadia-cert.key, vircadia-cert.crt, and vircadia-crt-ca.crt.", "type": "checkbox", "default": false, "advanced": true diff --git a/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp b/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp index bd8cd2fef48..f3837a874b8 100644 --- a/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp +++ b/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp @@ -31,9 +31,9 @@ WebRTCSignalingServer::WebRTCSignalingServer(QObject* parent, bool isWSSEnabled) this)); auto dsDirPath = PathUtils::getAppLocalDataPath(); - const QString KEY_FILENAME = "cert.key"; - const QString CRT_FILENAME = "cert.crt"; - const QString CA_CRT_FILENAME = "cert-ca.crt"; + const QString KEY_FILENAME = "vircadia-cert.key"; + const QString CRT_FILENAME = "vircadia-cert.crt"; + const QString CA_CRT_FILENAME = "vircadia-cert-ca.crt"; qCDebug(networking_webrtc) << "WebSocket WSS key file:" << dsDirPath + KEY_FILENAME; qCDebug(networking_webrtc) << "WebSocket WSS cert file:" << dsDirPath + CRT_FILENAME; qCDebug(networking_webrtc) << "WebSocket WSS CA cert file:" << dsDirPath + CA_CRT_FILENAME; From afd79931523722f86d7390b0947ee7afdea2e926 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sat, 6 Nov 2021 22:31:52 +1300 Subject: [PATCH 77/82] Disable WebRTC debug --- libraries/networking/src/webrtc/WebRTCDataChannels.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp index 128d48f42af..ad67004d586 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp @@ -28,7 +28,7 @@ const std::list ICE_SERVER_URIS = { }; const int MAX_WEBRTC_BUFFER_SIZE = 16777216; // 16MB -#define WEBRTC_DEBUG +// #define WEBRTC_DEBUG using namespace webrtc; From a2ac43289c1c5abf9bdcebd1aa9cc5cc39fbd24a Mon Sep 17 00:00:00 2001 From: David Rowe Date: Tue, 9 Nov 2021 12:12:55 +1300 Subject: [PATCH 78/82] Use avatar position as orb position when head position not available --- interface/src/avatar/OtherAvatar.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/interface/src/avatar/OtherAvatar.cpp b/interface/src/avatar/OtherAvatar.cpp index b6a2ea3ed98..fe0e83dfa04 100755 --- a/interface/src/avatar/OtherAvatar.cpp +++ b/interface/src/avatar/OtherAvatar.cpp @@ -64,7 +64,11 @@ void OtherAvatar::removeOrb() { void OtherAvatar::updateOrbPosition() { if (!_otherAvatarOrbMeshPlaceholderID.isNull()) { EntityItemProperties properties; - properties.setPosition(getHead()->getPosition()); + glm::vec3 headPosition; + if (!_skeletonModel->getHeadPosition(headPosition)) { + headPosition = getWorldPosition(); + } + properties.setPosition(headPosition); DependencyManager::get()->editEntity(_otherAvatarOrbMeshPlaceholderID, properties); } } From fb79d57fefa299d807203eafedd3eae6bdb73aff Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sat, 20 Nov 2021 11:06:05 +1300 Subject: [PATCH 79/82] Improve WebRTC debug --- .../src/webrtc/WebRTCDataChannels.cpp | 64 +++++++++++++++++-- .../src/webrtc/WebRTCDataChannels.h | 8 +++ 2 files changed, 65 insertions(+), 7 deletions(-) diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp index ad67004d586..6ed9761c639 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp @@ -71,7 +71,7 @@ WDCPeerConnectionObserver::WDCPeerConnectionObserver(WDCConnection* parent) : void WDCPeerConnectionObserver::OnSignalingChange(PeerConnectionInterface::SignalingState newState) { #ifdef WEBRTC_DEBUG - QStringList states{ + QStringList states { "Stable", "HaveLocalOffer", "HaveLocalPrAnswer", @@ -79,7 +79,7 @@ void WDCPeerConnectionObserver::OnSignalingChange(PeerConnectionInterface::Signa "HaveRemotePrAnswer", "Closed" }; - qCDebug(networking_webrtc) << "WDCPeerConnectionObserver::OnSignalingChange()" << newState << states[newState]; + qCDebug(networking_webrtc) << "WDCPeerConnectionObserver::OnSignalingChange() :" << newState << states[newState]; #endif } @@ -91,7 +91,12 @@ void WDCPeerConnectionObserver::OnRenegotiationNeeded() { void WDCPeerConnectionObserver::OnIceGatheringChange(PeerConnectionInterface::IceGatheringState newState) { #ifdef WEBRTC_DEBUG - qCDebug(networking_webrtc) << "WDCPeerConnectionObserver::OnIceGatheringChange()" << newState; + QStringList states { + "New", + "Gathering", + "Complete" + }; + qCDebug(networking_webrtc) << "WDCPeerConnectionObserver::OnIceGatheringChange() :" << newState << states[newState]; #endif } @@ -102,6 +107,39 @@ void WDCPeerConnectionObserver::OnIceCandidate(const IceCandidateInterface* cand _parent->sendIceCandidate(candidate); } +void WDCPeerConnectionObserver::OnIceConnectionChange(PeerConnectionInterface::IceConnectionState newState) { +#ifdef WEBRTC_DEBUG + QStringList states { + "New", + "Checking", + "Connected", + "Completed", + "Failed", + "Disconnected", + "Closed", + "Max" + }; + qCDebug(networking_webrtc) << "WDCPeerConnectionObserver::OnIceConnectionChange() :" << newState << states[newState]; +#endif +} + +void WDCPeerConnectionObserver::OnStandardizedIceConnectionChange(PeerConnectionInterface::IceConnectionState newState) { +#ifdef WEBRTC_DEBUG + QStringList states { + "New", + "Checking", + "Connected", + "Completed", + "Failed", + "Disconnected", + "Closed", + "Max" + }; + qCDebug(networking_webrtc) << "WDCPeerConnectionObserver::OnStandardizedIceConnectionChange() :" << newState + << states[newState]; +#endif +} + void WDCPeerConnectionObserver::OnDataChannel(rtc::scoped_refptr dataChannel) { #ifdef WEBRTC_DEBUG qCDebug(networking_webrtc) << "WDCPeerConnectionObserver::OnDataChannel()"; @@ -111,7 +149,16 @@ void WDCPeerConnectionObserver::OnDataChannel(rtc::scoped_refptronPeerConnectionStateChanged(newState); } @@ -267,7 +314,7 @@ void WDCConnection::sendIceCandidate(const IceCandidateInterface* candidate) { void WDCConnection::onPeerConnectionStateChanged(PeerConnectionInterface::PeerConnectionState state) { #ifdef WEBRTC_DEBUG - const char* STATES[] = { + QStringList states { "New", "Connecting", "Connected", @@ -275,7 +322,7 @@ void WDCConnection::onPeerConnectionStateChanged(PeerConnectionInterface::PeerCo "Failed", "Closed" }; - qCDebug(networking_webrtc) << "WDCConnection::onPeerConnectionStateChanged() :" << (int)state << STATES[(int)state]; + qCDebug(networking_webrtc) << "WDCConnection::onPeerConnectionStateChanged() :" << (int)state << states[(int)state]; #endif } @@ -372,7 +419,7 @@ bool WDCConnection::sendDataMessage(const DataBuffer& buffer) { void WDCConnection::closePeerConnection() { #ifdef WEBRTC_DEBUG - qCDebug(networking_webrtc) << "WDCConnection::closePeerConnection()"; + qCDebug(networking_webrtc) << "WDCConnection::closePeerConnection() :" << (int)_peerConnection->peer_connection_state(); #endif _peerConnection->Close(); _peerConnection = nullptr; @@ -430,6 +477,9 @@ WebRTCDataChannels::~WebRTCDataChannels() { } void WebRTCDataChannels::reset() { +#ifdef WEBRTC_DEBUG + qCDebug(networking_webrtc) << "WebRTCDataChannels::reset() :" << _connectionsByID.count(); +#endif QHashIterator i(_connectionsByID); while (i.hasNext()) { i.next(); diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.h b/libraries/networking/src/webrtc/WebRTCDataChannels.h index fe8af770789..68bb439b619 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.h +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.h @@ -88,6 +88,14 @@ class WDCPeerConnectionObserver : public webrtc::PeerConnectionObserver { /// @param candidate The new ICE candidate. void OnIceCandidate(const webrtc::IceCandidateInterface* candidate) override; + /// @brief Called when the legacy ICE connection state changes. + /// @param new_state The new ICE connection state. + virtual void OnIceConnectionChange(webrtc::PeerConnectionInterface::IceConnectionState newState) override; + + /// @brief Called when the standards-compliant ICE connection state changes. + /// @param new_state The new ICE connection state. + virtual void OnStandardizedIceConnectionChange(webrtc::PeerConnectionInterface::IceConnectionState newState) override; + /// @brief Called when a remote peer opens a data channel. /// @param dataChannel The data channel. void OnDataChannel(rtc::scoped_refptr dataChannel) override; From cf57b506b5dae9ac7f3cbd1796311bbb25aacb1b Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sat, 20 Nov 2021 12:13:30 +1300 Subject: [PATCH 80/82] Fix domain server crash --- libraries/networking/src/webrtc/WebRTCSignalingServer.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp b/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp index f3837a874b8..2c557ceaee3 100644 --- a/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp +++ b/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp @@ -131,8 +131,9 @@ void WebRTCSignalingServer::webSocketDisconnected() { auto source = qobject_cast(sender()); if (source) { auto address = source->peerAddress().toString() + ":" + QString::number(source->peerPort()); - delete _webSockets.value(address); _webSockets.remove(address); + source->abort(); + source->deleteLater(); } } From 56d33b555eb87aebc6bde234141724ef216cfb39 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Sat, 20 Nov 2021 17:26:06 +1300 Subject: [PATCH 81/82] Fix assignment client crash --- .../src/webrtc/WebRTCDataChannels.cpp | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp index 6ed9761c639..44c1c5e82cb 100644 --- a/libraries/networking/src/webrtc/WebRTCDataChannels.cpp +++ b/libraries/networking/src/webrtc/WebRTCDataChannels.cpp @@ -355,13 +355,11 @@ void WDCConnection::onDataChannelStateChanged() { << DataChannelInterface::DataStateString(state); #endif if (state == DataChannelInterface::kClosed) { - // Close data channel. + // Finish with the data channel. _dataChannel->UnregisterObserver(); + // Don't set _dataChannel = nullptr because it is a scoped_refptr. _dataChannelObserver = nullptr; - _dataChannel = nullptr; -#ifdef WEBRTC_DEBUG - qCDebug(networking_webrtc) << "Disposed of data channel"; -#endif + // Close peer connection. _parent->closePeerConnection(this); } @@ -396,17 +394,21 @@ qint64 WDCConnection::getBufferedAmount() const { #ifdef WEBRTC_DEBUG qCDebug(networking_webrtc) << "WDCConnection::getBufferedAmount()"; #endif - return _dataChannel ? _dataChannel->buffered_amount() : 0; + return _dataChannel && _dataChannel->state() != DataChannelInterface::kClosing + && _dataChannel->state() != DataChannelInterface::kClosed + ? _dataChannel->buffered_amount() : 0; } bool WDCConnection::sendDataMessage(const DataBuffer& buffer) { #ifdef WEBRTC_DEBUG qCDebug(networking_webrtc) << "WDCConnection::sendDataMessage()"; - if (!_dataChannel) { + if (!_dataChannel || _dataChannel->state() == DataChannelInterface::kClosing + || _dataChannel->state() == DataChannelInterface::kClosed) { qCDebug(networking_webrtc) << "No data channel to send on"; } #endif - if (!_dataChannel) { + if (!_dataChannel || _dataChannel->state() == DataChannelInterface::kClosing + || _dataChannel->state() == DataChannelInterface::kClosed) { // Data channel may have been closed while message to send was being prepared. return false; } else if (_dataChannel->buffered_amount() + buffer.size() > MAX_WEBRTC_BUFFER_SIZE) { @@ -422,7 +424,7 @@ void WDCConnection::closePeerConnection() { qCDebug(networking_webrtc) << "WDCConnection::closePeerConnection() :" << (int)_peerConnection->peer_connection_state(); #endif _peerConnection->Close(); - _peerConnection = nullptr; + // Don't set _peerConnection = nullptr because it is a scoped_refptr. _peerConnectionObserver = nullptr; #ifdef WEBRTC_DEBUG qCDebug(networking_webrtc) << "Disposed of peer connection"; From 075c0d8b89221706768e87e25b3cd51766e14416 Mon Sep 17 00:00:00 2001 From: David Rowe Date: Tue, 23 Nov 2021 09:57:42 +1300 Subject: [PATCH 82/82] Fix domain server crash with SSL WebSocket --- libraries/networking/src/webrtc/WebRTCSignalingServer.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp b/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp index 2c557ceaee3..0534639419b 100644 --- a/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp +++ b/libraries/networking/src/webrtc/WebRTCSignalingServer.cpp @@ -132,7 +132,6 @@ void WebRTCSignalingServer::webSocketDisconnected() { if (source) { auto address = source->peerAddress().toString() + ":" + QString::number(source->peerPort()); _webSockets.remove(address); - source->abort(); source->deleteLater(); } }