From 786323fe013460c0537fe304a2f3edfce8a207a8 Mon Sep 17 00:00:00 2001 From: acgist <289547414@qq.com> Date: Wed, 17 May 2023 08:56:42 +0800 Subject: [PATCH] =?UTF-8?q?[*]=20=E6=AF=8F=E6=97=A5=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/Deploy.md | 2 +- docs/Docker.md | 0 docs/TODO.md | 1 + .../src/main/cpp/webrtc/MediaManager.cpp | 2 +- .../taoyao/media/src/main/cpp/webrtc/Room.cpp | 169 +++++++++++++++--- .../taoyao/media/client/SessionClient.java | 2 + taoyao-client-web/src/App.vue | 7 +- taoyao-client-web/src/components/Config.js | 7 +- .../src/components/LocalClient.vue | 14 +- .../src/components/RemoteClient.vue | 18 +- .../src/components/SessionClient.vue | 11 +- taoyao-client-web/src/components/Taoyao.js | 2 + taoyao-signal-server/docs/bin/wait.sh | 3 + .../boot/config/MediaVideoProperties.java | 3 + 14 files changed, 196 insertions(+), 45 deletions(-) create mode 100644 docs/Docker.md diff --git a/docs/Deploy.md b/docs/Deploy.md index 3762f21..791e861 100644 --- a/docs/Deploy.md +++ b/docs/Deploy.md @@ -24,7 +24,7 @@ wget /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7. yum makecache ``` -## 更新依赖 +## 安装依赖 ``` yum install zlib-devel libffi-devel openssl-devel diff --git a/docs/Docker.md b/docs/Docker.md new file mode 100644 index 0000000..e69de29 diff --git a/docs/TODO.md b/docs/TODO.md index 7491693..e250f80 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -34,6 +34,7 @@ * 录制底噪 * 分辨率调整 * 服务端录制 +* 安卓预览按钮 * 降低视频录制大小 * 防止重复邀请拉取 * 查询消费者生产者信息 diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/MediaManager.cpp b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/MediaManager.cpp index c921b1b..af4f67d 100644 --- a/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/MediaManager.cpp +++ b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/MediaManager.cpp @@ -39,7 +39,7 @@ namespace acgist { extern "C" JNIEXPORT void JNICALL Java_com_acgist_taoyao_media_MediaManager_nativeInit(JNIEnv* env, jobject me) { std::string version = mediasoupclient::Version(); - LOG_I("加载MediasoupClient %s", version.data()); + LOG_I("加载MediasoupClient：%s", version.data()); mediasoupclient::Initialize(); // => { spatialLayers: 2, temporalLayers: 3 } // mediasoupclient::parseScalabilityMode("L2T3"); diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/Room.cpp b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/Room.cpp index 23190a4..dd9d644 100644 --- a/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/Room.cpp +++ b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/Room.cpp @@ -2,22 +2,45 @@ namespace acgist { + /** + * 发送通道监听器 + */ class SendListener : public mediasoupclient::SendTransport::Listener { public: + /** + * 房间指针 + */ Room* room; public: + /** + * 发送通道监听器 + * + * @param room 房间指针 + */ explicit SendListener(Room* room) { this->room = room; } + /** + * 析构函数 + */ virtual ~SendListener() { } public: + /** + * 连接通道 + * + * @param transport 通道指针 + * @param dtlsParameters DTLS参数 + * + * @return future + */ std::future OnConnect(mediasoupclient::Transport* transport, const nlohmann::json& dtlsParameters) override { const std::string cTransportId = transport->GetId(); const std::string cDtlsParameters = dtlsParameters.dump(); + LOG_I("连接发送通道：%s - %s", this->room->roomId.data(), cTransportId.data()); JNIEnv* env = nullptr; if(taoyaoJavaVM->GetEnv((void**) &env, JNI_VERSION_1_6) == JNI_EDETACHED) { bindJavaThread(&env); @@ -31,13 +54,31 @@ namespace acgist { return promise.get_future(); } + /** + * 通道状态改变 + * + * @param transport 通道指针 + * @param connectionState 当前状态 + */ void OnConnectionStateChange(mediasoupclient::Transport* transport, const std::string& connectionState) override { + LOG_I("发送通道状态改变：%s - %s - %s", this->room->roomId.data(), transport->GetId().data(), connectionState.data()); } + /** + * 通道生产媒体 + * + * @param transport 通道指针 + * @param kind 媒体类型 + * @param rtpParameters RTP参数 + * @param appData 应用数据 + * + * @return 生产者ID + */ std::future OnProduce(mediasoupclient::SendTransport* transport, const std::string& kind, nlohmann::json rtpParameters, const nlohmann::json& appData) override { std::string result; const std::string cTransportId = transport->GetId(); const std::string cRtpParameters = rtpParameters.dump(); + LOG_I("生产媒体：%s - %s - %s", this->room->roomId.data(), cTransportId.data(), kind.data()); JNIEnv* env = nullptr; if(taoyaoJavaVM->GetEnv((void**) &env, JNI_VERSION_1_6) == JNI_EDETACHED) { bindJavaThread(&env); @@ -51,29 +92,65 @@ namespace acgist { return promise.get_future(); } + /** + * 通道生产数据 + * 注意：需要自己实现 + * + * @param transport 通道指针 + * @param sctpStreamParameters SCTP参数 + * @param label 标记 + * @param protocol 协议 + * @param appData 应用数据 + * + * @return 生产者ID + */ std::future OnProduceData(mediasoupclient::SendTransport* transport, const nlohmann::json& sctpStreamParameters, const std::string& label, const std::string& protocol, const nlohmann::json& appData) override { + const std::string cTransportId = transport->GetId(); + LOG_I("生产数据：%s - %s - %s -%s", this->room->roomId.data(), cTransportId.data(), label.data(), protocol.data()); std::promise promise; - // TODO：如果需要自行实现 return promise.get_future(); } }; + /** + * 接收通道监听器 + */ class RecvListener : public mediasoupclient::RecvTransport::Listener { public: + /** + * 房间指针 + */ Room* room; public: + /** + * 接收通道监听器 + * + * @param room 房间指针 + */ explicit RecvListener(Room* room) { this->room = room; } + /** + * 析构函数 + */ virtual ~RecvListener() { } + /** + * 连接通道 + * + * @param transport 通道指针 + * @param dtlsParameters DTLS参数 + * + * @return future + */ std::future OnConnect(mediasoupclient::Transport* transport, const nlohmann::json& dtlsParameters) override { const std::string cTransportId = transport->GetId(); const std::string cDtlsParameters = dtlsParameters.dump(); + LOG_I("连接接收通道：%s - %s", this->room->roomId.data(), cTransportId.data()); JNIEnv* env = nullptr; if(taoyaoJavaVM->GetEnv((void**) &env, JNI_VERSION_1_6) == JNI_EDETACHED) { bindJavaThread(&env); @@ -87,24 +164,51 @@ namespace acgist { return promise.get_future(); } + /** + * 通道状态改变 + * + * @param transport 通道指针 + * @param connectionState 通道状态 + */ void OnConnectionStateChange(mediasoupclient::Transport* transport, const std::string& connectionState) override { + LOG_I("接收通道状态改变：%s - %s - %s", this->room->roomId.data(), transport->GetId().data(), connectionState.data()); } }; + /** + * 生产者监听器 + */ class ProducerListener : public mediasoupclient::Producer::Listener { public: + /** + * 房间指针 + */ Room* room; public: + /** + * 生产者监听器 + * + * @param room 房间指针 + */ explicit ProducerListener(Room* room) { this->room = room; } + /** + * 析构函数 + */ virtual ~ProducerListener() { } + /** + * 通道关闭 + * + * @param producer 生产者 + */ void OnTransportClose(mediasoupclient::Producer* producer) override { + LOG_I("生产者通道关闭：%s - %s", this->room->roomId.data(), producer->GetId().data()); producer->Close(); JNIEnv* env = nullptr; if(taoyaoJavaVM->GetEnv((void**) &env, JNI_VERSION_1_6) == JNI_EDETACHED) { @@ -118,20 +222,39 @@ namespace acgist { }; + /** + * 消费者监听器 + */ class ConsumerListener : public mediasoupclient::Consumer::Listener { public: + /** + * 房间指针 + */ Room* room; public: + /** + * 消费者监听器 + * + * @param room 房间指针 + */ explicit ConsumerListener(Room* room) { this->room = room; } + /** + * 析构函数 + */ virtual ~ConsumerListener() { -// mediasoupclient::Consumer::Listener::~Listener(); } + /** + * 通道关闭 + * + * @param consumer 消费者 + */ void OnTransportClose(mediasoupclient::Consumer* consumer) override { + LOG_I("消费者通道关闭：%s - %s", this->room->roomId.data(), consumer->GetId().data()); consumer->Close(); JNIEnv* env = nullptr; if(taoyaoJavaVM->GetEnv((void**) &env, JNI_VERSION_1_6) == JNI_EDETACHED) { @@ -164,6 +287,7 @@ namespace acgist { } Room::~Room() { + // TODO：解决析构函数不是虚函数的问题 delete this->rtcConfiguration; delete this->device; delete this->sendListener; @@ -185,8 +309,8 @@ namespace acgist { this->factory = factory; this->rtcConfiguration = new webrtc::PeerConnectionInterface::RTCConfiguration(rtcConfiguration); mediasoupclient::PeerConnection::Options options; - options.config = rtcConfiguration; - options.factory = factory; + options.config = rtcConfiguration; + options.factory = factory; nlohmann::json json = nlohmann::json::parse(rtpCapabilities); this->device->Load(json, &options); const std::string cRtpCapabilities = this->device->GetRtpCapabilities().dump(); @@ -197,8 +321,8 @@ namespace acgist { void Room::createSendTransport(JNIEnv* env, const std::string& body) { nlohmann::json json = nlohmann::json::parse(body); mediasoupclient::PeerConnection::Options options; - options.config = *this->rtcConfiguration; - options.factory = this->factory; + options.config = *this->rtcConfiguration; + options.factory = this->factory; this->sendTransport = this->device->CreateSendTransport( this->sendListener, json["transportId"], @@ -213,8 +337,8 @@ namespace acgist { void Room::createRecvTransport(JNIEnv* env, const std::string& body) { nlohmann::json json = nlohmann::json::parse(body); mediasoupclient::PeerConnection::Options options; - options.config = *this->rtcConfiguration; - options.factory = this->factory; + options.config = *this->rtcConfiguration; + options.factory = this->factory; this->recvTransport = this->device->CreateRecvTransport( this->recvListener, json["transportId"], @@ -228,6 +352,7 @@ namespace acgist { void Room::mediaProduceAudio(JNIEnv* env, webrtc::MediaStreamInterface* mediaStream) { if(!this->device->CanProduce("audio")) { + LOG_I("不能生产音频媒体：%s", this->roomId.data()); return; } nlohmann::json codecOptions = @@ -247,13 +372,17 @@ namespace acgist { void Room::mediaProduceVideo(JNIEnv* env, webrtc::MediaStreamInterface* mediaStream) { if(!this->device->CanProduce("video")) { + LOG_I("不能生产视频媒体：%s", this->roomId.data()); return; } - // TODO：配置读取 + // TODO：配置读取同时测试效果 nlohmann::json codecOptions = { + // x-google-start-bitrate { "videoGoogleStartBitrate", 400 }, + // x-google-min-bitrate { "videoGoogleMinBitrate", 800 }, + // x-google-max-bitrate { "videoGoogleMaxBitrate", 1600 } }; // 如果需要使用`Simulcast`打开下面配置 @@ -385,8 +514,8 @@ namespace acgist { jstring jRoomId, jobject jRouterCallback ) { jobject routerCallback = env->NewGlobalRef(jRouterCallback); - const char* roomId = env->GetStringUTFChars(jRoomId, nullptr); - Room* room = new Room(roomId, routerCallback); + const char* roomId = env->GetStringUTFChars(jRoomId, nullptr); + Room* room = new Room(roomId, routerCallback); env->DeleteLocalRef(jRoomId); env->ReleaseStringUTFChars(jRoomId, roomId); return (jlong) room; @@ -424,7 +553,7 @@ namespace acgist { extern "C" JNIEXPORT void JNICALL Java_com_acgist_taoyao_media_client_Room_nativeCreateSendTransport(JNIEnv* env, jobject me, jlong nativeRoomPointer, jstring jBody) { - Room* room = (Room*) nativeRoomPointer; + Room* room = (Room*) nativeRoomPointer; const char* body = env->GetStringUTFChars(jBody, nullptr); room->createSendTransport(env, body); env->DeleteLocalRef(jBody); @@ -433,7 +562,7 @@ namespace acgist { extern "C" JNIEXPORT void JNICALL Java_com_acgist_taoyao_media_client_Room_nativeCreateRecvTransport(JNIEnv* env, jobject me, jlong nativeRoomPointer, jstring jBody) { - Room* room = (Room*) nativeRoomPointer; + Room* room = (Room*) nativeRoomPointer; const char* body = env->GetStringUTFChars(jBody, nullptr); room->createRecvTransport(env, body); env->DeleteLocalRef(jBody); @@ -454,7 +583,7 @@ namespace acgist { extern "C" JNIEXPORT void JNICALL Java_com_acgist_taoyao_media_client_Room_nativeMediaConsume(JNIEnv* env, jobject me, jlong nativeRoomPointer, jstring jMessage) { - Room* room = (Room*) nativeRoomPointer; + Room* room = (Room*) nativeRoomPointer; const char* message = env->GetStringUTFChars(jMessage, nullptr); room->mediaConsume(env, message); env->DeleteLocalRef(jMessage); @@ -463,7 +592,7 @@ namespace acgist { extern "C" JNIEXPORT void JNICALL Java_com_acgist_taoyao_media_client_Room_nativeMediaProducerPause(JNIEnv* env, jobject me, jlong nativeRoomPointer, jstring jProducerId) { - Room* room = (Room*) nativeRoomPointer; + Room* room = (Room*) nativeRoomPointer; const char* producerId = env->GetStringUTFChars(jProducerId, nullptr); room->mediaProducerPause(env, producerId); env->DeleteLocalRef(jProducerId); @@ -472,7 +601,7 @@ namespace acgist { extern "C" JNIEXPORT void JNICALL Java_com_acgist_taoyao_media_client_Room_nativeMediaProducerResume(JNIEnv* env, jobject me, jlong nativeRoomPointer, jstring jProducerId) { - Room* room = (Room*) nativeRoomPointer; + Room* room = (Room*) nativeRoomPointer; const char* producerId = env->GetStringUTFChars(jProducerId, nullptr); room->mediaProducerResume(env, producerId); env->DeleteLocalRef(jProducerId); @@ -481,7 +610,7 @@ namespace acgist { extern "C" JNIEXPORT void JNICALL Java_com_acgist_taoyao_media_client_Room_nativeMediaProducerClose(JNIEnv* env, jobject me, jlong nativeRoomPointer, jstring jProducerId) { - Room* room = (Room*) nativeRoomPointer; + Room* room = (Room*) nativeRoomPointer; const char* producerId = env->GetStringUTFChars(jProducerId, nullptr); room->mediaProducerClose(env, producerId); env->DeleteLocalRef(jProducerId); @@ -490,7 +619,7 @@ namespace acgist { extern "C" JNIEXPORT void JNICALL Java_com_acgist_taoyao_media_client_Room_nativeMediaConsumerPause(JNIEnv* env, jobject me, jlong nativeRoomPointer, jstring jConsumerId) { - Room* room = (Room*) nativeRoomPointer; + Room* room = (Room*) nativeRoomPointer; const char* consumerId = env->GetStringUTFChars(jConsumerId, nullptr); room->mediaConsumerPause(env, consumerId); env->DeleteLocalRef(jConsumerId); @@ -499,7 +628,7 @@ namespace acgist { extern "C" JNIEXPORT void JNICALL Java_com_acgist_taoyao_media_client_Room_nativeMediaConsumerResume(JNIEnv* env, jobject me, jlong nativeRoomPointer, jstring jConsumerId) { - Room* room = (Room*) nativeRoomPointer; + Room* room = (Room*) nativeRoomPointer; const char* consumerId = env->GetStringUTFChars(jConsumerId, nullptr); room->mediaConsumerResume(env, consumerId); env->DeleteLocalRef(jConsumerId); @@ -508,7 +637,7 @@ namespace acgist { extern "C" JNIEXPORT void JNICALL Java_com_acgist_taoyao_media_client_Room_nativeMediaConsumerClose(JNIEnv* env, jobject me, jlong nativeRoomPointer, jstring jConsumerId) { - Room* room = (Room*) nativeRoomPointer; + Room* room = (Room*) nativeRoomPointer; const char* consumerId = env->GetStringUTFChars(jConsumerId, nullptr); room->mediaConsumerClose(env, consumerId); env->DeleteLocalRef(jConsumerId); diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java index 51558d4..bdb0d0e 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java @@ -30,6 +30,8 @@ import java.util.function.Consumer; * 视频会话 * SDK + WebRTC实现视频会话 * + * TODO：通过setRemoteDescription设置x-google-start-bitrate、x-google-min-bitrate、x-google-max-bitrate + * * 注意： * 2. offer/answer/candidate枚举大小 * 1. candidate格式安卓和浏览器格式不同 diff --git a/taoyao-client-web/src/App.vue b/taoyao-client-web/src/App.vue index 9c8461c..570e7b4 100644 --- a/taoyao-client-web/src/App.vue +++ b/taoyao-client-web/src/App.vue @@ -218,9 +218,10 @@ export default { .menus{width:100%;top:1rem;left:0;text-align:center;position:fixed;z-index:1;} .clients{width:100%;height:100%;top:0;left:0;position:fixed;} .client{float:left;width:50vw;height:50vh;box-shadow:0 0 1px 0px rgba(0,0,0,0.4);} -.client .buttons{width:100%;bottom:2px;left:0;text-align:center;position:absolute;padding:0.8rem 0;background:rgba(0,0,0,0.4);} -.client .buttons:after{width:0;height:2px;bottom:0;left:0;position:absolute;background:#C00;content:"";transition: all 400ms linear;} .client audio{display:none;} .client video{width:100%;height:100%;} +.client .mic{background:linear-gradient(to top, var(--el-color-primary) 10%, transparent 0%);} .client .title{position:absolute;top:0;left:0;text-align:center;width:100%;} - \ No newline at end of file +.client .buttons{width:100%;bottom:0;left:0;text-align:center;position:absolute;padding:0.8rem 0;background:rgba(0,0,0,0.4);} +.client .buttons .el-button{margin:0 6px;} + diff --git a/taoyao-client-web/src/components/Config.js b/taoyao-client-web/src/components/Config.js index e0b05d8..50488a2 100644 --- a/taoyao-client-web/src/components/Config.js +++ b/taoyao-client-web/src/components/Config.js @@ -1,6 +1,11 @@ /** * 音频默认配置 - * TODO：MediaStreamTrack.applyConstraints() + * TODO：MediaStreamTrack.applyConstraints().then().catch(); + * let setting = { + * autoGainControl: true, + * noiseSuppression: true + * } + await track.applyConstraints(Object.assign(track.getSettings(), setting)); * TODO：播放音量（audio标签配置）、采集音量 * 支持属性：navigator.mediaDevices.getSupportedConstraints() * https://developer.mozilla.org/en-US/docs/Web/API/MediaTrackSettings diff --git a/taoyao-client-web/src/components/LocalClient.vue b/taoyao-client-web/src/components/LocalClient.vue index a780021..cd0f640 100644 --- a/taoyao-client-web/src/components/LocalClient.vue +++ b/taoyao-client-web/src/components/LocalClient.vue @@ -4,11 +4,11 @@

{{ client?.name || "" }}

-
- - - - +
+ + + + @@ -107,5 +107,5 @@ export default { }; \ No newline at end of file +.client .mic{background:linear-gradient(to top, var(--el-color-primary) var(--volume), transparent 0%);} + diff --git a/taoyao-client-web/src/components/RemoteClient.vue b/taoyao-client-web/src/components/RemoteClient.vue index 518392e..97e8b22 100644 --- a/taoyao-client-web/src/components/RemoteClient.vue +++ b/taoyao-client-web/src/components/RemoteClient.vue @@ -1,15 +1,17 @@ +