From 8f2f0dd1a8af3c3165364a0df89fd84d1a5e1b3a Mon Sep 17 00:00:00 2001 From: acgist <289547414@qq.com> Date: Sat, 8 Apr 2023 15:14:56 +0800 Subject: [PATCH] =?UTF-8?q?[*]=20=E6=89=8D=E5=AD=90=E8=AF=8D=E4=BA=BA=20?= =?UTF-8?q?=E8=87=AA=E6=98=AF=E7=99=BD=E8=A1=A3=E5=8D=BF=E7=9B=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 19 +- .../acgist/taoyao/client/signal/Taoyao.java | 49 +++- .../media/src/main/cpp/include/Room.hpp | 27 +- .../taoyao/media/src/main/cpp/webrtc/Room.cpp | 262 +++++++++++++++--- .../com/acgist/taoyao/media/MediaManager.java | 80 ++---- .../acgist/taoyao/media/RouterCallback.java | 20 +- .../acgist/taoyao/media/client/Client.java | 46 +++ .../taoyao/media/client/LocalClient.java | 43 +++ .../taoyao/media/client/RemoteClient.java | 42 +++ .../com/acgist/taoyao/media/client/Room.java | 198 ++++++++++--- .../taoyao/media/client/RoomClient.java | 19 +- .../taoyao/media/client/SessionClient.java | 47 +++- taoyao-client-web/src/App.vue | 2 +- taoyao-client-web/src/components/Taoyao.js | 56 ++-- 14 files changed, 673 insertions(+), 237 deletions(-) diff --git a/README.md b/README.md index 40ddd38..50831e9 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,8 @@ ---- +> 当前程序处于开发阶段，大部分功能没有实现，可以使用的功能也没有经过大量测试，建议不要用于生产。 + ## 模块 |模块|名称|描述| @@ -40,6 +42,12 @@ |P2P|支持|实现|P2P监控模式| |WebRTC|支持|暂未实现|安卓终端支持同时进入多个房间| |RTP|支持|暂未实现|支持房间RTP推流（不会拉流）| +|录像|支持|暂未实现|| +|拍照|支持|暂未实现|| +|变声|支持|暂未实现|| +|水印|支持|暂未实现|| +|美颜|支持|暂未实现|| +|AI识别|支持|暂未实现|| ## 证书 @@ -60,14 +68,3 @@ #### 代理终端 将下级信令服务的终端全部使用代理终端注册到上级信令服务，上级信令服务代理终端处理信令时直接路由到下级路由服务，这样一级一级路由直到发送给真正的终端为止。 - -## TODO - -* 标识 -> ID -* 所有字段获取 -> get -* 优化JS错误回调 -> platform::error -* 反复测试推流拉流、拉人踢人、音频视频控制 -* 24小时不关闭媒体/一秒一次推拉流十分钟测试/三十秒推拉流一小时测试 - -* AI、美颜、水印、滤镜 -* 混音、降噪、回音消除、声音特效 \ No newline at end of file diff --git a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java index 3e3e004..8354c3e 100644 --- a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java +++ b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java @@ -499,19 +499,31 @@ public final class Taoyao implements ITaoyao { private void dispatch(final String content, final Header header, final Message message) { final Map body = message.body(); switch (header.getSignal()) { - case "client::config" -> this.clientConfig(message, body); - case "client::register" -> this.clientRegister(message, body); - case "client::reboot" -> this.clientReboot(message, body); - case "client::shutdown" -> this.clientShutdown(message, body); - case "room::close" -> this.roomClose(message, body); - case "room::enter" -> this.roomEnter(message, body); -// case "room::expel" -> this.roomExpel(message, body); - case "room::invite" -> this.roomInivte(message, body); -// case "room::leave" -> this.roomLeave(message, body); - case "session::call" -> this.sessionCall(message, body); - case "session::close" -> this.sessionClose(message, body); - case "session::exchange" -> this.sessionExchange(message, body); - default -> Log.d(Taoyao.class.getSimpleName(), "没有适配信令：" + content); + case "client::config" -> this.clientConfig(message, body); + case "client::register" -> this.clientRegister(message, body); + case "client::reboot" -> this.clientReboot(message, body); + case "client::shutdown" -> this.clientShutdown(message, body); + case "media::consume" -> this.mediaConsume(message, body); +// case "media::audio::volume" -> this.mediaAudioVolume(message, body); +// case "media::consumer::close" -> this.mediaConsumerClose(message, body); +// case "media::consumer::pause" -> this.mediaConsumerPause(message, body); +// case "media::consumer::request::key::frame" -> this.mediaConsumerRequestKeyFrame(message, body); +// case "media::consumer::resume" -> this.mediaConsumerResume(message, body); +// case "media::consumer::set::preferred::layers" -> this.mediaConsumerSetPreferredLayers(message, body); +// case "media::consumer::status" -> this.mediaConsumerStatus(message, body); +// case "media::producer::close" -> this.mediaProducerClose(message, body); +// case "media::producer::pause" -> this.mediaProducerPause(message, body); +// case "media::producer::resume" -> this.mediaProducerResume(message, body); +// case "media::producer::video::orientation:change" -> this.mediaVideoOrientationChange(message, body); + case "room::close" -> this.roomClose(message, body); + case "room::enter" -> this.roomEnter(message, body); +// case "room::expel" -> this.roomExpel(message, body); + case "room::invite" -> this.roomInivte(message, body); +// case "room::leave" -> this.roomLeave(message, body); + case "session::call" -> this.sessionCall(message, body); + case "session::close" -> this.sessionClose(message, body); + case "session::exchange" -> this.sessionExchange(message, body); + default -> Log.d(Taoyao.class.getSimpleName(), "没有适配信令：" + content); } } @@ -569,6 +581,15 @@ public final class Taoyao implements ITaoyao { Process.killProcess(Process.myPid()); } + private void mediaConsume(Message message, Map body) { + final String roomId = MapUtils.get(body, "roomId"); + final Room room = this.rooms.get(roomId); + if(room == null) { + return; + } + room.mediaConsume(message, body); + } + private void roomClose(Message message, Map body) { final String roomId = MapUtils.get(body, "roomId"); final Room room = this.rooms.remove(roomId); @@ -610,7 +631,7 @@ public final class Taoyao implements ITaoyao { ) ); room.enter(); - room.produceMedia(); + room.mediaProduce(); return room; } diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/include/Room.hpp b/taoyao-client-android/taoyao/media/src/main/cpp/include/Room.hpp index 5c49c5b..1d2d2ec 100644 --- a/taoyao-client-android/taoyao/media/src/main/cpp/include/Room.hpp +++ b/taoyao-client-android/taoyao/media/src/main/cpp/include/Room.hpp @@ -15,13 +15,18 @@ namespace acgist { class Room { public: + std::string roomId; mediasoupclient::Device *device; + mediasoupclient::PeerConnection *peerConnection; mediasoupclient::SendTransport *sendTransport; mediasoupclient::RecvTransport *recvTransport; - mediasoupclient::PeerConnection *peerConnection; mediasoupclient::SendTransport::Listener *sendListener; mediasoupclient::RecvTransport::Listener *recvListener; - std::string roomId; + mediasoupclient::Producer *audioProducer; + mediasoupclient::Producer *videoProducer; + mediasoupclient::Producer::Listener *producerListener; + mediasoupclient::Consumer::Listener *consumerListener; + std::map consumers; public: JNIEnv *env; jobject routerCallback; @@ -29,16 +34,18 @@ namespace acgist { Room(std::string roomId, JNIEnv *env, jobject routerCallback); virtual ~Room(); public: - void enter( - std::string rtpCapabilities, - webrtc::PeerConnectionFactoryInterface *factory, - webrtc::PeerConnectionInterface::RTCConfiguration &rtcConfiguration - ); + void enter(std::string rtpCapabilities, webrtc::PeerConnectionFactoryInterface *factory, webrtc::PeerConnectionInterface::RTCConfiguration &rtcConfiguration); void createSendTransport(std::string body); void createRecvTransport(std::string body); - void produceMedia(webrtc::MediaStreamInterface mediaStream); - void closeLocalClient(); - void closeRemoteClient(); + void mediaProduceAudio(webrtc::MediaStreamInterface *mediaStream); + void mediaProduceVideo(webrtc::MediaStreamInterface *mediaStream); + void mediaConsume(std::string message); + void mediaProducerPause(std::string producerId); + void mediaProducerResume(std::string producerId); + void mediaProducerClose(std::string producerId); + void mediaConsumerPause(std::string consumerId); + void mediaConsumerResume(std::string consumerId); + void mediaConsumerClose(std::string consumerId); void close(); }; diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/Room.cpp b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/Room.cpp index 2129528..0b7cdb7 100644 --- a/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/Room.cpp +++ b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/Room.cpp @@ -21,20 +21,20 @@ namespace acgist { jclass jCallbackClazz = this->env->GetObjectClass(this->routerCallback); jmethodID sendTransportConnectCallback = this->env->GetMethodID(jCallbackClazz, "sendTransportConnectCallback", "(Ljava/lang/String;Ljava/lang/String;)V"); const char *cTransportId = transport->GetId().data(); - jstring jTransportId = env->NewStringUTF(cTransportId); + jstring jTransportId = this->env->NewStringUTF(cTransportId); const char *cDtlsParameters = dtlsParameters.dump().data(); - jstring jDtlsParameters = env->NewStringUTF(cDtlsParameters); + jstring jDtlsParameters = this->env->NewStringUTF(cDtlsParameters); this->env->CallVoidMethod( this->routerCallback, sendTransportConnectCallback, jTransportId, jDtlsParameters ); - env->DeleteLocalRef(jTransportId); - env->ReleaseStringUTFChars(jTransportId, cTransportId); - env->DeleteLocalRef(jDtlsParameters); - env->ReleaseStringUTFChars(jDtlsParameters, cDtlsParameters); - env->DeleteLocalRef(jCallbackClazz); + this->env->DeleteLocalRef(jTransportId); + this->env->ReleaseStringUTFChars(jTransportId, cTransportId); + this->env->DeleteLocalRef(jDtlsParameters); + this->env->ReleaseStringUTFChars(jDtlsParameters, cDtlsParameters); + this->env->DeleteLocalRef(jCallbackClazz); return std::future(); } @@ -46,11 +46,12 @@ namespace acgist { jclass jCallbackClazz = this->env->GetObjectClass(this->routerCallback); jmethodID sendTransportProduceCallback = this->env->GetMethodID(jCallbackClazz, "sendTransportProduceCallback", "(Ljava/lang/String;Ljava/lang/String;)V"); const char *cKind = kind.data(); - jstring jKind = env->NewStringUTF(cKind); + jstring jKind = this-> env->NewStringUTF(cKind); const char *cTransportId = transport->GetId().data(); - jstring jTransportId = env->NewStringUTF(cTransportId); + jstring jTransportId = this-> env->NewStringUTF(cTransportId); const char *cRtpParameters = rtpParameters.dump().data(); - jstring jRtpParameters = env->NewStringUTF(cRtpParameters); + jstring jRtpParameters = this-> env->NewStringUTF(cRtpParameters); + std::promise promise; jstring jResult = (jstring) this->env->CallObjectMethod( this->routerCallback, sendTransportProduceCallback, @@ -58,19 +59,20 @@ namespace acgist { jTransportId, jRtpParameters ); - const char *cResult = env->GetStringUTFChars(jResult, 0); + const char *cResult = this-> env->GetStringUTFChars(jResult, 0); std::string result(cResult); - env->DeleteLocalRef(jResult); - env->DeleteLocalRef(jKind); - env->ReleaseStringUTFChars(jKind, cKind); - env->DeleteLocalRef(jResult); - env->ReleaseStringUTFChars(jResult, cResult); - env->DeleteLocalRef(jTransportId); - env->ReleaseStringUTFChars(jTransportId, cTransportId); - env->DeleteLocalRef(jRtpParameters); - env->ReleaseStringUTFChars(jRtpParameters, cRtpParameters); - env->DeleteLocalRef(jCallbackClazz); - return std::future(); + promise.set_value(result); + this-> env->DeleteLocalRef(jResult); + this-> env->DeleteLocalRef(jKind); + this-> env->ReleaseStringUTFChars(jKind, cKind); + this-> env->DeleteLocalRef(jResult); + this-> env->ReleaseStringUTFChars(jResult, cResult); + this-> env->DeleteLocalRef(jTransportId); + this-> env->ReleaseStringUTFChars(jTransportId, cTransportId); + this-> env->DeleteLocalRef(jRtpParameters); + this-> env->ReleaseStringUTFChars(jRtpParameters, cRtpParameters); + this-> env->DeleteLocalRef(jCallbackClazz); + return promise.get_future(); } std::future OnProduceData(mediasoupclient::SendTransport *transport, const nlohmann::json &sctpStreamParameters, const std::string &label, const std::string &protocol, const nlohmann::json &appData) override { @@ -95,23 +97,23 @@ namespace acgist { } std::future OnConnect(mediasoupclient::Transport *transport, const nlohmann::json &dtlsParameters) override { - jclass jCallbackClazz = env->GetObjectClass(this->routerCallback); + jclass jCallbackClazz = this->env->GetObjectClass(this->routerCallback); jmethodID recvTransportConnectCallback = this->env->GetMethodID(jCallbackClazz, "recvTransportConnectCallback", "(Ljava/lang/String;Ljava/lang/String;)V"); const char *cTransportId = transport->GetId().data(); - jstring jTransportId = env->NewStringUTF(cTransportId); + jstring jTransportId = this-> env->NewStringUTF(cTransportId); const char *cDtlsParameters = dtlsParameters.dump().data(); - jstring jDtlsParameters = env->NewStringUTF(cDtlsParameters); + jstring jDtlsParameters = this-> env->NewStringUTF(cDtlsParameters); this->env->CallVoidMethod( this->routerCallback, recvTransportConnectCallback, jTransportId, jDtlsParameters ); - env->DeleteLocalRef(jTransportId); - env->ReleaseStringUTFChars(jTransportId, cTransportId); - env->DeleteLocalRef(jDtlsParameters); - env->ReleaseStringUTFChars(jDtlsParameters, cDtlsParameters); - env->DeleteLocalRef(jCallbackClazz); + this-> env->DeleteLocalRef(jTransportId); + this-> env->ReleaseStringUTFChars(jTransportId, cTransportId); + this-> env->DeleteLocalRef(jDtlsParameters); + this-> env->ReleaseStringUTFChars(jDtlsParameters, cDtlsParameters); + this-> env->DeleteLocalRef(jCallbackClazz); return std::future(); } @@ -121,6 +123,46 @@ namespace acgist { }; + class ProducerListener : public mediasoupclient::Producer::Listener { + + public: + Room *room; + JNIEnv *env; + jobject routerCallback; + + public: + ProducerListener(Room *room, JNIEnv *env, jobject routerCallback) { + this->room = room; + this->env = env; + this->routerCallback = routerCallback; + } + + void OnTransportClose(mediasoupclient::Producer *producer) override { + + } + + }; + + class ConsumerListener : public mediasoupclient::Consumer::Listener { + + public: + Room *room; + JNIEnv *env; + jobject routerCallback; + + public: + ConsumerListener(Room *room, JNIEnv *env, jobject routerCallback) { + this->room = room; + this->env = env; + this->routerCallback = routerCallback; + } + + void OnTransportClose(mediasoupclient::Consumer *consumer) override { + + } + + }; + Room::Room( std::string roomId, JNIEnv *env, @@ -132,6 +174,8 @@ namespace acgist { this->device = new mediasoupclient::Device(); this->sendListener = new SendListener(this, env, routerCallback); this->recvListener = new RecvListener(this, env, routerCallback); + this->producerListener = new ProducerListener(this, env, routerCallback); + this->consumerListener = new ConsumerListener(this, env, routerCallback); } Room::~Room() { @@ -140,8 +184,12 @@ namespace acgist { delete this->sendTransport; delete this->recvListener; delete this->recvTransport; - env->DeleteLocalRef(this->routerCallback); - env->DeleteGlobalRef(this->routerCallback); + delete this->audioProducer; + delete this->videoProducer; + delete this->producerListener; + delete this->consumerListener; + this-> env->DeleteLocalRef(this->routerCallback); + this-> env->DeleteGlobalRef(this->routerCallback); } void Room::enter( @@ -184,10 +232,86 @@ namespace acgist { ); } - void Room::produceMedia(webrtc::MediaStreamInterface mediaStream) { -// this->device->CanProduce(); + void Room::mediaProduceAudio(webrtc::MediaStreamInterface *mediaStream) { + if(!this->device->CanProduce("audio")) { + return; + } + nlohmann::json codecOptions = + { + { "opusStereo", true }, + { "opusDtx", true } + }; + this->audioProducer = this->sendTransport->Produce( + this->producerListener, + mediaStream->GetAudioTracks()[0], + nullptr, + &codecOptions, + nullptr + ); } + void Room::mediaProduceVideo(webrtc::MediaStreamInterface *mediaStream) { + if(this->device->CanProduce("video")) { + return; + } + // TODO：配置读取 + nlohmann::json codecOptions = + { + { "videoGoogleStartBitrate", 400 }, + { "videoGoogleMinBitrate", 800 }, + { "videoGoogleMaxBitrate", 1600 } + }; + // 设置动态码率，帧率、分辨率在摄像头初始化处设置。 +// 如果需要使用`Simulcast`打开下面配置 +// std::vector encodings; +// webrtc::RtpEncodingParameters min; +// webrtc::RtpEncodingParameters mid; +// webrtc::RtpEncodingParameters max; +// min.active = true; +// min.max_framerate = 15; +// min.min_bitrate_bps = 400; +// min.max_bitrate_bps = 800; +// encodings.emplace_back(min); +// encodings.emplace_back(mid); +// encodings.emplace_back(max); + // 强制设置编码器 +// nlohmann::json codec = this->device->GetRtpCapabilities()["codec"]; + this->videoProducer = this->sendTransport->Produce( + this->producerListener, + mediaStream->GetVideoTracks()[0], + nullptr, + &codecOptions, + nullptr + ); + } + + void Room::mediaConsume(std::string message) { + nlohmann::json json = nlohmann::json::parse(message); + nlohmann::json body = json["body"]; + mediasoupclient::Consumer *consumer = this->recvTransport->Consume( + this->consumerListener, + body["consumerId"], + body["producerId"], + body["kind"], + &body["rtpParameters"] + ); + this->consumers.insert({ consumer->GetId(), consumer }); + webrtc::MediaStreamTrackInterface* trackPointer = consumer->GetTrack(); + jclass jCallbackClazz = this->env->GetObjectClass(this->routerCallback); + jmethodID consumerNewCallback = this->env->GetMethodID(jCallbackClazz, "consumerNewCallback", "(Ljava/lang/String;J;)V"); + const char *cMessage = message.data(); + jstring jMessage = this-> env->NewStringUTF(cMessage); + this->env->CallVoidMethod( + this->routerCallback, + consumerNewCallback, + jMessage, + (jlong) trackPointer + ); + this-> env->DeleteLocalRef(jMessage); + this-> env->ReleaseStringUTFChars(jMessage, cMessage); + this-> env->DeleteLocalRef(jCallbackClazz); + }; + void Room::close() { delete this->device; } @@ -249,13 +373,79 @@ namespace acgist { Room* room = (Room*) nativeRoomPointer; const char* body = env->GetStringUTFChars(jBody, 0); room->createRecvTransport(body); + env->DeleteLocalRef(jBody); env->ReleaseStringUTFChars(jBody, body); } extern "C" JNIEXPORT void JNICALL - Java_com_acgist_taoyao_media_client_Room_nativeProduceMedia(JNIEnv *env, jobject me, jlong nativeRoomPointer, jlong mediaStreamPointer) { + Java_com_acgist_taoyao_media_client_Room_nativeMediaProduceAudio(JNIEnv *env, jobject me, jlong nativeRoomPointer, jlong mediaStreamPointer) { Room* room = (Room*) nativeRoomPointer; - webrtc::MediaStreamInterface* mediaStream = reinterpret_cast(mediaStreamPointer); + webrtc::MediaStreamInterface *mediaStream = reinterpret_cast(mediaStreamPointer); + room->mediaProduceAudio(mediaStream); + } + + extern "C" JNIEXPORT void JNICALL + Java_com_acgist_taoyao_media_client_Room_nativeMediaProduceVideo(JNIEnv *env, jobject me, jlong nativeRoomPointer, jlong mediaStreamPointer) { + Room* room = (Room*) nativeRoomPointer; + webrtc::MediaStreamInterface *mediaStream = reinterpret_cast(mediaStreamPointer); + room->mediaProduceVideo(mediaStream); + } + + extern "C" JNIEXPORT void JNICALL + Java_com_acgist_taoyao_media_client_Room_nativeMediaConsume(JNIEnv *env, jobject me, jlong nativeRoomPointer, jstring jMessage) { + Room* room = (Room*) nativeRoomPointer; + const char *message = env->GetStringUTFChars(jMessage, 0); + room->mediaConsume(message); + env->DeleteLocalRef(jMessage); + env->ReleaseStringUTFChars(jMessage, message); + } + + extern "C" JNIEXPORT void JNICALL + Java_com_acgist_taoyao_media_client_Room_nativeMediaProducerPause(JNIEnv *env, jobject me, jlong nativeRoomPointer, jstring jProducerId) { + Room* room = (Room*) nativeRoomPointer; + const char *producerId = env->GetStringUTFChars(jProducerId, 0); + env->DeleteLocalRef(jProducerId); + env->ReleaseStringUTFChars(jProducerId, producerId); + } + + extern "C" JNIEXPORT void JNICALL + Java_com_acgist_taoyao_media_client_Room_nativeMediaProducerResume(JNIEnv *env, jobject me, jlong nativeRoomPointer, jstring jProducerId) { + Room* room = (Room*) nativeRoomPointer; + const char *producerId = env->GetStringUTFChars(jProducerId, 0); + env->DeleteLocalRef(jProducerId); + env->ReleaseStringUTFChars(jProducerId, producerId); + } + + extern "C" JNIEXPORT void JNICALL + Java_com_acgist_taoyao_media_client_Room_nativeMediaProducerClose(JNIEnv *env, jobject me, jlong nativeRoomPointer, jstring jProducerId) { + Room* room = (Room*) nativeRoomPointer; + const char *producerId = env->GetStringUTFChars(jProducerId, 0); + env->DeleteLocalRef(jProducerId); + env->ReleaseStringUTFChars(jProducerId, producerId); + } + + extern "C" JNIEXPORT void JNICALL + Java_com_acgist_taoyao_media_client_Room_nativeMediaConsumerPause(JNIEnv *env, jobject me, jlong nativeRoomPointer, jstring jConsumerId) { + Room* room = (Room*) nativeRoomPointer; + const char *consumerId = env->GetStringUTFChars(jConsumerId, 0); + env->DeleteLocalRef(jConsumerId); + env->ReleaseStringUTFChars(jConsumerId, consumerId); + } + + extern "C" JNIEXPORT void JNICALL + Java_com_acgist_taoyao_media_client_Room_nativeMediaConsumerResume(JNIEnv *env, jobject me, jlong nativeRoomPointer, jstring jConsumerId) { + Room* room = (Room*) nativeRoomPointer; + const char *consumerId = env->GetStringUTFChars(jConsumerId, 0); + env->DeleteLocalRef(jConsumerId); + env->ReleaseStringUTFChars(jConsumerId, consumerId); + } + + extern "C" JNIEXPORT void JNICALL + Java_com_acgist_taoyao_media_client_Room_nativeMediaConsumerClose(JNIEnv *env, jobject me, jlong nativeRoomPointer, jstring jConsumerId) { + Room* room = (Room*) nativeRoomPointer; + const char *consumerId = env->GetStringUTFChars(jConsumerId, 0); + env->DeleteLocalRef(jConsumerId); + env->ReleaseStringUTFChars(jConsumerId, consumerId); } } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java index 3f2efb6..cce0a7e 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java @@ -2,6 +2,7 @@ package com.acgist.taoyao.media; import android.content.Context; import android.content.Intent; +import android.media.AudioFormat; import android.media.MediaCodecInfo; import android.media.MediaCodecList; import android.media.projection.MediaProjection; @@ -23,6 +24,7 @@ import org.webrtc.DefaultVideoEncoderFactory; import org.webrtc.EglBase; import org.webrtc.MediaConstraints; import org.webrtc.MediaStream; +import org.webrtc.MediaStreamTrack; import org.webrtc.PeerConnectionFactory; import org.webrtc.RendererCommon; import org.webrtc.ScreenCapturerAndroid; @@ -168,10 +170,6 @@ public final class MediaManager { * 视频捕获 */ private VideoCapturer videoCapturer; - /** - * 本地视频预览 - */ - private SurfaceViewRenderer localVideoRenderer; /** * PeerConnectionFactory */ @@ -339,16 +337,20 @@ public final class MediaManager { this.recordClient.putAudio(audioSamples); } }) + // 超低延迟 +// .setUseLowLatency() // 远程音频 // .setAudioTrackStateCallback() +// .setAudioFormat(AudioFormat.ENCODING_PCM_32BIT) // .setUseHardwareNoiseSuppressor(true) // .setUseHardwareAcousticEchoCanceler(true) .createAudioDeviceModule(); this.peerConnectionFactory = PeerConnectionFactory.builder() + .setAudioDeviceModule(javaAudioDeviceModule) + // 变声 // .setAudioProcessingFactory() // .setAudioEncoderFactoryFactory(new BuiltinAudioEncoderFactoryFactory()) // .setAudioDecoderFactoryFactory(new BuiltinAudioDecoderFactoryFactory()) - .setAudioDeviceModule(javaAudioDeviceModule) .setVideoDecoderFactory(videoDecoderFactory) .setVideoEncoderFactory(videoEncoderFactory) .createPeerConnectionFactory(); @@ -479,10 +481,6 @@ public final class MediaManager { this.videoCapturer.initialize(surfaceTextureHelper, this.context, videoSource.getCapturerObserver()); this.videoCapturer.startCapture(480, 640, 30); final VideoTrack videoTrack = this.peerConnectionFactory.createVideoTrack("ARDAMSv0", videoSource); - if(preview) { - this.localVideoRenderer = this.localVideoRenderer(); - videoTrack.addSink(this.localVideoRenderer); - } videoTrack.addSink(videoFrame -> { if(this.recordClient != null) { this.recordClient.putVideo(videoFrame); @@ -493,12 +491,26 @@ public final class MediaManager { Log.i(MediaManager.class.getSimpleName(), "加载视频：" + videoTrack.id()); } + public boolean isPreview() { + return this.preview; + } + public MediaStream getMediaStream() { return this.mediaStream; } - private SurfaceViewRenderer localVideoRenderer() { - // 设置预览 + /** + * @param flag Config.WHAT_* + * @param forceEnabled 是否强制播放 + * @param videoTrack 视频媒体流Track + * + * @return 播放控件 + */ + public SurfaceViewRenderer buildSurfaceViewRenderer( + final int flag, + final VideoTrack videoTrack + ) { + // 预览控件 final SurfaceViewRenderer surfaceViewRenderer = new SurfaceViewRenderer(this.context); this.handler.post(() -> { // 视频反转 @@ -507,52 +519,20 @@ public final class MediaManager { surfaceViewRenderer.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FILL); // 硬件拉伸 surfaceViewRenderer.setEnableHardwareScaler(true); - // 加载 + // 加载OpenSL ES surfaceViewRenderer.init(this.eglBase.getEglBaseContext(), null); - }); - // 事件 -// surfaceViewRenderer.setOnClickListener(); - // TODO：迁移localvideo -// surfaceViewRenderer.release(); - // 页面加载 - final Message message = new Message(); - message.obj = surfaceViewRenderer; - message.what = Config.WHAT_NEW_LOCAL_VIDEO; - // TODO：恢复 - this.handler.sendMessage(message); - // 暂停 -// surfaceViewRenderer.pauseVideo(); - // 恢复 -// surfaceViewRenderer.disableFpsReduction(); - return surfaceViewRenderer; - } - - public void remoteVideoRenderer(final MediaStream mediaStream) { - // 设置预览 - final SurfaceViewRenderer surfaceViewRenderer = new SurfaceViewRenderer(this.context); - this.handler.post(() -> { - // 视频反转 - surfaceViewRenderer.setMirror(false); - // 视频拉伸 - surfaceViewRenderer.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FILL); - // 硬件拉伸 - surfaceViewRenderer.setEnableHardwareScaler(true); - // 加载 - surfaceViewRenderer.init(this.eglBase.getEglBaseContext(), null); - // 开始播放 - final VideoTrack videoTrack = mediaStream.videoTracks.get(0); - videoTrack.setEnabled(true); + // 强制播放 + if(!videoTrack.enabled()) { + videoTrack.setEnabled(true); + } videoTrack.addSink(surfaceViewRenderer); }); // 页面加载 final Message message = new Message(); message.obj = surfaceViewRenderer; - message.what = Config.WHAT_NEW_REMOTE_VIDEO; + message.what = flag; this.handler.sendMessage(message); - // 暂停 -// surfaceViewRenderer.pauseVideo(); - // 恢复 -// surfaceViewRenderer.disableFpsReduction(); + return surfaceViewRenderer; } public void pauseAudio() { diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RouterCallback.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RouterCallback.java index 68dec44..d74ca62 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RouterCallback.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RouterCallback.java @@ -1,5 +1,7 @@ package com.acgist.taoyao.media; +import org.webrtc.MediaStream; + /** * 路由回调 * @@ -9,15 +11,15 @@ public interface RouterCallback { default void enterCallback(String rtpCapabilities, String sctpCapabilities) {}; default void sendTransportConnectCallback(String transportId, String dtlsParameters) {}; - default String sendTransportProduceCallback(String kind, String transportId, String rtpParameters) { - return null; - }; + default String sendTransportProduceCallback(String kind, String transportId, String rtpParameters) { return null; }; default void recvTransportConnectCallback(String transportId, String dtlsParameters) {}; - default void newRemoteClientCallback() {}; - default void closeRemoteClientCallback() {}; - default void consumerPauseCallback() {}; - default void consumerResumeCallback() {}; - default void producerPauseCallback() {}; - default void producerResumeCallback() {}; + default void producerNewCallback(String kind, String producerId, long producerMediaTrackPointer) { }; + default void producerPauseCallback(String producerId) {}; + default void producerResumeCallback(String producerId) {}; + default void producerCloseCallback(String producerId) {}; + default void consumerNewCallback(String message, long consumerMediaTrackPointer) {}; + default void consumerPauseCallback(String consumerId) {}; + default void consumerResumeCallback(String consumerId) {}; + default void consumerCloseCallback(String consumerId) {}; } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Client.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Client.java index 7eaf428..40bc64a 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Client.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Client.java @@ -1,11 +1,18 @@ package com.acgist.taoyao.media.client; import android.os.Handler; +import android.os.Message; import android.util.Log; import com.acgist.taoyao.media.MediaManager; +import com.acgist.taoyao.media.config.Config; import com.acgist.taoyao.media.signal.ITaoyao; +import org.webrtc.MediaStreamTrack; +import org.webrtc.RendererCommon; +import org.webrtc.SurfaceViewRenderer; +import org.webrtc.VideoTrack; + import java.io.Closeable; /** @@ -35,6 +42,10 @@ public abstract class Client implements Closeable { * 媒体服务 */ protected final MediaManager mediaManager; + /** + * 视频预览 + */ + protected SurfaceViewRenderer surfaceViewRenderer; public Client(String name, String clientId, Handler handler, ITaoyao taoyao) { this.name = name; @@ -44,9 +55,44 @@ public abstract class Client implements Closeable { this.mediaManager = MediaManager.getInstance(); } + /** + * 播放音频 + */ + public void playAudio() { + } + + public void pauseAudio() { + } + + public void resumeAudio() { + } + + /** + * 播放视频 + */ + public void playVideo() { + } + + public void pauseVideo() { + if(this.surfaceViewRenderer != null) { + this.surfaceViewRenderer.pauseVideo(); + } + } + + public void resumeVideo() { + if(this.surfaceViewRenderer != null) { + // TODO：验证是否正确 + this.surfaceViewRenderer.disableFpsReduction(); + } + } + @Override public void close() { Log.i(this.getClass().getSimpleName(), "关闭终端：" + this.clientId); + if(this.surfaceViewRenderer != null) { + this.surfaceViewRenderer.release(); + this.surfaceViewRenderer = null; + } } } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/LocalClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/LocalClient.java index 4742dbf..1daf1be 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/LocalClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/LocalClient.java @@ -2,8 +2,12 @@ package com.acgist.taoyao.media.client; import android.os.Handler; +import com.acgist.taoyao.media.config.Config; import com.acgist.taoyao.media.signal.ITaoyao; +import org.webrtc.MediaStream; +import org.webrtc.VideoTrack; + /** * 房间本地终端 * @@ -11,13 +15,52 @@ import com.acgist.taoyao.media.signal.ITaoyao; */ public class LocalClient extends RoomClient { + /** + * 媒体流 + */ + protected MediaStream mediaStream; + public LocalClient(String name, String clientId, Handler handler, ITaoyao taoyao) { super(name, clientId, handler, taoyao); } + public MediaStream getMediaStream() { + return this.mediaStream; + } + + public void setMediaStream(MediaStream mediaStream) { + this.mediaStream = mediaStream; + } + + @Override + public void playVideo() { + super.playVideo(); + if(this.mediaStream == null) { + return; + } + final VideoTrack videoTrack = this.mediaStream.videoTracks.get(0); + if(this.surfaceViewRenderer == null) { + this.surfaceViewRenderer = this.mediaManager.buildSurfaceViewRenderer(Config.WHAT_NEW_LOCAL_VIDEO, videoTrack); + } else { + videoTrack.setEnabled(true); + } + } + + @Override + public void playAudio() { + super.playAudio(); + if(this.mediaStream == null) { + return; + } + this.mediaStream.audioTracks.forEach(v -> v.setEnabled(true)); + } + @Override public void close() { super.close(); + if(this.mediaStream != null) { + this.mediaStream.dispose(); + } } } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RemoteClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RemoteClient.java index 7d5ce9f..54b1e5a 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RemoteClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RemoteClient.java @@ -1,9 +1,17 @@ package com.acgist.taoyao.media.client; import android.os.Handler; +import android.provider.MediaStore; +import com.acgist.taoyao.media.config.Config; import com.acgist.taoyao.media.signal.ITaoyao; +import org.webrtc.MediaStreamTrack; +import org.webrtc.VideoTrack; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + /** * 房间远程终端 * @@ -11,13 +19,47 @@ import com.acgist.taoyao.media.signal.ITaoyao; */ public class RemoteClient extends RoomClient { + /** + * 媒体流Track + * 消费者ID = 媒体流Track + */ + protected final Map tracks; + public RemoteClient(String name, String clientId, Handler handler, ITaoyao taoyao) { super(name, clientId, handler, taoyao); + this.tracks = new ConcurrentHashMap<>(); + } + + @Override + public void playVideo() { + super.playVideo(); + final VideoTrack videoTrack = (VideoTrack) this.tracks.values().stream() + .filter(v -> MediaStreamTrack.VIDEO_TRACK_KIND.equals(v.kind())) + .findFirst() + .orElse(null); + if(videoTrack == null) { + return; + } + if(this.surfaceViewRenderer == null) { + this.surfaceViewRenderer = this.mediaManager.buildSurfaceViewRenderer(Config.WHAT_NEW_REMOTE_VIDEO, videoTrack); + } else { + videoTrack.setEnabled(true); + } + } + + @Override + public void playAudio() { + super.playAudio(); + this.tracks.values().stream() + .filter(v -> MediaStreamTrack.AUDIO_TRACK_KIND.equals(v.kind())) + .forEach(v -> v.setEnabled(true)); } @Override public void close() { super.close(); + this.tracks.values().forEach(MediaStreamTrack::dispose); + this.tracks.clear(); } } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Room.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Room.java index da0461c..c1d4b14 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Room.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Room.java @@ -6,18 +6,24 @@ import android.util.Log; import com.acgist.taoyao.boot.model.Message; import com.acgist.taoyao.boot.utils.JSONUtils; import com.acgist.taoyao.boot.utils.MapUtils; +import com.acgist.taoyao.boot.utils.PointerUtils; import com.acgist.taoyao.media.MediaManager; import com.acgist.taoyao.media.RouterCallback; +import com.acgist.taoyao.media.config.Config; import com.acgist.taoyao.media.signal.ITaoyao; +import org.webrtc.AudioTrack; +import org.webrtc.MediaStream; +import org.webrtc.MediaStreamTrack; import org.webrtc.PeerConnection; import org.webrtc.PeerConnectionFactory; +import org.webrtc.VideoTrack; import java.io.Closeable; import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.ConcurrentHashMap; /** * 房间 @@ -42,7 +48,7 @@ public class Room implements Closeable, RouterCallback { private final MediaManager mediaManager; private volatile boolean enter; private LocalClient localClient; - private List remoteClients; + private Map remoteClients; private PeerConnection.RTCConfiguration rtcConfiguration; private PeerConnectionFactory peerConnectionFactory; private String sctpCapabilities; @@ -54,39 +60,35 @@ public class Room implements Closeable, RouterCallback { boolean dataConsume, boolean audioConsume, boolean videoConsume, boolean dataProduce, boolean audioProduce, boolean videoProduce ) { - this.name = name; + this.name = name; this.clientId = clientId; - this.roomId = roomId; + this.roomId = roomId; this.password = password; - this.handler = handler; - this.taoyao = taoyao; - this.dataConsume = dataConsume; + this.handler = handler; + this.taoyao = taoyao; + this.dataConsume = dataConsume; this.audioConsume = audioConsume; this.videoConsume = videoConsume; - this.dataProduce = dataProduce; + this.dataProduce = dataProduce; this.audioProduce = audioProduce; this.videoProduce = videoProduce; this.nativeRoomPointer = this.nativeNewRoom(roomId, this); this.mediaManager = MediaManager.getInstance(); - this.remoteClients = new CopyOnWriteArrayList<>(); + this.remoteClients = new ConcurrentHashMap<>(); this.enter = false; } - /** - * 远程终端列表 - */ - private List remoteClientList; - public synchronized void enter() { - if(this.enter) { + if (this.enter) { return; } final Message response = this.taoyao.request(this.taoyao.buildMessage("media::router::rtp::capabilities", "roomId", this.roomId)); - if(response == null) { + if (response == null) { Log.w(Room.class.getSimpleName(), "获取通道能力失败"); return; } this.localClient = new LocalClient(this.name, this.clientId, this.handler, this.taoyao); + this.localClient.setMediaStream(this.mediaManager.getMediaStream()); // STUN | TURN final List iceServers = new ArrayList<>(); // TODO：读取配置 @@ -98,14 +100,21 @@ public class Room implements Closeable, RouterCallback { this.nativeEnter(this.nativeRoomPointer, JSONUtils.toJSON(rtpCapabilities), this.peerConnectionFactory.getNativePeerConnectionFactory(), this.rtcConfiguration); } - public void produceMedia() { - if(this.audioProduce || this.videoProduce) { + public void mediaProduce() { + if (this.audioProduce || this.videoProduce) { this.createSendTransport(); } - if(this.audioConsume || this.videoConsume) { + if (this.audioConsume || this.videoConsume) { this.createRecvTransport(); } - + final MediaStream mediaStream = this.localClient.getMediaStream(); + final long mediaStreamPointer = PointerUtils.getNativePointer(mediaStream, "nativeStream"); + if (this.audioProduce) { + this.nativeMediaProduceAudio(this.nativeRoomPointer, mediaStreamPointer); + } + if (this.videoProduce) { + this.nativeMediaProduceVideo(this.nativeRoomPointer, mediaStreamPointer); + } } private void createSendTransport() { @@ -117,7 +126,7 @@ public class Room implements Closeable, RouterCallback { "consuming", false, "sctpCapabilities", this.dataProduce ? this.sctpCapabilities : null )); - if(response == null) { + if (response == null) { Log.w(Room.class.getSimpleName(), "创建发送通道失败"); return; } @@ -132,42 +141,110 @@ public class Room implements Closeable, RouterCallback { "consuming", true, "sctpCapabilities", this.dataProduce ? this.sctpCapabilities : null )); - if(response == null) { + if (response == null) { Log.w(Room.class.getSimpleName(), "创建接收通道失败"); return; } this.nativeCreateRecvTransport(this.nativeRoomPointer, JSONUtils.toJSON(response.body())); } + public void mediaConsume(Message message, Map body) { + this.nativeMediaConsume(this.nativeRoomPointer, JSONUtils.toJSON(message)); + } + /** * 新增远程终端 * * @param body 消息主体 */ public void newRemoteClient(Map body) { - final String clientId = MapUtils.get(body, "clientId"); - final Map status = MapUtils.get(body, "status"); - final String name = MapUtils.get(status, "name"); - final RemoteClient remoteClient = new RemoteClient(name, clientId, this.handler, this.taoyao); - this.remoteClients.add(remoteClient); + synchronized (this.remoteClients) { + final String clientId = MapUtils.get(body, "clientId"); + final Map status = MapUtils.get(body, "status"); + final String name = MapUtils.get(status, "name"); + final RemoteClient remoteClient = new RemoteClient(name, clientId, this.handler, this.taoyao); + final RemoteClient old = this.remoteClients.put(clientId, remoteClient); + if(old != null) { + // 关闭旧的资源 + old.close(); + } + } + } + + public void closeRemoteClient(String clientId) { + synchronized (this.remoteClients) { + final RemoteClient remoteClient = this.remoteClients.get(clientId); + if(remoteClient == null) { + return; + } + remoteClient.close(); + } } @Override public void close() { Log.i(Room.class.getSimpleName(), "关闭房间：" + this.roomId); this.localClient.close(); - this.remoteClientList.forEach(RemoteClient::close); + this.remoteClients.values().forEach(v -> this.closeRemoteClient(v.clientId)); this.mediaManager.closeClient(); this.nativeCloseRoom(this.nativeRoomPointer); } + public void mediaConsumerClose(String consumerId) { + this.taoyao.push(this.taoyao.buildMessage( + "media::consumer::close", + "roomId", this.roomId, + "consumerId", consumerId + )); + } + + public void mediaConsumerPause(String consumerId) { + this.taoyao.push(this.taoyao.buildMessage( + "media::consumer::pause", + "roomId", this.roomId, + "consumerId", consumerId + )); + } + + public void mediaConsumerResume(String consumerId) { + this.taoyao.push(this.taoyao.buildMessage( + "media::consumer::resume", + "roomId", this.roomId, + "consumerId", consumerId + )); + } + + public void mediaProducerClose(String producerId) { + this.taoyao.push(this.taoyao.buildMessage( + "media::producer::close", + "roomId", this.roomId, + "producerId", producerId + )); + } + + public void mediaProducerPause(String producerId) { + this.taoyao.push(this.taoyao.buildMessage( + "media::producer::pause", + "roomId", this.roomId, + "producerId", producerId + )); + } + + public void mediaProducerResume(String producerId) { + this.taoyao.push(this.taoyao.buildMessage( + "media::producer::resume", + "roomId", this.roomId, + "producerId", producerId + )); + } + @Override public void enterCallback(String rtpCapabilities, String sctpCapabilities) { this.taoyao.request(this.taoyao.buildMessage( "room::enter", - "roomId", this.roomId, - "password", this.password, - "rtpCapabilities", rtpCapabilities, + "roomId", this.roomId, + "password", this.password, + "rtpCapabilities", rtpCapabilities, "sctpCapabilities", sctpCapabilities )); this.enter = true; @@ -177,8 +254,8 @@ public class Room implements Closeable, RouterCallback { public void sendTransportConnectCallback(String transportId, String dtlsParameters) { this.taoyao.request(this.taoyao.buildMessage( "media::transport::webrtc::connect", - "roomId", this.roomId, - "transportId", transportId, + "roomId", this.roomId, + "transportId", transportId, "dtlsParameters", JSONUtils.toMap(dtlsParameters) )); } @@ -187,9 +264,9 @@ public class Room implements Closeable, RouterCallback { public String sendTransportProduceCallback(String kind, String transportId, String rtpParameters) { final Message response = this.taoyao.request(this.taoyao.buildMessage( "media::produce", - "kind", kind, - "roomId", this.roomId, - "transportId", transportId, + "kind", kind, + "roomId", this.roomId, + "transportId", transportId, "rtpParameters", JSONUtils.toMap(rtpParameters) )); final Map body = response.body(); @@ -200,22 +277,53 @@ public class Room implements Closeable, RouterCallback { public void recvTransportConnectCallback(String transportId, String dtlsParameters) { this.taoyao.request(this.taoyao.buildMessage( "media::transport::webrtc::connect", - "roomId", this.roomId, - "transportId", transportId, + "roomId", this.roomId, + "transportId", transportId, "dtlsParameters", JSONUtils.toMap(dtlsParameters) )); } - private native void nativeEnter( - long nativePointer, - String rtpCapabilities, - long peerConnectionFactoryPointer, - PeerConnection.RTCConfiguration rtcConfiguration - ); + @Override + public void consumerNewCallback(String message, long consumerMediaTrackPointer) { + final Message response = JSONUtils.toJava(message, Message.class); + final Map body = response.body(); + final String kind = MapUtils.get(body, "kind"); + final String clientId = MapUtils.get(body, "clientId"); + final String consumerId = MapUtils.get(body, "consumerId"); + final RemoteClient remoteClient = this.remoteClients.get(clientId); + if(remoteClient == null) { + // TODO：资源释放 + return; + } + if(MediaStreamTrack.AUDIO_TRACK_KIND.equals(kind)) { +// WebRtcAudioTrack + final AudioTrack audioTrack = new AudioTrack(consumerMediaTrackPointer); + remoteClient.tracks.put(consumerId, audioTrack); + } else if(MediaStreamTrack.VIDEO_TRACK_KIND.equals(kind)) { + final VideoTrack videoTrack = new VideoTrack(consumerMediaTrackPointer); + remoteClient.tracks.put(consumerId, videoTrack); + remoteClient.playVideo(); + } else { + Log.w(Room.class.getSimpleName(), "未知媒体类型：" + kind); + // TODO：资源释放 + return; + } + this.taoyao.push(response); + } + + private native void nativeEnter(long nativePointer, String rtpCapabilities, long peerConnectionFactoryPointer, PeerConnection.RTCConfiguration rtcConfiguration); private native long nativeNewRoom(String roomId, RouterCallback routerCallback); private native void nativeCloseRoom(long nativePointer); private native void nativeCreateSendTransport(long nativeRoomPointer, String body); private native void nativeCreateRecvTransport(long nativeRoomPointer, String body); - private native void nativeProduceMedia(long nativeRoomPointer, long mediaStreamPointer); + private native void nativeMediaProduceAudio(long nativeRoomPointer, long mediaStreamPointer); + private native void nativeMediaProduceVideo(long nativeRoomPointer, long mediaStreamPointer); + private native void nativeMediaConsume(long nativeRoomPointer, String message); + private native void nativeMediaProducerPause(long nativeRoomPointer, String producerId); + private native void nativeMediaProducerResume(long nativeRoomPointer, String producerId); + private native void nativeMediaProducerClose(long nativeRoomPointer, String producerId); + private native void nativeMediaConsumerPause(long nativeRoomPointer, String consumerId); + private native void nativeMediaConsumerResume(long nativeRoomPointer, String consumerId); + private native void nativeMediaConsumerClose(long nativeRoomPointer, String consumerId); } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RoomClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RoomClient.java index 70bcd86..deb4fc8 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RoomClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RoomClient.java @@ -4,7 +4,7 @@ import android.os.Handler; import com.acgist.taoyao.media.signal.ITaoyao; -import org.webrtc.MediaStream; +import org.webrtc.MediaStreamTrack; /** * 房间终端 @@ -14,30 +14,13 @@ import org.webrtc.MediaStream; */ public class RoomClient extends Client { - protected MediaStream mediaStream; - public RoomClient(String name, String clientId, Handler handler, ITaoyao taoyao) { super(name, clientId, handler, taoyao); } - /** - * 打开预览 - */ - private void preview() { - - } - @Override public void close() { super.close(); } - public MediaStream getMediaStream() { - return mediaStream; - } - - public void setMediaStream(MediaStream mediaStream) { - this.mediaStream = mediaStream; - } - } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java index 8be4898..22def61 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java @@ -1,11 +1,13 @@ package com.acgist.taoyao.media.client; import android.os.Handler; +import android.se.omapi.Session; import android.util.Log; import com.acgist.taoyao.boot.model.Message; import com.acgist.taoyao.boot.utils.MapUtils; import com.acgist.taoyao.media.MediaManager; +import com.acgist.taoyao.media.config.Config; import com.acgist.taoyao.media.signal.ITaoyao; import org.webrtc.DataChannel; @@ -16,6 +18,7 @@ import org.webrtc.PeerConnection; import org.webrtc.PeerConnectionFactory; import org.webrtc.SdpObserver; import org.webrtc.SessionDescription; +import org.webrtc.VideoTrack; import java.util.ArrayList; import java.util.List; @@ -43,7 +46,7 @@ public class SessionClient extends Client { /** * 远程媒体 */ - private final List remoteMediaStreams; + private MediaStream remoteMediaStream; /** * SDPObserver */ @@ -64,7 +67,6 @@ public class SessionClient extends Client { public SessionClient(String sessionId, String name, String clientId, Handler handler, ITaoyao taoyao) { super(name, clientId, handler, taoyao); this.sessionId = sessionId; - this.remoteMediaStreams = new CopyOnWriteArrayList<>(); } public void init() { @@ -151,14 +153,36 @@ public class SessionClient extends Client { } } + @Override + public void playVideo() { + super.playVideo(); + if(this.remoteMediaStream == null) { + return; + } + final VideoTrack videoTrack = this.remoteMediaStream.videoTracks.get(0); + if(this.surfaceViewRenderer == null) { + this.surfaceViewRenderer = this.mediaManager.buildSurfaceViewRenderer(Config.WHAT_NEW_REMOTE_VIDEO, videoTrack); + } else { + videoTrack.setEnabled(true); + } + } + + @Override + public void playAudio() { + super.playAudio(); + if(this.remoteMediaStream == null) { + return; + } + this.remoteMediaStream.audioTracks.forEach(v -> v.setEnabled(true)); + } + @Override public void close() { super.close(); - // 本地资源释放 + // 本地资源释放：不要直接关闭MediaStream（共享资源） this.mediaManager.closeClient(); // 远程资源释放 - this.remoteMediaStreams.forEach(v -> v.dispose()); - this.remoteMediaStreams.clear(); + this.remoteMediaStream.dispose(); } /** @@ -201,14 +225,19 @@ public class SessionClient extends Client { @Override public void onAddStream(MediaStream mediaStream) { Log.i(SessionClient.class.getSimpleName(), "添加远程媒体：" + SessionClient.this.clientId); - SessionClient.this.mediaStream = mediaStream; - SessionClient.this.mediaManager.remoteVideoRenderer(mediaStream); - SessionClient.this.remoteMediaStreams.add(mediaStream); + if(SessionClient.this.remoteMediaStream != null) { + // TODO：验证音频视频是否合在一起 + } + SessionClient.this.remoteMediaStream = mediaStream; + SessionClient.this.playVideo(); + SessionClient.this.playAudio(); } @Override public void onRemoveStream(MediaStream mediaStream) { - SessionClient.this.remoteMediaStreams.remove(mediaStream); + if(SessionClient.this.remoteMediaStream == mediaStream) { + + } } @Override diff --git a/taoyao-client-web/src/App.vue b/taoyao-client-web/src/App.vue index 05b3f4c..7ed1186 100644 --- a/taoyao-client-web/src/App.vue +++ b/taoyao-client-web/src/App.vue @@ -170,7 +170,7 @@ export default { }, async roomEnter() { await this.taoyao.roomEnter(this.room.roomId, this.room.password); - await this.taoyao.produceMedia(); + await this.taoyao.mediaProduce(); this.roomVisible = false; }, async roomInvite() { diff --git a/taoyao-client-web/src/components/Taoyao.js b/taoyao-client-web/src/components/Taoyao.js index 1c2f627..4dc7398 100644 --- a/taoyao-client-web/src/components/Taoyao.js +++ b/taoyao-client-web/src/components/Taoyao.js @@ -1173,13 +1173,13 @@ class Taoyao extends RemoteClient { kind, type, roomId, + appData, clientId, sourceId, streamId, producerId, consumerId, rtpParameters, - appData, producerPaused, } = message.body; try { @@ -1188,11 +1188,8 @@ class Taoyao extends RemoteClient { kind, producerId, rtpParameters, - // NOTE: Force streamId to be same in mic and webcam and different - // in screen sharing so libwebrtc will just try to sync mic and - // webcam streams from the same remote peer. - //streamId: `${peerId}-${appData.share ? "share" : "mic-webcam"}`, - streamId: `${clientId}-${appData.share ? "share" : "mic-webcam"}`, + // 强制设置streamId，让libwebrtc同步麦克风和摄像头，屏幕共享不要求同步。 + streamId: `${clientId}-${appData?.videoSource ? appData.videoSource : "unknown"}`, appData, // Trick. }); consumer.clientId = clientId; @@ -1206,25 +1203,6 @@ class Taoyao extends RemoteClient { mediasoupClient.parseScalabilityMode( consumer.rtpParameters.encodings[0].scalabilityMode ); - // store.dispatch( - // stateActions.addConsumer( - // { - // id: consumer.id, - // type: type, - // locallyPaused: false, - // remotelyPaused: producerPaused, - // rtpParameters: consumer.rtpParameters, - // spatialLayers: spatialLayers, - // temporalLayers: temporalLayers, - // preferredSpatialLayer: spatialLayers - 1, - // preferredTemporalLayer: temporalLayers - 1, - // priority: 1, - // codec: consumer.rtpParameters.codecs[0].mimeType.split("/")[1], - // track: consumer.track, - // }, - // peerId - // ) - // ); self.push(message); console.debug("远程媒体：", consumer); const remoteClient = self.remoteClients.get(consumer.sourceId); @@ -1244,7 +1222,7 @@ class Taoyao extends RemoteClient { } else { console.warn("远程终端没有实现服务代理：", remoteClient); } - // If audio-only mode is enabled, pause it. + // 实现进入自动暂停视频，注：必须订阅所有类型媒体，不然媒体服务直接不会转发视频媒体 if (consumer.kind === "video" && !self.videoProduce) { // this.pauseConsumer(consumer); // TODO：实现 @@ -1509,7 +1487,7 @@ class Taoyao extends RemoteClient { // return; // } await me.roomEnter(roomId, password); - await me.produceMedia(); + await me.mediaProduce(); } /** * 离开房间 @@ -1583,7 +1561,7 @@ class Taoyao extends RemoteClient { /** * 生产媒体 */ - async produceMedia() { + async mediaProduce() { const self = this; if (!self.roomId) { this.callbackError("无效房间"); @@ -1828,6 +1806,7 @@ class Taoyao extends RemoteClient { self.callbackError("麦克风打开失败"); } } + async closeAudioProducer() { console.debug("closeAudioProducer()"); if (!this.audioProducer) { @@ -1871,27 +1850,36 @@ class Taoyao extends RemoteClient { let codec; let encodings; const codecOptions = { - videoGoogleStartBitrate: 1000, + videoGoogleStartBitrate: 400, + videoGoogleMaxBitrate : 1600, + videoGoogleMinBitrate : 800 }; +// encodings : +// [ +// { maxBitrate: 100000 }, +// { maxBitrate: 300000 }, +// { maxBitrate: 900000 } +// ], +// codecOptions : +// { +// videoGoogleStartBitrate : 1000 +// } if (self.forceH264) { codec = self.mediasoupDevice.rtpCapabilities.codecs.find( (c) => c.mimeType.toLowerCase() === "video/h264" ); if (!codec) { - throw new Error( - "desired H264 codec+configuration is not supported" - ); + self.callbackError("不支持H264视频编码"); } } else if (self.forceVP9) { codec = self.mediasoupDevice.rtpCapabilities.codecs.find( (c) => c.mimeType.toLowerCase() === "video/vp9" ); if (!codec) { - throw new Error("desired VP9 codec+configuration is not supported"); + self.callbackError("不支持VP9视频编码"); } } if (this.useSimulcast) { - // If VP9 is the only available video codec then use SVC. const firstVideoCodec = this.mediasoupDevice.rtpCapabilities.codecs.find( (c) => c.kind === "video"