From 9978039c9d0af4cedd667b495dc51f2180acc2ee Mon Sep 17 00:00:00 2001 From: acgist <289547414@qq.com> Date: Fri, 7 Apr 2023 21:12:25 +0800 Subject: [PATCH] =?UTF-8?q?[*]=20=E8=BF=9B=E6=AD=A5=E4=B8=80=E7=82=B9?= =?UTF-8?q?=E7=82=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../media/src/main/cpp/include/Room.hpp | 24 ++- .../taoyao/media/src/main/cpp/webrtc/Room.cpp | 147 ++++++++++++++---- .../acgist/taoyao/media/RouterCallback.java | 7 +- .../com/acgist/taoyao/media/client/Room.java | 59 ++++++- taoyao-client-web/src/components/Taoyao.js | 38 +++-- 5 files changed, 202 insertions(+), 73 deletions(-) diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/include/Room.hpp b/taoyao-client-android/taoyao/media/src/main/cpp/include/Room.hpp index d7a9f01..5c49c5b 100644 --- a/taoyao-client-android/taoyao/media/src/main/cpp/include/Room.hpp +++ b/taoyao-client-android/taoyao/media/src/main/cpp/include/Room.hpp @@ -5,37 +5,31 @@ #include #include "jni.h" -#include "Log.hpp" -#include "mediasoupclient.hpp" - #include "sdk/android/src/jni/pc/peer_connection.h" #include "sdk/android/native_api/jni/scoped_java_ref.h" +#include "Log.hpp" +#include "mediasoupclient.hpp" + namespace acgist { class Room { public: mediasoupclient::Device *device; - mediasoupclient::PeerConnection *peerConnection; mediasoupclient::SendTransport *sendTransport; mediasoupclient::RecvTransport *recvTransport; + mediasoupclient::PeerConnection *peerConnection; mediasoupclient::SendTransport::Listener *sendListener; mediasoupclient::RecvTransport::Listener *recvListener; - jstring roomId; + std::string roomId; public: - /** - * 新建Transport回调 - */ - jmethodID newCallback; - /** - * 房间关闭回调 - */ - jmethodID closeCallback; + JNIEnv *env; + jobject routerCallback; public: - Room(jstring roomId); + Room(std::string roomId, JNIEnv *env, jobject routerCallback); virtual ~Room(); public: - void load( + void enter( std::string rtpCapabilities, webrtc::PeerConnectionFactoryInterface *factory, webrtc::PeerConnectionInterface::RTCConfiguration &rtcConfiguration diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/Room.cpp b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/Room.cpp index b35a6c8..2129528 100644 --- a/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/Room.cpp +++ b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/Room.cpp @@ -4,18 +4,77 @@ namespace acgist { class SendListener : public mediasoupclient::SendTransport::Listener { + public: + Room *room; + JNIEnv *env; + jobject routerCallback; + + public: + SendListener(Room *room, JNIEnv *env, jobject routerCallback) { + this->room = room; + this->env = env; + this->routerCallback = routerCallback; + } + + public: std::future OnConnect(mediasoupclient::Transport *transport, const nlohmann::json &dtlsParameters) override { + jclass jCallbackClazz = this->env->GetObjectClass(this->routerCallback); + jmethodID sendTransportConnectCallback = this->env->GetMethodID(jCallbackClazz, "sendTransportConnectCallback", "(Ljava/lang/String;Ljava/lang/String;)V"); + const char *cTransportId = transport->GetId().data(); + jstring jTransportId = env->NewStringUTF(cTransportId); + const char *cDtlsParameters = dtlsParameters.dump().data(); + jstring jDtlsParameters = env->NewStringUTF(cDtlsParameters); + this->env->CallVoidMethod( + this->routerCallback, + sendTransportConnectCallback, + jTransportId, + jDtlsParameters + ); + env->DeleteLocalRef(jTransportId); + env->ReleaseStringUTFChars(jTransportId, cTransportId); + env->DeleteLocalRef(jDtlsParameters); + env->ReleaseStringUTFChars(jDtlsParameters, cDtlsParameters); + env->DeleteLocalRef(jCallbackClazz); return std::future(); } void OnConnectionStateChange(mediasoupclient::Transport *transport, const std::string &connectionState) override { + // 状态变化 } std::future OnProduce(mediasoupclient::SendTransport *transport, const std::string &kind, nlohmann::json rtpParameters, const nlohmann::json &appData) override { + jclass jCallbackClazz = this->env->GetObjectClass(this->routerCallback); + jmethodID sendTransportProduceCallback = this->env->GetMethodID(jCallbackClazz, "sendTransportProduceCallback", "(Ljava/lang/String;Ljava/lang/String;)V"); + const char *cKind = kind.data(); + jstring jKind = env->NewStringUTF(cKind); + const char *cTransportId = transport->GetId().data(); + jstring jTransportId = env->NewStringUTF(cTransportId); + const char *cRtpParameters = rtpParameters.dump().data(); + jstring jRtpParameters = env->NewStringUTF(cRtpParameters); + jstring jResult = (jstring) this->env->CallObjectMethod( + this->routerCallback, + sendTransportProduceCallback, + jKind, + jTransportId, + jRtpParameters + ); + const char *cResult = env->GetStringUTFChars(jResult, 0); + std::string result(cResult); + env->DeleteLocalRef(jResult); + env->DeleteLocalRef(jKind); + env->ReleaseStringUTFChars(jKind, cKind); + env->DeleteLocalRef(jResult); + env->ReleaseStringUTFChars(jResult, cResult); + env->DeleteLocalRef(jTransportId); + env->ReleaseStringUTFChars(jTransportId, cTransportId); + env->DeleteLocalRef(jRtpParameters); + env->ReleaseStringUTFChars(jRtpParameters, cRtpParameters); + env->DeleteLocalRef(jCallbackClazz); return std::future(); } std::future OnProduceData(mediasoupclient::SendTransport *transport, const nlohmann::json &sctpStreamParameters, const std::string &label, const std::string &protocol, const nlohmann::json &appData) override { + // 数据生产 return std::future(); } @@ -23,20 +82,56 @@ namespace acgist { class RecvListener : public mediasoupclient::RecvTransport::Listener { + public: + Room *room; + JNIEnv *env; + jobject routerCallback; + + public: + RecvListener(Room *room, JNIEnv *env, jobject routerCallback) { + this->room = room; + this->env = env; + this->routerCallback = routerCallback; + } + std::future OnConnect(mediasoupclient::Transport *transport, const nlohmann::json &dtlsParameters) override { + jclass jCallbackClazz = env->GetObjectClass(this->routerCallback); + jmethodID recvTransportConnectCallback = this->env->GetMethodID(jCallbackClazz, "recvTransportConnectCallback", "(Ljava/lang/String;Ljava/lang/String;)V"); + const char *cTransportId = transport->GetId().data(); + jstring jTransportId = env->NewStringUTF(cTransportId); + const char *cDtlsParameters = dtlsParameters.dump().data(); + jstring jDtlsParameters = env->NewStringUTF(cDtlsParameters); + this->env->CallVoidMethod( + this->routerCallback, + recvTransportConnectCallback, + jTransportId, + jDtlsParameters + ); + env->DeleteLocalRef(jTransportId); + env->ReleaseStringUTFChars(jTransportId, cTransportId); + env->DeleteLocalRef(jDtlsParameters); + env->ReleaseStringUTFChars(jDtlsParameters, cDtlsParameters); + env->DeleteLocalRef(jCallbackClazz); return std::future(); } void OnConnectionStateChange(mediasoupclient::Transport *transport, const std::string &connectionState) override { + // 状态变化 } }; - Room::Room(jstring roomId) { + Room::Room( + std::string roomId, + JNIEnv *env, + jobject routerCallback + ) { this->roomId = roomId; + this->env = env; + this->routerCallback = routerCallback; this->device = new mediasoupclient::Device(); - this->sendListener = new SendListener(); - this->recvListener = new RecvListener(); + this->sendListener = new SendListener(this, env, routerCallback); + this->recvListener = new RecvListener(this, env, routerCallback); } Room::~Room() { @@ -45,9 +140,11 @@ namespace acgist { delete this->sendTransport; delete this->recvListener; delete this->recvTransport; + env->DeleteLocalRef(this->routerCallback); + env->DeleteGlobalRef(this->routerCallback); } - void Room::load( + void Room::enter( std::string rtpCapabilities, webrtc::PeerConnectionFactoryInterface *factory, webrtc::PeerConnectionInterface::RTCConfiguration &rtcConfiguration @@ -93,11 +190,10 @@ namespace acgist { void Room::close() { delete this->device; - this->device = nullptr; } extern "C" JNIEXPORT void JNICALL - Java_com_acgist_taoyao_media_client_Room_nativeLoad( + Java_com_acgist_taoyao_media_client_Room_nativeEnter( JNIEnv* env, jobject me, jlong nativeRoomPointer, jstring jRtpCapabilities, jlong factoryPointer, jobject jRtcConfiguration @@ -105,48 +201,36 @@ namespace acgist { Room* room = (Room*) nativeRoomPointer; webrtc::PeerConnectionInterface::RTCConfiguration rtcConfiguration(webrtc::PeerConnectionInterface::RTCConfigurationType::kAggressive); // TODO：为什么不能转换？测试是否因为stun配置问题 - jobject jRtcConfigurationGlobal = webrtc::jni::NewGlobalRef(env, jRtcConfiguration); - webrtc::JavaParamRef jRtcConfigurationRef(jRtcConfigurationGlobal); + webrtc::JavaParamRef jRtcConfigurationRef(jRtcConfiguration); // webrtc::jni::JavaToNativeMediaConstraints() webrtc::jni::JavaToNativeRTCConfiguration(env, jRtcConfigurationRef, &rtcConfiguration); - webrtc::jni::DeleteGlobalRef(env, jRtcConfigurationGlobal); const char* rtpCapabilities = env->GetStringUTFChars(jRtpCapabilities, 0); - room->load( + room->enter( rtpCapabilities, reinterpret_cast(factoryPointer), // (webrtc::PeerConnectionFactoryInterface*) factoryPointer, rtcConfiguration ); env->ReleaseStringUTFChars(jRtpCapabilities, rtpCapabilities); + env->DeleteLocalRef(jRtpCapabilities); + env->DeleteLocalRef(jRtcConfiguration); // delete rtpCapabilities; } extern "C" JNIEXPORT jlong JNICALL - Java_com_acgist_taoyao_media_client_Room_nativeNewRoom(JNIEnv *env, jobject me, jstring roomId) { - const Room* room = new Room(roomId); + Java_com_acgist_taoyao_media_client_Room_nativeNewRoom( + JNIEnv *env, jobject me, + jstring jRoomId, jobject jRouterCallback + ) { + const char* roomId = env->GetStringUTFChars(jRoomId, 0); + jobject routerCallback = env->NewGlobalRef(jRouterCallback); + Room* room = new Room(roomId, env, routerCallback); + env->ReleaseStringUTFChars(jRoomId, roomId); return (jlong) room; } extern "C" JNIEXPORT void JNICALL Java_com_acgist_taoyao_media_client_Room_nativeCloseRoom(JNIEnv *env, jobject me, jlong nativeRoomPointer) { -// JNIEXPORT void JNICALL -// Java_nativeMethod -// (JNIEnv *env, jobject thiz) { -// MyCPlusObj *obj = new MyCPlusObj(); -// jclass clazz = (jclass)(*env).GetObjectClass(thiz); -// jfieldID fid = (jfieldID)(*env).GetFieldID(clazz, "mObj", "I"); -// (*env).SetIntField(thiz, fid, (jint)obj); -// } - -// jclass objClazz = (jclass)env->GetObjectClass(obj);//obj为对应的JAVA对象 -// jfieldID fid = env->GetFieldID(objClazz, "mObj", "I"); -// jlong p = (jlong)env->GetObjectField(obj, fid); -// MyCPlusObj *cPlusObj = (MyCPlusObj *)p; -////cPlusObj 为JAVA对象对应的C++对象 - -// jobject gThiz = (jobject)env->NewGlobalRef(thiz);//thiz为JAVA对象 -// (*obj).javaObj = (jint)gThiz; - Room* room = (Room*) nativeRoomPointer; room->close(); delete room; @@ -163,6 +247,9 @@ namespace acgist { extern "C" JNIEXPORT void JNICALL Java_com_acgist_taoyao_media_client_Room_nativeCreateRecvTransport(JNIEnv *env, jobject me, jlong nativeRoomPointer, jstring jBody) { Room* room = (Room*) nativeRoomPointer; + const char* body = env->GetStringUTFChars(jBody, 0); + room->createRecvTransport(body); + env->ReleaseStringUTFChars(jBody, body); } extern "C" JNIEXPORT void JNICALL diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RouterCallback.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RouterCallback.java index 5a31ee2..68dec44 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RouterCallback.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RouterCallback.java @@ -7,7 +7,12 @@ package com.acgist.taoyao.media; */ public interface RouterCallback { - default void enterCallback() {}; + default void enterCallback(String rtpCapabilities, String sctpCapabilities) {}; + default void sendTransportConnectCallback(String transportId, String dtlsParameters) {}; + default String sendTransportProduceCallback(String kind, String transportId, String rtpParameters) { + return null; + }; + default void recvTransportConnectCallback(String transportId, String dtlsParameters) {}; default void newRemoteClientCallback() {}; default void closeRemoteClientCallback() {}; default void consumerPauseCallback() {}; diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Room.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Room.java index e7b5b84..da0461c 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Room.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Room.java @@ -66,7 +66,7 @@ public class Room implements Closeable, RouterCallback { this.dataProduce = dataProduce; this.audioProduce = audioProduce; this.videoProduce = videoProduce; - this.nativeRoomPointer = this.nativeNewRoom(roomId); + this.nativeRoomPointer = this.nativeNewRoom(roomId, this); this.mediaManager = MediaManager.getInstance(); this.remoteClients = new CopyOnWriteArrayList<>(); this.enter = false; @@ -95,7 +95,7 @@ public class Room implements Closeable, RouterCallback { this.rtcConfiguration = new PeerConnection.RTCConfiguration(iceServers); this.peerConnectionFactory = this.mediaManager.newClient(MediaManager.Type.BACK); final Object rtpCapabilities = MapUtils.get(response.body(), "rtpCapabilities"); - this.nativeLoad(this.nativeRoomPointer, JSONUtils.toJSON(rtpCapabilities), this.peerConnectionFactory.getNativePeerConnectionFactory(), this.rtcConfiguration); + this.nativeEnter(this.nativeRoomPointer, JSONUtils.toJSON(rtpCapabilities), this.peerConnectionFactory.getNativePeerConnectionFactory(), this.rtcConfiguration); } public void produceMedia() { @@ -121,13 +121,24 @@ public class Room implements Closeable, RouterCallback { Log.w(Room.class.getSimpleName(), "创建发送通道失败"); return; } - final Map body = response.body(); - this.nativeCreateSendTransport(this.nativeRoomPointer, JSONUtils.toJSON(body)); + this.nativeCreateSendTransport(this.nativeRoomPointer, JSONUtils.toJSON(response.body())); } private void createRecvTransport() { + final Message response = this.taoyao.request(this.taoyao.buildMessage( + "media::transport::webrtc::create", + "forceTcp", false, + "producing", false, + "consuming", true, + "sctpCapabilities", this.dataProduce ? this.sctpCapabilities : null + )); + if(response == null) { + Log.w(Room.class.getSimpleName(), "创建接收通道失败"); + return; + } + this.nativeCreateRecvTransport(this.nativeRoomPointer, JSONUtils.toJSON(response.body())); } - + /** * 新增远程终端 * @@ -150,6 +161,7 @@ public class Room implements Closeable, RouterCallback { this.nativeCloseRoom(this.nativeRoomPointer); } + @Override public void enterCallback(String rtpCapabilities, String sctpCapabilities) { this.taoyao.request(this.taoyao.buildMessage( "room::enter", @@ -161,13 +173,46 @@ public class Room implements Closeable, RouterCallback { this.enter = true; } - private native void nativeLoad( + @Override + public void sendTransportConnectCallback(String transportId, String dtlsParameters) { + this.taoyao.request(this.taoyao.buildMessage( + "media::transport::webrtc::connect", + "roomId", this.roomId, + "transportId", transportId, + "dtlsParameters", JSONUtils.toMap(dtlsParameters) + )); + } + + @Override + public String sendTransportProduceCallback(String kind, String transportId, String rtpParameters) { + final Message response = this.taoyao.request(this.taoyao.buildMessage( + "media::produce", + "kind", kind, + "roomId", this.roomId, + "transportId", transportId, + "rtpParameters", JSONUtils.toMap(rtpParameters) + )); + final Map body = response.body(); + return MapUtils.get(body, "producerId"); + } + + @Override + public void recvTransportConnectCallback(String transportId, String dtlsParameters) { + this.taoyao.request(this.taoyao.buildMessage( + "media::transport::webrtc::connect", + "roomId", this.roomId, + "transportId", transportId, + "dtlsParameters", JSONUtils.toMap(dtlsParameters) + )); + } + + private native void nativeEnter( long nativePointer, String rtpCapabilities, long peerConnectionFactoryPointer, PeerConnection.RTCConfiguration rtcConfiguration ); - private native long nativeNewRoom(String roomId); + private native long nativeNewRoom(String roomId, RouterCallback routerCallback); private native void nativeCloseRoom(long nativePointer); private native void nativeCreateSendTransport(long nativeRoomPointer, String body); private native void nativeCreateRecvTransport(long nativeRoomPointer, String body); diff --git a/taoyao-client-web/src/components/Taoyao.js b/taoyao-client-web/src/components/Taoyao.js index a9a33b3..1c2f627 100644 --- a/taoyao-client-web/src/components/Taoyao.js +++ b/taoyao-client-web/src/components/Taoyao.js @@ -1644,16 +1644,15 @@ class Taoyao extends RemoteClient { self.sendTransport.on( "connect", ({ dtlsParameters }, callback, errback) => { - self - .request( - protocol.buildMessage("media::transport::webrtc::connect", { - roomId: self.roomId, - transportId: self.sendTransport.id, - dtlsParameters, - }) - ) - .then(callback) - .catch(errback); + self.request( + protocol.buildMessage("media::transport::webrtc::connect", { + roomId: self.roomId, + transportId: self.sendTransport.id, + dtlsParameters, + }) + ) + .then(callback) + .catch(errback); } ); self.sendTransport.on( @@ -1742,16 +1741,15 @@ class Taoyao extends RemoteClient { callback, errback // eslint-disable-line no-shadow ) => { - self - .request( - protocol.buildMessage("media::transport::webrtc::connect", { - roomId: self.roomId, - transportId: self.recvTransport.id, - dtlsParameters, - }) - ) - .then(callback) - .catch(errback); + self.request( + protocol.buildMessage("media::transport::webrtc::connect", { + roomId: self.roomId, + transportId: self.recvTransport.id, + dtlsParameters, + }) + ) + .then(callback) + .catch(errback); } ); }