From 9c31ca4f64675b3aa1ea936cea820bedc6921f0f Mon Sep 17 00:00:00 2001 From: acgist <289547414@qq.com> Date: Tue, 11 Apr 2023 08:13:16 +0800 Subject: [PATCH] =?UTF-8?q?[*]=20=E5=9B=9E=E8=B0=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../taoyao/boot/build.gradle | 8 +- .../com/acgist/taoyao/boot/model/Message.java | 12 +- .../taoyao/client/build.gradle | 6 +- .../acgist/taoyao/client/MainActivity.java | 3 +- .../acgist/taoyao/client/MediaService.java | 8 +- .../acgist/taoyao/client/signal/Taoyao.java | 70 +++-- .../taoyao/media/CMakeLists.txt | 2 + .../taoyao/media/build.gradle | 8 +- .../taoyao/media/src/main/cpp/include/Log.hpp | 14 +- .../src/main/cpp/include/MediaManager.hpp | 5 - .../media/src/main/cpp/include/Room.hpp | 37 ++- .../src/main/cpp/include/RouterCallback.hpp | 29 ++ .../src/main/cpp/webrtc/MediaManager.cpp | 10 +- .../taoyao/media/src/main/cpp/webrtc/Room.cpp | 259 ++++++++---------- .../src/main/cpp/webrtc/RouterCallback.cpp | 100 +++++++ .../com/acgist/taoyao/media/MediaManager.java | 70 ++--- .../acgist/taoyao/media/RouterCallback.java | 2 +- ...cesser.java => AudioChangerProcesser.java} | 3 +- .../com/acgist/taoyao/media/client/Room.java | 22 +- .../taoyao/media/signal/ITaoyaoListener.java | 59 ++++ .../taoyao/media/video/AiProcesser.java | 23 +- .../taoyao/media/video/BeautyProcesser.java | 22 +- .../media/video/WatermarkProcesser.java | 23 +- 23 files changed, 507 insertions(+), 288 deletions(-) create mode 100644 taoyao-client-android/taoyao/media/src/main/cpp/include/RouterCallback.hpp create mode 100644 taoyao-client-android/taoyao/media/src/main/cpp/webrtc/RouterCallback.cpp rename taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/{VoiceChangerProcesser.java => AudioChangerProcesser.java} (70%) create mode 100644 taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/signal/ITaoyaoListener.java diff --git a/taoyao-client-android/taoyao/boot/build.gradle b/taoyao-client-android/taoyao/boot/build.gradle index 744c28b..52910f1 100644 --- a/taoyao-client-android/taoyao/boot/build.gradle +++ b/taoyao-client-android/taoyao/boot/build.gradle @@ -25,9 +25,9 @@ android { } dependencies { - implementation 'org.apache.commons:commons-lang3:3.12.0' - implementation 'org.apache.commons:commons-collections4:4.4' - implementation 'com.fasterxml.jackson.core:jackson-databind:2.14.2' - implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.14.2' + api 'org.apache.commons:commons-lang3:3.12.0' + api 'org.apache.commons:commons-collections4:4.4' + api 'com.fasterxml.jackson.core:jackson-databind:2.14.2' + api 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.14.2' testImplementation 'junit:junit:4.13.2' } diff --git a/taoyao-client-android/taoyao/boot/src/main/java/com/acgist/taoyao/boot/model/Message.java b/taoyao-client-android/taoyao/boot/src/main/java/com/acgist/taoyao/boot/model/Message.java index ea9f152..4905e1c 100644 --- a/taoyao-client-android/taoyao/boot/src/main/java/com/acgist/taoyao/boot/model/Message.java +++ b/taoyao-client-android/taoyao/boot/src/main/java/com/acgist/taoyao/boot/model/Message.java @@ -156,16 +156,8 @@ public class Message implements Cloneable, Serializable { /** * @return Map消息主体 */ - @SuppressWarnings({"rawtypes", "unchecked"}) - public Map body() { - if (this.body instanceof Map) { - return (Map) this.body; - } else if (this.body == null) { - return new HashMap<>(); - } else { - Log.w(Message.class.getSimpleName(), "信令主体类型错误：" + this.body); - return new HashMap<>(); - } + public T body() { + return (T) this.body; } @Override diff --git a/taoyao-client-android/taoyao/client/build.gradle b/taoyao-client-android/taoyao/client/build.gradle index d68ea6d..97dc894 100644 --- a/taoyao-client-android/taoyao/client/build.gradle +++ b/taoyao-client-android/taoyao/client/build.gradle @@ -32,11 +32,7 @@ dependencies { implementation project(path: ':boot') implementation project(path: ':media') implementation 'androidx.appcompat:appcompat:1.5.1' - implementation 'com.google.android.material:material:1.8.0' implementation 'androidx.constraintlayout:constraintlayout:2.1.4' - implementation 'org.apache.commons:commons-lang3:3.12.0' - implementation 'org.apache.commons:commons-collections4:4.4' - implementation 'com.fasterxml.jackson.core:jackson-databind:2.14.2' - implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.14.2' + implementation 'com.google.android.material:material:1.8.0' testImplementation 'junit:junit:4.13.2' } diff --git a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java index c77942a..4d1f016 100644 --- a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java +++ b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java @@ -147,7 +147,6 @@ public class MainActivity extends AppCompatActivity implements Serializable { final Resources resources = this.getResources(); MediaManager.getInstance().initContext( this.mainHandler, this.getApplicationContext(), - resources.getBoolean(R.bool.preview), resources.getBoolean(R.bool.playAudio), resources.getBoolean(R.bool.playVideo), resources.getBoolean(R.bool.audioConsume), @@ -210,7 +209,7 @@ public class MainActivity extends AppCompatActivity implements Serializable { private void action(View view) { this.threadHandler.post(() -> { // 进入房间 - Taoyao.taoyao.roomEnter("d8f1e91c-58d0-4e58-ad67-decc0fd61df2", null); + Taoyao.taoyao.roomEnter("022a16bd-94b9-4d40-889e-d364da065bd2", null); }); } diff --git a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MediaService.java b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MediaService.java index 8c811c4..a1a231a 100644 --- a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MediaService.java +++ b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MediaService.java @@ -18,8 +18,12 @@ import android.widget.Toast; import androidx.core.app.NotificationCompat; +import com.acgist.taoyao.boot.model.Message; import com.acgist.taoyao.client.signal.Taoyao; import com.acgist.taoyao.media.MediaManager; +import com.acgist.taoyao.media.signal.ITaoyaoListener; + +import java.util.Map; /** * 媒体服务 @@ -53,6 +57,8 @@ public class MediaService extends Service { private Taoyao taoyao; private Handler mainHandler; + private final ITaoyaoListener taoyaoListener = new ITaoyaoListener() { + }; @Override public void onCreate() { @@ -154,7 +160,7 @@ public class MediaService extends Service { port, host, resources.getString(R.string.version), name, clientId, resources.getString(R.string.clientType), username, password, resources.getInteger(R.integer.timeout), resources.getString(R.string.encrypt), resources.getString(R.string.encryptSecret), - this.mainHandler, context + this.mainHandler, context, this.taoyaoListener ); MediaManager.getInstance().initTaoyao(this.taoyao); Toast.makeText(this.getApplicationContext(), "连接信令", Toast.LENGTH_SHORT).show(); diff --git a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java index f905c39..f62184b 100644 --- a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java +++ b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java @@ -30,6 +30,7 @@ import com.acgist.taoyao.media.MediaManager; import com.acgist.taoyao.media.client.Room; import com.acgist.taoyao.media.client.SessionClient; import com.acgist.taoyao.media.signal.ITaoyao; +import com.acgist.taoyao.media.signal.ITaoyaoListener; import org.apache.commons.lang3.ArrayUtils; @@ -130,6 +131,10 @@ public final class Taoyao implements ITaoyao { * 服务上下文 */ private final Context context; + /** + * 信令监听 + */ + private final ITaoyaoListener taoyaoListener; /** * Wifi管理器 */ @@ -175,7 +180,7 @@ public final class Taoyao implements ITaoyao { int port, String host, String version, String name, String clientId, String clientType, String username, String password, int timeout, String algo, String secret, - Handler mainHandler, Context context + Handler mainHandler, Context context, ITaoyaoListener taoyaoListener ) { this.close = false; this.connect = false; @@ -193,6 +198,7 @@ public final class Taoyao implements ITaoyao { this.decrypt = plaintext ? null : this.buildCipher(Cipher.DECRYPT_MODE, algo, secret); this.mainHandler = mainHandler; this.context = context; + this.taoyaoListener = taoyaoListener; this.wifiManager = context.getSystemService(WifiManager.class); this.powerManager = context.getSystemService(PowerManager.class); this.batteryManager = context.getSystemService(BatteryManager.class); @@ -246,6 +252,7 @@ public final class Taoyao implements ITaoyao { // 开始连接 Log.d(Taoyao.class.getSimpleName(), "连接信令：" + this.host + ":" + this.port); this.socket = new Socket(); + this.taoyaoListener.onConnect(); try { // 设置读取超时时间：不要设置一直阻塞 // socket.setSoTimeout(this.timeout); @@ -255,6 +262,7 @@ public final class Taoyao implements ITaoyao { this.output = this.socket.getOutputStream(); this.clientRegister(); this.connect = true; + this.taoyaoListener.onConnected(); synchronized (this) { this.notifyAll(); } @@ -320,6 +328,7 @@ public final class Taoyao implements ITaoyao { Taoyao.this.on(content); } catch (Exception e) { Log.e(Taoyao.class.getSimpleName(), "处理信令异常：" + content, e); + this.taoyaoListener.onError(e); } } } @@ -407,6 +416,7 @@ public final class Taoyao implements ITaoyao { if(!this.connect) { return; } + this.taoyaoListener.onDisconnect(); Log.d(Taoyao.class.getSimpleName(), "释放信令：" + this.host + ":" + this.port); this.connect = false; CloseableUtils.close(this.input); @@ -497,36 +507,39 @@ public final class Taoyao implements ITaoyao { } private void dispatch(final String content, final Header header, final Message message) { - final Map body = message.body(); + if(this.taoyaoListener.preOnMessage(message)) { + return; + } switch (header.getSignal()) { - case "client::config" -> this.clientConfig(message, body); - case "client::register" -> this.clientRegister(message, body); - case "client::reboot" -> this.clientReboot(message, body); - case "client::shutdown" -> this.clientShutdown(message, body); - case "media::consume" -> this.mediaConsume(message, body); -// case "media::audio::volume" -> this.mediaAudioVolume(message, body); - case "media::consumer::close" -> this.mediaConsumerClose(message, body); - case "media::consumer::pause" -> this.mediaConsumerPause(message, body); -// case "media::consumer::request::key::frame" -> this.mediaConsumerRequestKeyFrame(message, body); - case "media::consumer::resume" -> this.mediaConsumerResume(message, body); -// case "media::consumer::set::preferred::layers" -> this.mediaConsumerSetPreferredLayers(message, body); -// case "media::consumer::status" -> this.mediaConsumerStatus(message, body); - case "media::producer::close" -> this.mediaProducerClose(message, body); - case "media::producer::pause" -> this.mediaProducerPause(message, body); - case "media::producer::resume" -> this.mediaProducerResume(message, body); -// case "media::producer::video::orientation:change" -> this.mediaVideoOrientationChange(message, body); - case "room::close" -> this.roomClose(message, body); - case "room::enter" -> this.roomEnter(message, body); - case "room::expel" -> this.roomExpel(message, body); - case "room::invite" -> this.roomInivte(message, body); - case "room::leave" -> this.roomLeave(message, body); - case "session::call" -> this.sessionCall(message, body); - case "session::close" -> this.sessionClose(message, body); - case "session::exchange" -> this.sessionExchange(message, body); - case "session::pause" -> this.sessionPause(message, body); - case "session::resume" -> this.sessionResume(message, body); + case "client::config" -> this.clientConfig(message, message.body()); + case "client::register" -> this.clientRegister(message, message.body()); + case "client::reboot" -> this.clientReboot(message, message.body()); + case "client::shutdown" -> this.clientShutdown(message, message.body()); + case "media::consume" -> this.mediaConsume(message, message.body()); +// case "media::audio::volume" -> this.mediaAudioVolume(message, message.body()); + case "media::consumer::close" -> this.mediaConsumerClose(message, message.body()); + case "media::consumer::pause" -> this.mediaConsumerPause(message, message.body()); +// case "media::consumer::request::key::frame" -> this.mediaConsumerRequestKeyFrame(message, message.body()); + case "media::consumer::resume" -> this.mediaConsumerResume(message, message.body()); +// case "media::consumer::set::preferred::layers" -> this.mediaConsumerSetPreferredLayers(message, message.body()); +// case "media::consumer::status" -> this.mediaConsumerStatus(message, message.body()); + case "media::producer::close" -> this.mediaProducerClose(message, message.body()); + case "media::producer::pause" -> this.mediaProducerPause(message, message.body()); + case "media::producer::resume" -> this.mediaProducerResume(message, message.body()); +// case "media::producer::video::orientation:change" -> this.mediaVideoOrientationChange(message, message.body()); + case "room::close" -> this.roomClose(message, message.body()); + case "room::enter" -> this.roomEnter(message, message.body()); + case "room::expel" -> this.roomExpel(message, message.body()); + case "room::invite" -> this.roomInivte(message, message.body()); + case "room::leave" -> this.roomLeave(message, message.body()); + case "session::call" -> this.sessionCall(message, message.body()); + case "session::close" -> this.sessionClose(message, message.body()); + case "session::exchange" -> this.sessionExchange(message, message.body()); + case "session::pause" -> this.sessionPause(message, message.body()); + case "session::resume" -> this.sessionResume(message, message.body()); default -> Log.d(Taoyao.class.getSimpleName(), "没有适配信令：" + content); } + this.taoyaoListener.postOnMessage(message); } /** @@ -672,6 +685,7 @@ public final class Taoyao implements ITaoyao { this.name, this.clientId, key, password, this, this.mainHandler, + resources.getBoolean(R.bool.preview), resources.getBoolean(R.bool.dataConsume), resources.getBoolean(R.bool.audioConsume), resources.getBoolean(R.bool.videoConsume), diff --git a/taoyao-client-android/taoyao/media/CMakeLists.txt b/taoyao-client-android/taoyao/media/CMakeLists.txt index 8021f67..77b8dea 100644 --- a/taoyao-client-android/taoyao/media/CMakeLists.txt +++ b/taoyao-client-android/taoyao/media/CMakeLists.txt @@ -28,6 +28,7 @@ set( ${SOURCE_DIR}/include/Log.hpp ${SOURCE_DIR}/include/MediaManager.hpp ${SOURCE_DIR}/include/Room.hpp + ${SOURCE_DIR}/include/RouterCallback.hpp ${SOURCE_DIR}/include/RtpAudioPublisher.hpp ${SOURCE_DIR}/include/RtpClient.hpp ${SOURCE_DIR}/include/RtpVideoPublisher.hpp @@ -36,6 +37,7 @@ set( ${SOURCE_DIR}/rtp/RtpVideoPublisher.cpp ${SOURCE_DIR}/webrtc/MediaManager.cpp ${SOURCE_DIR}/webrtc/Room.cpp + ${SOURCE_DIR}/webrtc/RouterCallback.cpp ) set(LIBWEBRTC_BINARY_PATH ${LIBWEBRTC_BINARY_PATH}/${ANDROID_ABI} CACHE STRING "libwebrtc binary path" FORCE) diff --git a/taoyao-client-android/taoyao/media/build.gradle b/taoyao-client-android/taoyao/media/build.gradle index 5bb82e2..9ac8926 100644 --- a/taoyao-client-android/taoyao/media/build.gradle +++ b/taoyao-client-android/taoyao/media/build.gradle @@ -53,11 +53,7 @@ android { } dependencies { - implementation project(path: ':boot') - implementation fileTree(dir: 'libs', include: ['*.jar']) - implementation 'org.apache.commons:commons-lang3:3.12.0' - implementation 'org.apache.commons:commons-collections4:4.4' - implementation 'com.fasterxml.jackson.core:jackson-databind:2.14.2' - implementation 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.14.2' + api project(path: ':boot') + api fileTree(dir: 'libs', include: ['*.jar']) testImplementation 'junit:junit:4.13.2' } diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/include/Log.hpp b/taoyao-client-android/taoyao/media/src/main/cpp/include/Log.hpp index d9ce581..217272c 100644 --- a/taoyao-client-android/taoyao/media/src/main/cpp/include/Log.hpp +++ b/taoyao-client-android/taoyao/media/src/main/cpp/include/Log.hpp @@ -6,8 +6,14 @@ namespace acgist { #define LOG_TAG "libtaoyao" -#define LOG_D(message, ...) acgist::__android_log_print(ANDROID_LOG_DEBUG, LOG_TAG, message, __VA_ARGS__) -#define LOG_I(message, ...) acgist::__android_log_print(ANDROID_LOG_INFO, LOG_TAG, message, __VA_ARGS__) -#define LOG_W(message, ...) acgist::__android_log_print(ANDROID_LOG_WARN, LOG_TAG, message, __VA_ARGS__) -#define LOG_E(message, ...) acgist::__android_log_print(ANDROID_LOG_ERROR, LOG_TAG, message, __VA_ARGS__) +#define LOG_D(format, ...) __android_log_print(ANDROID_LOG_DEBUG, LOG_TAG, "%s " format, __func__, ##__VA_ARGS__) +#define LOG_I(format, ...) __android_log_print(ANDROID_LOG_INFO, LOG_TAG, "%s " format, __func__, ##__VA_ARGS__) +#define LOG_W(format, ...) __android_log_print(ANDROID_LOG_WARN, LOG_TAG, "%s " format, __func__, ##__VA_ARGS__) +#define LOG_E(format, ...) __android_log_print(ANDROID_LOG_ERROR, LOG_TAG, "%s " format, __func__, ##__VA_ARGS__) + +//#define LOG_D(...) acgist::__android_log_print(ANDROID_LOG_DEBUG, LOG_TAG, __VA_ARGS__) +//#define LOG_I(...) acgist::__android_log_print(ANDROID_LOG_INFO, LOG_TAG, __VA_ARGS__) +//#define LOG_W(...) acgist::__android_log_print(ANDROID_LOG_WARN, LOG_TAG, __VA_ARGS__) +//#define LOG_E(...) acgist::__android_log_print(ANDROID_LOG_ERROR, LOG_TAG, __VA_ARGS__) + } diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/include/MediaManager.hpp b/taoyao-client-android/taoyao/media/src/main/cpp/include/MediaManager.hpp index 1178352..0cc210c 100644 --- a/taoyao-client-android/taoyao/media/src/main/cpp/include/MediaManager.hpp +++ b/taoyao-client-android/taoyao/media/src/main/cpp/include/MediaManager.hpp @@ -1,14 +1,9 @@ #pragma once -#include -#include - #include "jni.h" #include "Log.hpp" #include "mediasoupclient.hpp" namespace acgist { - jmethodID closeCallback; - } \ No newline at end of file diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/include/Room.hpp b/taoyao-client-android/taoyao/media/src/main/cpp/include/Room.hpp index 1d2d2ec..7dfe265 100644 --- a/taoyao-client-android/taoyao/media/src/main/cpp/include/Room.hpp +++ b/taoyao-client-android/taoyao/media/src/main/cpp/include/Room.hpp @@ -1,44 +1,39 @@ #pragma once #include -#include -#include #include "jni.h" #include "sdk/android/src/jni/pc/peer_connection.h" #include "sdk/android/native_api/jni/scoped_java_ref.h" #include "Log.hpp" -#include "mediasoupclient.hpp" +#include "RouterCallback.hpp" namespace acgist { - class Room { + class Room : public RouterCallback { public: std::string roomId; - mediasoupclient::Device *device; - mediasoupclient::PeerConnection *peerConnection; - mediasoupclient::SendTransport *sendTransport; - mediasoupclient::RecvTransport *recvTransport; - mediasoupclient::SendTransport::Listener *sendListener; - mediasoupclient::RecvTransport::Listener *recvListener; - mediasoupclient::Producer *audioProducer; - mediasoupclient::Producer *videoProducer; - mediasoupclient::Producer::Listener *producerListener; - mediasoupclient::Consumer::Listener *consumerListener; + mediasoupclient::Device* device; + mediasoupclient::PeerConnection* peerConnection; + mediasoupclient::SendTransport* sendTransport; + mediasoupclient::RecvTransport* recvTransport; + mediasoupclient::SendTransport::Listener* sendListener; + mediasoupclient::RecvTransport::Listener* recvListener; + mediasoupclient::Producer* audioProducer; + mediasoupclient::Producer* videoProducer; + mediasoupclient::Producer::Listener* producerListener; + mediasoupclient::Consumer::Listener* consumerListener; std::map consumers; public: - JNIEnv *env; - jobject routerCallback; - public: - Room(std::string roomId, JNIEnv *env, jobject routerCallback); + Room(std::string roomId, JNIEnv* env, jobject routerCallback); virtual ~Room(); public: - void enter(std::string rtpCapabilities, webrtc::PeerConnectionFactoryInterface *factory, webrtc::PeerConnectionInterface::RTCConfiguration &rtcConfiguration); + void enter(std::string rtpCapabilities, webrtc::PeerConnectionFactoryInterface* factory, webrtc::PeerConnectionInterface::RTCConfiguration& rtcConfiguration); void createSendTransport(std::string body); void createRecvTransport(std::string body); - void mediaProduceAudio(webrtc::MediaStreamInterface *mediaStream); - void mediaProduceVideo(webrtc::MediaStreamInterface *mediaStream); + void mediaProduceAudio(webrtc::MediaStreamInterface* mediaStream); + void mediaProduceVideo(webrtc::MediaStreamInterface* mediaStream); void mediaConsume(std::string message); void mediaProducerPause(std::string producerId); void mediaProducerResume(std::string producerId); diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/include/RouterCallback.hpp b/taoyao-client-android/taoyao/media/src/main/cpp/include/RouterCallback.hpp new file mode 100644 index 0000000..0a40f83 --- /dev/null +++ b/taoyao-client-android/taoyao/media/src/main/cpp/include/RouterCallback.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include + +#include "jni.h" +#include "mediasoupclient.hpp" + +namespace acgist { + + class RouterCallback { + public: + JNIEnv* env; + jobject routerCallback; + public: + void enterCallback(std::string rtpCapabilities, std::string sctpCapabilities); + void sendTransportConnectCallback(std::string transportId, std::string dtlsParameters); + std::string sendTransportProduceCallback(std::string kind, std::string transportId, std::string rtpParameters); + void recvTransportConnectCallback(std::string transportId, std::string dtlsParameters); + void producerNewCallback(std::string kind, std::string producerId, webrtc::MediaStreamTrackInterface* producerMediaTrackPointer); + void producerPauseCallback(std::string producerId); + void producerResumeCallback(std::string producerId); + void producerCloseCallback(std::string producerId); + void consumerNewCallback(std::string message, webrtc::MediaStreamTrackInterface* consumerMediaTrackPointer); + void consumerPauseCallback(std::string consumerId); + void consumerResumeCallback(std::string consumerId); + void consumerCloseCallback(std::string consumerId); + }; + +} \ No newline at end of file diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/MediaManager.cpp b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/MediaManager.cpp index 31a2d73..cfcf221 100644 --- a/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/MediaManager.cpp +++ b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/MediaManager.cpp @@ -1,11 +1,11 @@ -#include "Room.hpp" #include "MediaManager.hpp" namespace acgist { extern "C" JNIEXPORT void JNICALL - Java_com_acgist_taoyao_media_MediaManager_nativeInit(JNIEnv *env, jobject me) { - LOG_I("加载MediasoupClient：", mediasoupclient::Version().data()); + Java_com_acgist_taoyao_media_MediaManager_nativeInit(JNIEnv* env, jobject me) { + std::string version = mediasoupclient::Version(); + LOG_I("加载MediasoupClient", version.data()); mediasoupclient::Initialize(); // => { spatialLayers: 2, temporalLayers: 3 } // mediasoupclient::parseScalabilityMode("L2T3"); @@ -14,8 +14,8 @@ namespace acgist { } extern "C" JNIEXPORT void JNICALL - Java_com_acgist_taoyao_media_MediaManager_nativeStop(JNIEnv *env, jobject me) { - std::cout << "释放mediasoupclient" << std::endl; + Java_com_acgist_taoyao_media_MediaManager_nativeStop(JNIEnv* env, jobject me) { + LOG_I("释放mediasoupclient"); mediasoupclient::Cleanup(); } diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/Room.cpp b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/Room.cpp index 0b7cdb7..ccb0ecf 100644 --- a/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/Room.cpp +++ b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/Room.cpp @@ -5,77 +5,38 @@ namespace acgist { class SendListener : public mediasoupclient::SendTransport::Listener { public: - Room *room; - JNIEnv *env; - jobject routerCallback; + Room* room; public: - SendListener(Room *room, JNIEnv *env, jobject routerCallback) { + SendListener(Room* room) { this->room = room; - this->env = env; - this->routerCallback = routerCallback; + } + + virtual ~SendListener() { } public: - std::future OnConnect(mediasoupclient::Transport *transport, const nlohmann::json &dtlsParameters) override { - jclass jCallbackClazz = this->env->GetObjectClass(this->routerCallback); - jmethodID sendTransportConnectCallback = this->env->GetMethodID(jCallbackClazz, "sendTransportConnectCallback", "(Ljava/lang/String;Ljava/lang/String;)V"); - const char *cTransportId = transport->GetId().data(); - jstring jTransportId = this->env->NewStringUTF(cTransportId); - const char *cDtlsParameters = dtlsParameters.dump().data(); - jstring jDtlsParameters = this->env->NewStringUTF(cDtlsParameters); - this->env->CallVoidMethod( - this->routerCallback, - sendTransportConnectCallback, - jTransportId, - jDtlsParameters - ); - this->env->DeleteLocalRef(jTransportId); - this->env->ReleaseStringUTFChars(jTransportId, cTransportId); - this->env->DeleteLocalRef(jDtlsParameters); - this->env->ReleaseStringUTFChars(jDtlsParameters, cDtlsParameters); - this->env->DeleteLocalRef(jCallbackClazz); + std::future OnConnect(mediasoupclient::Transport* transport, const nlohmann::json& dtlsParameters) override { + const std::string cTransportId = transport->GetId(); + const std::string cDtlsParameters = dtlsParameters.dump(); + this->room->sendTransportConnectCallback(cTransportId, cDtlsParameters); return std::future(); } - void OnConnectionStateChange(mediasoupclient::Transport *transport, const std::string &connectionState) override { + void OnConnectionStateChange(mediasoupclient::Transport* transport, const std::string& connectionState) override { // 状态变化 } - std::future OnProduce(mediasoupclient::SendTransport *transport, const std::string &kind, nlohmann::json rtpParameters, const nlohmann::json &appData) override { - jclass jCallbackClazz = this->env->GetObjectClass(this->routerCallback); - jmethodID sendTransportProduceCallback = this->env->GetMethodID(jCallbackClazz, "sendTransportProduceCallback", "(Ljava/lang/String;Ljava/lang/String;)V"); - const char *cKind = kind.data(); - jstring jKind = this-> env->NewStringUTF(cKind); - const char *cTransportId = transport->GetId().data(); - jstring jTransportId = this-> env->NewStringUTF(cTransportId); - const char *cRtpParameters = rtpParameters.dump().data(); - jstring jRtpParameters = this-> env->NewStringUTF(cRtpParameters); - std::promise promise; - jstring jResult = (jstring) this->env->CallObjectMethod( - this->routerCallback, - sendTransportProduceCallback, - jKind, - jTransportId, - jRtpParameters - ); - const char *cResult = this-> env->GetStringUTFChars(jResult, 0); - std::string result(cResult); + std::future OnProduce(mediasoupclient::SendTransport* transport, const std::string& kind, nlohmann::json rtpParameters, const nlohmann::json& appData) override { + const std::string cTransportId = transport->GetId(); + const std::string cRtpParameters = rtpParameters.dump(); + std::string result = this->room->sendTransportProduceCallback(kind, cTransportId, cRtpParameters); + std::promise promise; promise.set_value(result); - this-> env->DeleteLocalRef(jResult); - this-> env->DeleteLocalRef(jKind); - this-> env->ReleaseStringUTFChars(jKind, cKind); - this-> env->DeleteLocalRef(jResult); - this-> env->ReleaseStringUTFChars(jResult, cResult); - this-> env->DeleteLocalRef(jTransportId); - this-> env->ReleaseStringUTFChars(jTransportId, cTransportId); - this-> env->DeleteLocalRef(jRtpParameters); - this-> env->ReleaseStringUTFChars(jRtpParameters, cRtpParameters); - this-> env->DeleteLocalRef(jCallbackClazz); return promise.get_future(); } - std::future OnProduceData(mediasoupclient::SendTransport *transport, const nlohmann::json &sctpStreamParameters, const std::string &label, const std::string &protocol, const nlohmann::json &appData) override { + std::future OnProduceData(mediasoupclient::SendTransport* transport, const nlohmann::json& sctpStreamParameters, const std::string& label, const std::string& protocol, const nlohmann::json& appData) override { // 数据生产 return std::future(); } @@ -85,39 +46,24 @@ namespace acgist { class RecvListener : public mediasoupclient::RecvTransport::Listener { public: - Room *room; - JNIEnv *env; - jobject routerCallback; + Room* room; public: - RecvListener(Room *room, JNIEnv *env, jobject routerCallback) { + RecvListener(Room* room) { this->room = room; - this->env = env; - this->routerCallback = routerCallback; } - std::future OnConnect(mediasoupclient::Transport *transport, const nlohmann::json &dtlsParameters) override { - jclass jCallbackClazz = this->env->GetObjectClass(this->routerCallback); - jmethodID recvTransportConnectCallback = this->env->GetMethodID(jCallbackClazz, "recvTransportConnectCallback", "(Ljava/lang/String;Ljava/lang/String;)V"); - const char *cTransportId = transport->GetId().data(); - jstring jTransportId = this-> env->NewStringUTF(cTransportId); - const char *cDtlsParameters = dtlsParameters.dump().data(); - jstring jDtlsParameters = this-> env->NewStringUTF(cDtlsParameters); - this->env->CallVoidMethod( - this->routerCallback, - recvTransportConnectCallback, - jTransportId, - jDtlsParameters - ); - this-> env->DeleteLocalRef(jTransportId); - this-> env->ReleaseStringUTFChars(jTransportId, cTransportId); - this-> env->DeleteLocalRef(jDtlsParameters); - this-> env->ReleaseStringUTFChars(jDtlsParameters, cDtlsParameters); - this-> env->DeleteLocalRef(jCallbackClazz); + virtual ~RecvListener() { + } + + std::future OnConnect(mediasoupclient::Transport* transport, const nlohmann::json& dtlsParameters) override { + const std::string cTransportId = transport->GetId(); + const std::string cDtlsParameters = dtlsParameters.dump(); + this->room->recvTransportConnectCallback(cTransportId, cDtlsParameters); return std::future(); } - void OnConnectionStateChange(mediasoupclient::Transport *transport, const std::string &connectionState) override { + void OnConnectionStateChange(mediasoupclient::Transport* transport, const std::string& connectionState) override { // 状态变化 } @@ -126,18 +72,17 @@ namespace acgist { class ProducerListener : public mediasoupclient::Producer::Listener { public: - Room *room; - JNIEnv *env; - jobject routerCallback; + Room* room; public: - ProducerListener(Room *room, JNIEnv *env, jobject routerCallback) { + ProducerListener(Room* room) { this->room = room; - this->env = env; - this->routerCallback = routerCallback; } - void OnTransportClose(mediasoupclient::Producer *producer) override { + virtual ~ProducerListener() { + } + + void OnTransportClose(mediasoupclient::Producer* producer) override { } @@ -146,18 +91,17 @@ namespace acgist { class ConsumerListener : public mediasoupclient::Consumer::Listener { public: - Room *room; - JNIEnv *env; - jobject routerCallback; + Room* room; public: - ConsumerListener(Room *room, JNIEnv *env, jobject routerCallback) { + ConsumerListener(Room* room) { this->room = room; - this->env = env; - this->routerCallback = routerCallback; } - void OnTransportClose(mediasoupclient::Consumer *consumer) override { + virtual ~ConsumerListener() { + } + + void OnTransportClose(mediasoupclient::Consumer* consumer) override { } @@ -165,17 +109,17 @@ namespace acgist { Room::Room( std::string roomId, - JNIEnv *env, + JNIEnv* env, jobject routerCallback ) { this->roomId = roomId; this->env = env; this->routerCallback = routerCallback; this->device = new mediasoupclient::Device(); - this->sendListener = new SendListener(this, env, routerCallback); - this->recvListener = new RecvListener(this, env, routerCallback); - this->producerListener = new ProducerListener(this, env, routerCallback); - this->consumerListener = new ConsumerListener(this, env, routerCallback); + this->sendListener = new SendListener(this); + this->recvListener = new RecvListener(this); + this->producerListener = new ProducerListener(this); + this->consumerListener = new ConsumerListener(this); } Room::~Room() { @@ -188,22 +132,25 @@ namespace acgist { delete this->videoProducer; delete this->producerListener; delete this->consumerListener; - this-> env->DeleteLocalRef(this->routerCallback); - this-> env->DeleteGlobalRef(this->routerCallback); + this->env->DeleteLocalRef(this->routerCallback); + this->env->DeleteGlobalRef(this->routerCallback); } void Room::enter( std::string rtpCapabilities, - webrtc::PeerConnectionFactoryInterface *factory, - webrtc::PeerConnectionInterface::RTCConfiguration &rtcConfiguration + webrtc::PeerConnectionFactoryInterface* factory, + webrtc::PeerConnectionInterface::RTCConfiguration& rtcConfiguration ) { nlohmann::json json; // TODO：全局 mediasoupclient::PeerConnection::Options options; - options.config = rtcConfiguration; + options.config = rtcConfiguration; options.factory = factory; json["routerRtpCapabilities"] = nlohmann::json::parse(rtpCapabilities); this->device->Load(json, &options); + const std::string cRtpCapabilities = this->device->GetRtpCapabilities().dump(); + const std::string cSctpCapabilities = this->device->GetSctpCapabilities().dump(); + this->enterCallback(cRtpCapabilities, cSctpCapabilities); } void Room::createSendTransport(std::string body) { @@ -232,7 +179,7 @@ namespace acgist { ); } - void Room::mediaProduceAudio(webrtc::MediaStreamInterface *mediaStream) { + void Room::mediaProduceAudio(webrtc::MediaStreamInterface* mediaStream) { if(!this->device->CanProduce("audio")) { return; } @@ -250,7 +197,7 @@ namespace acgist { ); } - void Room::mediaProduceVideo(webrtc::MediaStreamInterface *mediaStream) { + void Room::mediaProduceVideo(webrtc::MediaStreamInterface* mediaStream) { if(this->device->CanProduce("video")) { return; } @@ -288,7 +235,7 @@ namespace acgist { void Room::mediaConsume(std::string message) { nlohmann::json json = nlohmann::json::parse(message); nlohmann::json body = json["body"]; - mediasoupclient::Consumer *consumer = this->recvTransport->Consume( + mediasoupclient::Consumer* consumer = this->recvTransport->Consume( this->consumerListener, body["consumerId"], body["producerId"], @@ -297,23 +244,28 @@ namespace acgist { ); this->consumers.insert({ consumer->GetId(), consumer }); webrtc::MediaStreamTrackInterface* trackPointer = consumer->GetTrack(); - jclass jCallbackClazz = this->env->GetObjectClass(this->routerCallback); - jmethodID consumerNewCallback = this->env->GetMethodID(jCallbackClazz, "consumerNewCallback", "(Ljava/lang/String;J;)V"); - const char *cMessage = message.data(); - jstring jMessage = this-> env->NewStringUTF(cMessage); - this->env->CallVoidMethod( - this->routerCallback, - consumerNewCallback, - jMessage, - (jlong) trackPointer - ); - this-> env->DeleteLocalRef(jMessage); - this-> env->ReleaseStringUTFChars(jMessage, cMessage); - this-> env->DeleteLocalRef(jCallbackClazz); + this->consumerNewCallback(message, trackPointer); }; + void Room::mediaProducerPause(std::string producerId) { + } + + void Room::mediaProducerResume(std::string producerId) { + } + + void Room::mediaProducerClose(std::string producerId) { + } + + void Room::mediaConsumerPause(std::string consumerId) { + } + + void Room::mediaConsumerResume(std::string consumerId) { + } + + void Room::mediaConsumerClose(std::string consumerId) { + } + void Room::close() { - delete this->device; } extern "C" JNIEXPORT void JNICALL @@ -327,123 +279,130 @@ namespace acgist { // TODO：为什么不能转换？测试是否因为stun配置问题 webrtc::JavaParamRef jRtcConfigurationRef(jRtcConfiguration); // webrtc::jni::JavaToNativeMediaConstraints() - webrtc::jni::JavaToNativeRTCConfiguration(env, jRtcConfigurationRef, &rtcConfiguration); - const char* rtpCapabilities = env->GetStringUTFChars(jRtpCapabilities, 0); +// webrtc::jni::JavaToNativeRTCConfiguration(env, jRtcConfigurationRef, &rtcConfiguration); + const char* rtpCapabilities = env->GetStringUTFChars(jRtpCapabilities, nullptr); room->enter( rtpCapabilities, reinterpret_cast(factoryPointer), // (webrtc::PeerConnectionFactoryInterface*) factoryPointer, rtcConfiguration ); - env->ReleaseStringUTFChars(jRtpCapabilities, rtpCapabilities); env->DeleteLocalRef(jRtpCapabilities); + env->ReleaseStringUTFChars(jRtpCapabilities, rtpCapabilities); env->DeleteLocalRef(jRtcConfiguration); -// delete rtpCapabilities; } extern "C" JNIEXPORT jlong JNICALL Java_com_acgist_taoyao_media_client_Room_nativeNewRoom( - JNIEnv *env, jobject me, + JNIEnv* env, jobject me, jstring jRoomId, jobject jRouterCallback ) { - const char* roomId = env->GetStringUTFChars(jRoomId, 0); jobject routerCallback = env->NewGlobalRef(jRouterCallback); + const char* roomId = env->GetStringUTFChars(jRoomId, nullptr); Room* room = new Room(roomId, env, routerCallback); + env->DeleteLocalRef(jRoomId); env->ReleaseStringUTFChars(jRoomId, roomId); return (jlong) room; } extern "C" JNIEXPORT void JNICALL - Java_com_acgist_taoyao_media_client_Room_nativeCloseRoom(JNIEnv *env, jobject me, jlong nativeRoomPointer) { + Java_com_acgist_taoyao_media_client_Room_nativeCloseRoom(JNIEnv* env, jobject me, jlong nativeRoomPointer) { Room* room = (Room*) nativeRoomPointer; room->close(); delete room; } extern "C" JNIEXPORT void JNICALL - Java_com_acgist_taoyao_media_client_Room_nativeCreateSendTransport(JNIEnv *env, jobject me, jlong nativeRoomPointer, jstring jBody) { + Java_com_acgist_taoyao_media_client_Room_nativeCreateSendTransport(JNIEnv* env, jobject me, jlong nativeRoomPointer, jstring jBody) { Room* room = (Room*) nativeRoomPointer; - const char* body = env->GetStringUTFChars(jBody, 0); + const char* body = env->GetStringUTFChars(jBody, nullptr); room->createSendTransport(body); + env->DeleteLocalRef(jBody); env->ReleaseStringUTFChars(jBody, body); } extern "C" JNIEXPORT void JNICALL - Java_com_acgist_taoyao_media_client_Room_nativeCreateRecvTransport(JNIEnv *env, jobject me, jlong nativeRoomPointer, jstring jBody) { + Java_com_acgist_taoyao_media_client_Room_nativeCreateRecvTransport(JNIEnv* env, jobject me, jlong nativeRoomPointer, jstring jBody) { Room* room = (Room*) nativeRoomPointer; - const char* body = env->GetStringUTFChars(jBody, 0); + const char* body = env->GetStringUTFChars(jBody, nullptr); room->createRecvTransport(body); env->DeleteLocalRef(jBody); env->ReleaseStringUTFChars(jBody, body); } extern "C" JNIEXPORT void JNICALL - Java_com_acgist_taoyao_media_client_Room_nativeMediaProduceAudio(JNIEnv *env, jobject me, jlong nativeRoomPointer, jlong mediaStreamPointer) { + Java_com_acgist_taoyao_media_client_Room_nativeMediaProduceAudio(JNIEnv* env, jobject me, jlong nativeRoomPointer, jlong mediaStreamPointer) { Room* room = (Room*) nativeRoomPointer; - webrtc::MediaStreamInterface *mediaStream = reinterpret_cast(mediaStreamPointer); + webrtc::MediaStreamInterface* mediaStream = reinterpret_cast(mediaStreamPointer); room->mediaProduceAudio(mediaStream); } extern "C" JNIEXPORT void JNICALL - Java_com_acgist_taoyao_media_client_Room_nativeMediaProduceVideo(JNIEnv *env, jobject me, jlong nativeRoomPointer, jlong mediaStreamPointer) { + Java_com_acgist_taoyao_media_client_Room_nativeMediaProduceVideo(JNIEnv* env, jobject me, jlong nativeRoomPointer, jlong mediaStreamPointer) { Room* room = (Room*) nativeRoomPointer; - webrtc::MediaStreamInterface *mediaStream = reinterpret_cast(mediaStreamPointer); + webrtc::MediaStreamInterface* mediaStream = reinterpret_cast(mediaStreamPointer); room->mediaProduceVideo(mediaStream); } extern "C" JNIEXPORT void JNICALL - Java_com_acgist_taoyao_media_client_Room_nativeMediaConsume(JNIEnv *env, jobject me, jlong nativeRoomPointer, jstring jMessage) { + Java_com_acgist_taoyao_media_client_Room_nativeMediaConsume(JNIEnv* env, jobject me, jlong nativeRoomPointer, jstring jMessage) { Room* room = (Room*) nativeRoomPointer; - const char *message = env->GetStringUTFChars(jMessage, 0); + const char* message = env->GetStringUTFChars(jMessage, nullptr); room->mediaConsume(message); env->DeleteLocalRef(jMessage); env->ReleaseStringUTFChars(jMessage, message); } extern "C" JNIEXPORT void JNICALL - Java_com_acgist_taoyao_media_client_Room_nativeMediaProducerPause(JNIEnv *env, jobject me, jlong nativeRoomPointer, jstring jProducerId) { + Java_com_acgist_taoyao_media_client_Room_nativeMediaProducerPause(JNIEnv* env, jobject me, jlong nativeRoomPointer, jstring jProducerId) { Room* room = (Room*) nativeRoomPointer; - const char *producerId = env->GetStringUTFChars(jProducerId, 0); + const char* producerId = env->GetStringUTFChars(jProducerId, nullptr); + room->mediaProducerPause(producerId); env->DeleteLocalRef(jProducerId); env->ReleaseStringUTFChars(jProducerId, producerId); } extern "C" JNIEXPORT void JNICALL - Java_com_acgist_taoyao_media_client_Room_nativeMediaProducerResume(JNIEnv *env, jobject me, jlong nativeRoomPointer, jstring jProducerId) { + Java_com_acgist_taoyao_media_client_Room_nativeMediaProducerResume(JNIEnv* env, jobject me, jlong nativeRoomPointer, jstring jProducerId) { Room* room = (Room*) nativeRoomPointer; - const char *producerId = env->GetStringUTFChars(jProducerId, 0); + const char* producerId = env->GetStringUTFChars(jProducerId, nullptr); + room->mediaProducerResume(producerId); env->DeleteLocalRef(jProducerId); env->ReleaseStringUTFChars(jProducerId, producerId); } extern "C" JNIEXPORT void JNICALL - Java_com_acgist_taoyao_media_client_Room_nativeMediaProducerClose(JNIEnv *env, jobject me, jlong nativeRoomPointer, jstring jProducerId) { + Java_com_acgist_taoyao_media_client_Room_nativeMediaProducerClose(JNIEnv* env, jobject me, jlong nativeRoomPointer, jstring jProducerId) { Room* room = (Room*) nativeRoomPointer; - const char *producerId = env->GetStringUTFChars(jProducerId, 0); + const char* producerId = env->GetStringUTFChars(jProducerId, nullptr); + room->mediaProducerClose(producerId); env->DeleteLocalRef(jProducerId); env->ReleaseStringUTFChars(jProducerId, producerId); } extern "C" JNIEXPORT void JNICALL - Java_com_acgist_taoyao_media_client_Room_nativeMediaConsumerPause(JNIEnv *env, jobject me, jlong nativeRoomPointer, jstring jConsumerId) { + Java_com_acgist_taoyao_media_client_Room_nativeMediaConsumerPause(JNIEnv* env, jobject me, jlong nativeRoomPointer, jstring jConsumerId) { Room* room = (Room*) nativeRoomPointer; - const char *consumerId = env->GetStringUTFChars(jConsumerId, 0); + const char* consumerId = env->GetStringUTFChars(jConsumerId, nullptr); + room->mediaConsumerPause(consumerId); env->DeleteLocalRef(jConsumerId); env->ReleaseStringUTFChars(jConsumerId, consumerId); } extern "C" JNIEXPORT void JNICALL - Java_com_acgist_taoyao_media_client_Room_nativeMediaConsumerResume(JNIEnv *env, jobject me, jlong nativeRoomPointer, jstring jConsumerId) { + Java_com_acgist_taoyao_media_client_Room_nativeMediaConsumerResume(JNIEnv* env, jobject me, jlong nativeRoomPointer, jstring jConsumerId) { Room* room = (Room*) nativeRoomPointer; - const char *consumerId = env->GetStringUTFChars(jConsumerId, 0); + const char* consumerId = env->GetStringUTFChars(jConsumerId, nullptr); + room->mediaConsumerResume(consumerId); env->DeleteLocalRef(jConsumerId); env->ReleaseStringUTFChars(jConsumerId, consumerId); } extern "C" JNIEXPORT void JNICALL - Java_com_acgist_taoyao_media_client_Room_nativeMediaConsumerClose(JNIEnv *env, jobject me, jlong nativeRoomPointer, jstring jConsumerId) { + Java_com_acgist_taoyao_media_client_Room_nativeMediaConsumerClose(JNIEnv* env, jobject me, jlong nativeRoomPointer, jstring jConsumerId) { Room* room = (Room*) nativeRoomPointer; - const char *consumerId = env->GetStringUTFChars(jConsumerId, 0); + const char* consumerId = env->GetStringUTFChars(jConsumerId, nullptr); + room->mediaConsumerClose(consumerId); env->DeleteLocalRef(jConsumerId); env->ReleaseStringUTFChars(jConsumerId, consumerId); } diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/RouterCallback.cpp b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/RouterCallback.cpp new file mode 100644 index 0000000..8596ac8 --- /dev/null +++ b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/RouterCallback.cpp @@ -0,0 +1,100 @@ +#include "RouterCallback.hpp" + +namespace acgist { + + void RouterCallback::enterCallback(std::string rtpCapabilities, std::string sctpCapabilities) { + jclass jCallbackClazz = this->env->GetObjectClass(this->routerCallback); + jmethodID recvTransportConnectCallback = this->env->GetMethodID(jCallbackClazz, "enterCallback", "(Ljava/lang/String;Ljava/lang/String;)V"); + const char* cRtpCapabilities = rtpCapabilities.data(); + const char* cSctpCapabilities = sctpCapabilities.data(); + jstring jRtpCapabilities = this->env->NewStringUTF(cRtpCapabilities); + jstring jScrpCapabilities = this->env->NewStringUTF(cSctpCapabilities); + this->env->CallVoidMethod( + this->routerCallback, + recvTransportConnectCallback, + jRtpCapabilities, + jScrpCapabilities + ); + this->env->DeleteLocalRef(jRtpCapabilities); + this->env->DeleteLocalRef(jScrpCapabilities); + this->env->DeleteLocalRef(jCallbackClazz); + } + + void RouterCallback::sendTransportConnectCallback(std::string transportId, std::string dtlsParameters) { + jclass jCallbackClazz = this->env->GetObjectClass(this->routerCallback); + jmethodID sendTransportConnectCallback = this->env->GetMethodID(jCallbackClazz, "sendTransportConnectCallback", "(Ljava/lang/String;Ljava/lang/String;)V"); + const char* cTransportId = transportId.data(); + const char* cDtlsParameters = dtlsParameters.data(); + jstring jTransportId = this->env->NewStringUTF(cTransportId); + jstring jDtlsParameters = this->env->NewStringUTF(cDtlsParameters); + this->env->CallVoidMethod( + this->routerCallback, + sendTransportConnectCallback, + jTransportId, + jDtlsParameters + ); + this->env->DeleteLocalRef(jTransportId); + this->env->DeleteLocalRef(jDtlsParameters); + this->env->DeleteLocalRef(jCallbackClazz); + } + + std::string RouterCallback::sendTransportProduceCallback(std::string kind, std::string transportId, std::string rtpParameters) { + jclass jCallbackClazz = this->env->GetObjectClass(this->routerCallback); + jmethodID sendTransportProduceCallback = this->env->GetMethodID(jCallbackClazz, "sendTransportProduceCallback", "(Ljava/lang/String;Ljava/lang/String;)V"); + const char* cKind = kind.data(); + const char* cTransportId = transportId.data(); + const char* cRtpParameters = rtpParameters.data(); + jstring jKind = this->env->NewStringUTF(cKind); + jstring jTransportId = this->env->NewStringUTF(cTransportId); + jstring jRtpParameters = this->env->NewStringUTF(cRtpParameters); + jstring jResult = (jstring) this->env->CallObjectMethod( + this->routerCallback, + sendTransportProduceCallback, + jKind, + jTransportId, + jRtpParameters + ); + const char* result = this->env->GetStringUTFChars(jResult, nullptr); + this->env->DeleteLocalRef(jResult); + this->env->ReleaseStringUTFChars(jResult, result); + this->env->DeleteLocalRef(jKind); + this->env->DeleteLocalRef(jTransportId); + this->env->DeleteLocalRef(jRtpParameters); + this->env->DeleteLocalRef(jCallbackClazz); + return result; + } + + void RouterCallback::recvTransportConnectCallback(std::string transportId, std::string dtlsParameters) { + jclass jCallbackClazz = this->env->GetObjectClass(this->routerCallback); + jmethodID recvTransportConnectCallback = this->env->GetMethodID(jCallbackClazz, "recvTransportConnectCallback", "(Ljava/lang/String;Ljava/lang/String;)V"); + const char* cTransportId = transportId.data(); + const char* cDtlsParameters = dtlsParameters.data(); + jstring jTransportId = this->env->NewStringUTF(cTransportId); + jstring jDtlsParameters = this->env->NewStringUTF(cDtlsParameters); + this->env->CallVoidMethod( + this->routerCallback, + recvTransportConnectCallback, + jTransportId, + jDtlsParameters + ); + this->env->DeleteLocalRef(jTransportId); + this->env->DeleteLocalRef(jDtlsParameters); + this->env->DeleteLocalRef(jCallbackClazz); + } + + void RouterCallback::consumerNewCallback(std::string message, webrtc::MediaStreamTrackInterface* consumerMediaTrackPointer) { + jclass jCallbackClazz = this->env->GetObjectClass(this->routerCallback); + jmethodID consumerNewCallback = this->env->GetMethodID(jCallbackClazz, "consumerNewCallback", "(Ljava/lang/String;J;)V"); + const char* cMessage = message.data(); + jstring jMessage = this->env->NewStringUTF(cMessage); + this->env->CallVoidMethod( + this->routerCallback, + consumerNewCallback, + jMessage, + (jlong) consumerMediaTrackPointer + ); + this->env->DeleteLocalRef(jMessage); + this->env->DeleteLocalRef(jCallbackClazz); + } + +} \ No newline at end of file diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java index 625c779..eb30fec 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java @@ -106,10 +106,6 @@ public final class MediaManager { * 当前终端数量 */ private volatile int clientCount; - /** - * 是否预览视频 - */ - private boolean preview; /** * 是否打开音频播放 */ @@ -166,10 +162,13 @@ public final class MediaManager { * 媒体流：声音、视频 */ private MediaStream mediaStream; + private AudioSource audioSource; + private VideoSource videoSource; /** * 视频捕获 */ private VideoCapturer videoCapturer; + private SurfaceTextureHelper surfaceTextureHelper; /** * PeerConnectionFactory */ @@ -190,14 +189,14 @@ public final class MediaManager { for (MediaCodecInfo mediaCodecInfo : mediaCodecList.getCodecInfos()) { // if (mediaCodecInfo.isEncoder()) { final String[] supportedTypes = mediaCodecInfo.getSupportedTypes(); - Log.d(RecordClient.class.getSimpleName(), "编码器名称：" + mediaCodecInfo.getName()); - Log.d(RecordClient.class.getSimpleName(), "编码器类型：" + String.join(" , ", supportedTypes)); + Log.d(MediaManager.class.getSimpleName(), "编码器名称：" + mediaCodecInfo.getName()); + Log.d(MediaManager.class.getSimpleName(), "编码器类型：" + String.join(" , ", supportedTypes)); for (String supportType : supportedTypes) { final MediaCodecInfo.CodecCapabilities codecCapabilities = mediaCodecInfo.getCapabilitiesForType(supportType); - final int[] colorFormats = codecCapabilities.colorFormats; - Log.d(RecordClient.class.getSimpleName(), "编码器格式：" + codecCapabilities.getMimeType()); + Log.d(MediaManager.class.getSimpleName(), "编码器支持的文件格式：" + codecCapabilities.getMimeType()); // MediaCodecInfo.CodecCapabilities.COLOR_* - Log.d(RecordClient.class.getSimpleName(), "编码器支持格式：" + IntStream.of(colorFormats).boxed().map(String::valueOf).collect(Collectors.joining(" , "))); +// final int[] colorFormats = codecCapabilities.colorFormats; +// Log.d(MediaManager.class.getSimpleName(), "编码器支持的色彩格式：" + IntStream.of(colorFormats).boxed().map(String::valueOf).collect(Collectors.joining(" , "))); } // } } @@ -210,7 +209,6 @@ public final class MediaManager { /** * @param handler Handler * @param context 上下文 - * @param preview 是否预览视频 * @param playAudio 是否播放音频 * @param playVideo 是否播放视频 * @param audioConsume 是否消费音频 @@ -220,14 +218,13 @@ public final class MediaManager { */ public void initContext( Handler handler, Context context, - boolean preview, boolean playAudio, boolean playVideo, + boolean playAudio, boolean playVideo, boolean audioConsume, boolean videoConsume, boolean audioProduce, boolean videoProduce, TransportType transportType ) { this.handler = handler; this.context = context; - this.preview = preview; this.playAudio = playAudio; this.playVideo = playVideo; this.audioConsume = audioConsume; @@ -355,7 +352,7 @@ public final class MediaManager { .setVideoDecoderFactory(videoDecoderFactory) .setVideoEncoderFactory(videoEncoderFactory) .createPeerConnectionFactory(); - this.mediaStream = this.peerConnectionFactory.createLocalMediaStream("ARDAMS"); + this.mediaStream = this.peerConnectionFactory.createLocalMediaStream("Taoyao"); Arrays.stream(videoEncoderFactory.getSupportedCodecs()).forEach(v -> { Log.d(MediaManager.class.getSimpleName(), "支持的视频解码器：" + v.name); }); @@ -412,8 +409,8 @@ public final class MediaManager { mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googNoiseSuppression", "true")); // mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googNoiseSuppression2", "true")); // mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googTypingNoiseDetection", "true")); - final AudioSource audioSource = this.peerConnectionFactory.createAudioSource(mediaConstraints); - final AudioTrack audioTrack = this.peerConnectionFactory.createAudioTrack("ARDAMSa0", audioSource); + this.audioSource = this.peerConnectionFactory.createAudioSource(mediaConstraints); + final AudioTrack audioTrack = this.peerConnectionFactory.createAudioTrack("TaoyaoA0", this.audioSource); // audioTrack.setVolume(100); audioTrack.setEnabled(true); this.mediaStream.addTrack(audioTrack); @@ -473,15 +470,15 @@ public final class MediaManager { */ private void initVideoTrack() { // 加载视频 - final SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create("MediaVideoThread", this.eglBase.getEglBaseContext()); -// surfaceTextureHelper.setTextureSize(); -// surfaceTextureHelper.setFrameRotation(); - final VideoSource videoSource = this.peerConnectionFactory.createVideoSource(this.videoCapturer.isScreencast()); + this.surfaceTextureHelper = SurfaceTextureHelper.create("MediaVideoThread", this.eglBase.getEglBaseContext()); +// this.surfaceTextureHelper.setTextureSize(); +// this.surfaceTextureHelper.setFrameRotation(); + this.videoSource = this.peerConnectionFactory.createVideoSource(this.videoCapturer.isScreencast()); // 美颜水印 -// videoSource.setVideoProcessor(); - this.videoCapturer.initialize(surfaceTextureHelper, this.context, videoSource.getCapturerObserver()); +// this.videoSource.setVideoProcessor(); + this.videoCapturer.initialize(this.surfaceTextureHelper, this.context, this.videoSource.getCapturerObserver()); this.videoCapturer.startCapture(480, 640, 30); - final VideoTrack videoTrack = this.peerConnectionFactory.createVideoTrack("ARDAMSv0", videoSource); + final VideoTrack videoTrack = this.peerConnectionFactory.createVideoTrack("TaoyaoV0", this.videoSource); videoTrack.addSink(videoFrame -> { if(this.recordClient != null) { this.recordClient.putVideo(videoFrame); @@ -492,17 +489,12 @@ public final class MediaManager { Log.i(MediaManager.class.getSimpleName(), "加载视频：" + videoTrack.id()); } - public boolean isPreview() { - return this.preview; - } - public MediaStream getMediaStream() { return this.mediaStream; } /** * @param flag Config.WHAT_* - * @param forceEnabled 是否强制播放 * @param videoTrack 视频媒体流Track * * @return 播放控件 @@ -575,8 +567,8 @@ public final class MediaManager { final Iterator iterator = this.mediaStream.audioTracks.iterator(); while (iterator.hasNext()) { track = iterator.next(); - iterator.remove(); track.dispose(); + iterator.remove(); } } } @@ -585,9 +577,7 @@ public final class MediaManager { * 关闭视频 */ private void closeVideoTrack() { - // 次码流 this.closeVideoTrack(this.mediaStream.videoTracks); - // 主码流 this.closeVideoTrack(this.mediaStream.preservedVideoTracks); } @@ -602,8 +592,8 @@ public final class MediaManager { final Iterator iterator = list.iterator(); while (iterator.hasNext()) { track = iterator.next(); - iterator.remove(); track.dispose(); + iterator.remove(); } } } @@ -618,14 +608,26 @@ public final class MediaManager { this.eglBase.release(); this.eglBase = null; } - if (this.videoCapturer != null) { - this.videoCapturer.dispose(); - this.videoCapturer = null; + if(this.audioSource != null) { + this.audioSource.dispose(); + this.audioSource = null; + } + if(this.videoSource != null) { + this.videoSource.dispose(); + this.videoSource = null; } if (this.mediaStream != null) { this.mediaStream.dispose(); this.mediaStream = null; } + if (this.videoCapturer != null) { + this.videoCapturer.dispose(); + this.videoCapturer = null; + } + if(this.surfaceTextureHelper != null) { + this.surfaceTextureHelper.dispose(); + this.surfaceTextureHelper = null; + } if (this.peerConnectionFactory != null) { this.peerConnectionFactory.dispose(); this.peerConnectionFactory = null; diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RouterCallback.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RouterCallback.java index d74ca62..9db169a 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RouterCallback.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RouterCallback.java @@ -13,7 +13,7 @@ public interface RouterCallback { default void sendTransportConnectCallback(String transportId, String dtlsParameters) {}; default String sendTransportProduceCallback(String kind, String transportId, String rtpParameters) { return null; }; default void recvTransportConnectCallback(String transportId, String dtlsParameters) {}; - default void producerNewCallback(String kind, String producerId, long producerMediaTrackPointer) { }; + default void producerNewCallback(String kind, String producerId, long producerMediaTrackPointer) {}; default void producerPauseCallback(String producerId) {}; default void producerResumeCallback(String producerId) {}; default void producerCloseCallback(String producerId) {}; diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/VoiceChangerProcesser.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/AudioChangerProcesser.java similarity index 70% rename from taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/VoiceChangerProcesser.java rename to taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/AudioChangerProcesser.java index b47b1e1..d2e765f 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/VoiceChangerProcesser.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/AudioChangerProcesser.java @@ -5,5 +5,6 @@ package com.acgist.taoyao.media.audio; * * @author acgist */ -public class VoiceChangerProcesser { +public class AudioChangerProcesser { + } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Room.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Room.java index 9140076..0c5d953 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Room.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Room.java @@ -36,6 +36,7 @@ public class Room extends CloseableClient implements RouterCallback { private final String clientId; private final String roomId; private final String password; + private final boolean preview; private final boolean dataConsume; private final boolean audioConsume; private final boolean videoConsume; @@ -47,24 +48,27 @@ public class Room extends CloseableClient implements RouterCallback { private Map remoteClients; private PeerConnection.RTCConfiguration rtcConfiguration; private PeerConnectionFactory peerConnectionFactory; + private String rtpCapabilities; private String sctpCapabilities; public Room( String name, String clientId, String roomId, String password, ITaoyao taoyao, Handler handler, + boolean preview, boolean dataConsume, boolean audioConsume, boolean videoConsume, boolean dataProduce, boolean audioProduce, boolean videoProduce ) { super(taoyao, handler); this.name = name; this.clientId = clientId; - this.roomId = roomId; + this.roomId = roomId; this.password = password; - this.dataConsume = dataConsume; + this.preview = preview; + this.dataConsume = dataConsume; this.audioConsume = audioConsume; this.videoConsume = videoConsume; - this.dataProduce = dataProduce; + this.dataProduce = dataProduce; this.audioProduce = audioProduce; this.videoProduce = videoProduce; this.nativeRoomPointer = this.nativeNewRoom(roomId, this); @@ -268,11 +272,13 @@ public class Room extends CloseableClient implements RouterCallback { @Override public void enterCallback(String rtpCapabilities, String sctpCapabilities) { + this.rtpCapabilities = rtpCapabilities; + this.sctpCapabilities = sctpCapabilities; this.taoyao.request(this.taoyao.buildMessage( "room::enter", - "roomId", this.roomId, - "password", this.password, - "rtpCapabilities", rtpCapabilities, + "roomId", this.roomId, + "password", this.password, + "rtpCapabilities", rtpCapabilities, "sctpCapabilities", sctpCapabilities )); } @@ -281,8 +287,8 @@ public class Room extends CloseableClient implements RouterCallback { public void sendTransportConnectCallback(String transportId, String dtlsParameters) { this.taoyao.request(this.taoyao.buildMessage( "media::transport::webrtc::connect", - "roomId", this.roomId, - "transportId", transportId, + "roomId", this.roomId, + "transportId", transportId, "dtlsParameters", JSONUtils.toMap(dtlsParameters) )); } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/signal/ITaoyaoListener.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/signal/ITaoyaoListener.java new file mode 100644 index 0000000..67057d9 --- /dev/null +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/signal/ITaoyaoListener.java @@ -0,0 +1,59 @@ +package com.acgist.taoyao.media.signal; + +import com.acgist.taoyao.boot.model.Message; + +import java.util.Map; + +/** + * 信令监听 + * + * @author acgist + */ +public interface ITaoyaoListener { + + /** + * 前置信令处理 + * + * @param message 信令消息 + * + * @return 是否继续处理信令 + */ + default boolean preOnMessage(Message message) { + return false; + } + + /** + * 后置信令处理 + * + * @param message 信令消息 + */ + default void postOnMessage(Message message) { + } + + /** + * 信令正在连接 + */ + default void onConnect() { + } + + /** + * 信令连接成功 + */ + default void onConnected() { + } + + /** + * 信令断开连接 + */ + default void onDisconnect() { + } + + /** + * 信令异常 + * + * @param throwable 异常 + */ + default void onError(Throwable throwable) { + } + +} diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/AiProcesser.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/AiProcesser.java index 5c7c898..1dc85e5 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/AiProcesser.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/AiProcesser.java @@ -1,9 +1,30 @@ package com.acgist.taoyao.media.video; +import org.webrtc.VideoFrame; +import org.webrtc.VideoProcessor; +import org.webrtc.VideoSink; + /** * AI处理器 * * @author acgist */ -public class AiProcesser { +public class AiProcesser implements VideoProcessor { + + @Override + public void setSink(VideoSink videoSink) { + } + + @Override + public void onCapturerStarted(boolean status) { + } + + @Override + public void onCapturerStopped() { + } + + @Override + public void onFrameCaptured(VideoFrame videoFrame) { + } + } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/BeautyProcesser.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/BeautyProcesser.java index 4997753..61887e3 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/BeautyProcesser.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/BeautyProcesser.java @@ -1,9 +1,29 @@ package com.acgist.taoyao.media.video; +import org.webrtc.VideoFrame; +import org.webrtc.VideoProcessor; +import org.webrtc.VideoSink; + /** * 美颜处理器 * * @author acgist */ -public class BeautyProcesser { +public class BeautyProcesser implements VideoProcessor { + + @Override + public void setSink(VideoSink videoSink) { + } + + @Override + public void onCapturerStarted(boolean status) { + } + + @Override + public void onCapturerStopped() { + } + + @Override + public void onFrameCaptured(VideoFrame videoFrame) { + } } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/WatermarkProcesser.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/WatermarkProcesser.java index a5c6499..25fd9c6 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/WatermarkProcesser.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/WatermarkProcesser.java @@ -1,9 +1,30 @@ package com.acgist.taoyao.media.video; +import org.webrtc.VideoFrame; +import org.webrtc.VideoProcessor; +import org.webrtc.VideoSink; + /** * 水印处理器 * * @author acgist */ -public class WatermarkProcesser { +public class WatermarkProcesser implements VideoProcessor { + + @Override + public void setSink(VideoSink videoSink) { + } + + @Override + public void onCapturerStarted(boolean status) { + } + + @Override + public void onCapturerStopped() { + } + + @Override + public void onFrameCaptured(VideoFrame videoFrame) { + } + }