[*] 优化

taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/MixerProcesser.java

@@ -23,7 +23,7 @@ import java.util.concurrent.TimeUnit;
  * WebRtcAudioTrack#AudioTrackThread  ：远程音频
  * WebRtcAudioRecord#AudioRecordThread：本地音频
  *
- * AudioFormat.ENCODING_PCM_16BIT = 2KB
+ * AudioFormat.ENCODING_PCM_16BIT = 2Byte
  *
  * PCM时间计算：1_000_000 microseconds / 48000 hz / 2 bytes
  *

taoyao-client-openharmony/taoyao/media/src/main/cpp/CMakeLists.txt

@@ -22,6 +22,7 @@ set(LIBWEBRTC_BINARY_PATH  "${CMAKE_CURRENT_SOURCE_DIR}/deps/webrtc/lib/${OHOS_A
 set(LIBWEBRTC_INCLUDE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/deps/webrtc/src/")
 
 ADD_DEFINITIONS(-DNDEBUG)
+ADD_DEFINITIONS(-DLOG_TAG="libtaoyao")
 ADD_DEFINITIONS(-DVK_USE_PLATFORM_OHOS=1)
 
 add_subdirectory("./deps/libmediasoupclient")

taoyao-client-openharmony/taoyao/media/src/main/cpp/bind.cpp

@@ -25,8 +25,8 @@
 #include <multimedia/player_framework/native_avcapability.h>
 #include <multimedia/player_framework/native_avcodec_base.h>
 
-static std::mutex roomMutex;
+static std::recursive_mutex roomMutex;
-static std::mutex taoyaoMutex;
+static std::recursive_mutex taoyaoMutex;
 
 #ifndef TAOYAO_JSON_SIZE
 #define TAOYAO_JSON_SIZE 2048
@@ -74,11 +74,11 @@ static std::string clientId = "";
 // 终端名称
 static std::string name     = "";
 // ETS环境
-static napi_env env = nullptr;
+static napi_env env    = nullptr;
 // 是否加载
 static bool initTaoyao = false;
 // PUSH方法引用
-static napi_ref pushRef = nullptr;
+static napi_ref pushRef    = nullptr;
 // REQUEST方法引用
 static napi_ref requestRef = nullptr;
 // 媒体功能
@@ -153,7 +153,7 @@ std::string request(const std::string& signal, const std::string& body, uint64_t
 static napi_value init(napi_env env, napi_callback_info info) {
     TAOYAO_JSON_BODY(3);
     {
-        std::lock_guard<std::mutex> taoyaoLock(taoyaoMutex);
+        std::lock_guard<std::recursive_mutex> taoyaoLock(taoyaoMutex);
         if(initTaoyao) {
             napi_create_int32(env, -1, &ret);
             OH_LOG_WARN(LOG_APP, "libtaoyao已经加载");
@@ -183,8 +183,8 @@ static napi_value init(napi_env env, napi_callback_info info) {
 static napi_value shutdown(napi_env env, napi_callback_info info) {
     napi_value ret;
     {
-        std::lock_guard<std::mutex> taoyaoLock(taoyaoMutex);
+        std::lock_guard<std::recursive_mutex> taoyaoLock(taoyaoMutex);
-        if(!initTaoyao) {
+        if (!initTaoyao) {
             napi_create_int32(env, -1, &ret);
             OH_LOG_INFO(LOG_APP, "libtaoyao已经卸载");
             return ret;
@@ -221,7 +221,7 @@ static napi_value shutdown(napi_env env, napi_callback_info info) {
 static napi_value roomClose(napi_env env, napi_callback_info info) {
     TAOYAO_JSON_BODY(1);
     {
-        std::lock_guard<std::mutex> roomLock(roomMutex);
+        std::lock_guard<std::recursive_mutex> roomLock(roomMutex);
         std::string roomId = body["roomId"];
         auto iterator = acgist::roomMap.find(roomId);
         if(iterator == acgist::roomMap.end()) {
@@ -285,7 +285,7 @@ static napi_value roomExpel(napi_env env, napi_callback_info info) {
 static napi_value roomInvite(napi_env env, napi_callback_info info) {
     TAOYAO_JSON_BODY(1);
     {
-        std::lock_guard<std::mutex> roomLock(roomMutex);
+        std::lock_guard<std::recursive_mutex> roomLock(roomMutex);
         // TODO: 试试引用
         std::string roomId   = body["roomId"];
         std::string password = body["password"];

taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Capturer.hpp

@@ -1,6 +1,8 @@
 /**
  * 采集器
  * 
+ * 音频默认使用OHOS实现：modules/audio_device/ohos/audio_device_template.h
+ * 
  * @author acgist
  * 
  * https://docs.openharmony.cn/pages/v4.1/zh-cn/application-dev/media/camera/camera-overview.md
@@ -18,11 +20,16 @@
 #define TAOYAO_CAPTURER_HPP
 
 // OpenGL ES || VULKAN
-#define __VULKAN__ true
+#define __TAOYAO_VULKAN__ true
-#ifndef __VULKAN__
+#ifndef __TAOYAO_VULKAN__
-#define __OPENGL__ true
+#define __TAOYAO_OPENGL__ true
 #endif
 
+// 本地音频采集
+#define __TAOYAO_AUDIO_LOCAL__ false
+// 本地视频采集
+#define __TAOYAO_VIDEO_LOCAL__ true
+
 #include <map>
 
 #include <EGL/egl.h>
@@ -59,6 +66,7 @@ template <typename Source>
 class Capturer {
     
 protected:
+    // 是否运行
     bool running = false;
     
 public:

taoyao-client-openharmony/taoyao/media/src/main/cpp/include/MediaManager.hpp

@@ -28,7 +28,11 @@ public:
 
 public:
     int localClientRef = 0;
+    #if __TAOYAO_AUDIO_LOCAL__
+    acgist::TaoyaoAudioTrackSource* audioTrackSource = nullptr;
+    #else
     rtc::scoped_refptr<webrtc::AudioSourceInterface> audioTrackSource = nullptr;
+    #endif
     acgist::TaoyaoVideoTrackSource* videoTrackSource = nullptr;
     acgist::AudioCapturer* audioCapturer = nullptr;
     acgist::VideoCapturer* videoCapturer = nullptr;

taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Player.hpp

@@ -41,8 +41,10 @@ public:
 
 /**
  * 音频播放器
+ * 
+ * 默认交给音频设备播放
  */
-class AudioPlayer: public Player {
+class AudioPlayer : public Player {
 
 public:
     // 音频构造器
@@ -52,7 +54,7 @@ public:
 
 public:
     AudioPlayer();
-    virtual ~AudioPlayer();
+    virtual ~AudioPlayer() override;
 
 public:
     virtual bool start() override;
@@ -63,13 +65,13 @@ public:
 /**
  * 视频播放器
  * 
- * TODO: 实现留给有缘人了~.~!
+ * TODO: 如果需要自行实现
  */
-class VideoPlayer: public Player {
+class VideoPlayer : public Player {
 
 public:
     VideoPlayer();
-    virtual ~VideoPlayer();
+    virtual ~VideoPlayer() override;
 
 public:
     virtual bool start() override;

taoyao-client-openharmony/taoyao/media/src/main/cpp/include/WebRTC.hpp

@@ -43,10 +43,18 @@ namespace acgist {
 
 /**
  * 音频轨道来源
+ * 
+ * TODO: 媒体转发
  */
-class TaoyaoAudioTrackSource : public webrtc::AudioTrackSinkInterface {
+class TaoyaoAudioTrackSource : public webrtc::AudioTrackSinkInterface, public webrtc::Notifier<webrtc::AudioSourceInterface> {
 
 public:
+    TaoyaoAudioTrackSource();
+    virtual ~TaoyaoAudioTrackSource();
+    
+public:
+    virtual webrtc::MediaSourceInterface::SourceState state() const override;
+    virtual bool remote() const override;
     virtual void OnData(const void* audio_data, int bits_per_sample, int sample_rate, size_t number_of_channels, size_t number_of_frames) override;
 
 };
@@ -55,7 +63,7 @@ public:
  * 视频管道
  */
 class VideoTrackSinkInterface {
-
+    
 public:
     virtual void OnData(const webrtc::VideoFrame& videoFrame) = 0;
 
@@ -85,10 +93,16 @@ public:
 class TaoyaoVideoEncoder : public webrtc::VideoEncoder {
     
 public:
+    // 是否运行
+    bool running = false;
     // 视频编码器
     OH_AVCodec* avCodec = nullptr;
-    // 视频窗口
+    // 缓冲数据索引
-    OHNativeWindow* nativeWindow = nullptr;
+    uint32_t index      = 0;
+    // 缓冲数据
+    OH_AVBuffer* buffer = nullptr;
+    // 编码回调
+    webrtc::EncodedImageCallback* encodedImageCallback = nullptr;
     
 public:
     TaoyaoVideoEncoder();
@@ -115,7 +129,7 @@ public:
     virtual int32_t RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback* callback) override;
     virtual void SetRates(const webrtc::VideoEncoder::RateControlParameters& parameters) override;
     virtual webrtc::VideoEncoder::EncoderInfo GetEncoderInfo() const override;
-    virtual int32_t Encode(const webrtc::VideoFrame& frame, const std::vector<webrtc::VideoFrameType>* frame_types) override;
+    virtual int32_t Encode(const webrtc::VideoFrame& videoFrame, const std::vector<webrtc::VideoFrameType>* frame_types) override;
     
 };
 

taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioCapturer.cpp

@@ -4,8 +4,12 @@
  */
 #include "../include/Capturer.hpp"
 
+#include <mutex>
+
 #include <hilog/log.h>
 
+static std::recursive_mutex audioMutex;
+
 // 采集回调
 static int32_t OnError(OH_AudioCapturer* capturer, void* userData, OH_AudioStream_Result error);
 static int32_t OnReadData(OH_AudioCapturer* capturer, void* userData, void* buffer, int32_t length);
@@ -14,21 +18,21 @@ static int32_t OnInterruptEvent(OH_AudioCapturer* capturer, void* userData, OH_A
 
 acgist::AudioCapturer::AudioCapturer() {
     OH_AudioStream_Result ret = OH_AudioStreamBuilder_Create(&this->builder, AUDIOSTREAM_TYPE_RENDERER);
-    OH_LOG_INFO(LOG_APP, "构造音频构造器：%o", ret);
+    OH_LOG_INFO(LOG_APP, "配置音频构造器：%o", ret);
-    // 配置音频录制参数
+    // 配置音频采集参数
     OH_AudioStreamBuilder_SetSamplingRate(this->builder, acgist::samplingRate);
     OH_AudioStreamBuilder_SetChannelCount(this->builder, acgist::channelCount);
     OH_AudioStreamBuilder_SetLatencyMode(this->builder,  OH_AudioStream_LatencyMode::AUDIOSTREAM_LATENCY_MODE_NORMAL);
     OH_AudioStreamBuilder_SetSampleFormat(this->builder, OH_AudioStream_SampleFormat::AUDIOSTREAM_SAMPLE_S16LE);
-    OH_LOG_DEBUG(LOG_APP, "配置音频录制参数：%d %d", acgist::samplingRate, acgist::channelCount);
+    OH_LOG_DEBUG(LOG_APP, "配置音频采集参数：%d %d", acgist::samplingRate, acgist::channelCount);
-    // 设置回调函数
+    // 设置采集回调
     OH_AudioCapturer_Callbacks callbacks;
     callbacks.OH_AudioCapturer_OnError          = OnError;
     callbacks.OH_AudioCapturer_OnReadData       = OnReadData;
     callbacks.OH_AudioCapturer_OnStreamEvent    = OnStreamEvent;
     callbacks.OH_AudioCapturer_OnInterruptEvent = OnInterruptEvent;
     ret = OH_AudioStreamBuilder_SetCapturerCallback(this->builder, callbacks, this);
-    OH_LOG_DEBUG(LOG_APP, "设置录制回调函数：%o", ret);
+    OH_LOG_DEBUG(LOG_APP, "设置音频采集回调：%o", ret);
 }
 
 acgist::AudioCapturer::~AudioCapturer() {
@@ -36,18 +40,19 @@ acgist::AudioCapturer::~AudioCapturer() {
     if(this->builder != nullptr) {
         OH_AudioStream_Result ret = OH_AudioStreamBuilder_Destroy(this->builder);
         this->builder = nullptr;
-        OH_LOG_INFO(LOG_APP, "释放音频采集：%o", ret);
+        OH_LOG_INFO(LOG_APP, "释放音频构造器：%o", ret);
     }
 }
 
 bool acgist::AudioCapturer::start() {
+    std::lock_guard<std::recursive_mutex> audioLock(audioMutex);
     if(this->running) {
         return true;
     }
     this->running = true;
-    // 构造音频采集器
+    // 配置音频采集器
     OH_AudioStream_Result ret = OH_AudioStreamBuilder_GenerateCapturer(this->builder, &this->audioCapturer);
-    OH_LOG_DEBUG(LOG_APP, "构造音频采集器：%o", ret);
+    OH_LOG_DEBUG(LOG_APP, "配置音频采集器：%o", ret);
     // 开始音频采集
     ret = OH_AudioCapturer_Start(this->audioCapturer);
     OH_LOG_DEBUG(LOG_APP, "开始音频采集：%o", ret);
@@ -55,13 +60,11 @@ bool acgist::AudioCapturer::start() {
 }
 
 bool acgist::AudioCapturer::stop() {
+    std::lock_guard<std::recursive_mutex> audioLock(audioMutex);
     if(!this->running) {
         return true;
     }
     this->running = false;
-    if(this->audioCapturer == nullptr) {
-        return true;
-    }
     // 停止音频采集
     OH_AudioStream_Result ret = OH_AudioCapturer_Stop(this->audioCapturer);
     OH_LOG_DEBUG(LOG_APP, "停止音频采集：%o", ret);
@@ -78,8 +81,16 @@ static int32_t OnError(OH_AudioCapturer* capturer, void* userData, OH_AudioStrea
 }
 
 static int32_t OnReadData(OH_AudioCapturer* capturer, void* userData, void* buffer, int32_t length) {
+    if(userData == nullptr) {
+        return -1;
+    }
     acgist::AudioCapturer* audioCapturer = (acgist::AudioCapturer*) userData;
-    audioCapturer->source->OnData(buffer, acgist::bitsPerSample, acgist::samplingRate, acgist::channelCount, sizeof(buffer) / 2);
+    if(audioCapturer->source == nullptr) {
+        return -2;
+    }
+    // 单声道 48000 / 1000 * 10 * 2(16bit)
+    // 双声道 48000 / 1000 * 10 * 2(16bit) * 2
+    audioCapturer->source->OnData(buffer, acgist::bitsPerSample, acgist::samplingRate, acgist::channelCount, length / 2);
     return 0;
 }
 
@@ -89,6 +100,6 @@ static int32_t OnStreamEvent(OH_AudioCapturer* capturer, void* userData, OH_Audi
 }
 
 static int32_t OnInterruptEvent(OH_AudioCapturer* capturer, void* userData, OH_AudioInterrupt_ForceType type, OH_AudioInterrupt_Hint hint) {
-    OH_LOG_DEBUG(LOG_APP, "音频采集打断：%o %o", type, hint);
+    OH_LOG_DEBUG(LOG_APP, "打断音频采集：%o %o", type, hint);
     return 0;
 }

taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioPlayer.cpp

@@ -1,15 +1,7 @@
-/**
- * 音频播放不用实现（系统已经实现）
- * 这里只是用来学习使用
- */
-
 #include "../include/Player.hpp"
 
 #include <hilog/log.h>
 
-#include <ohaudio/native_audiorenderer.h>
-#include <ohaudio/native_audiostreambuilder.h>
-
 // 播放回调
 static int32_t OnError(OH_AudioRenderer* renderer, void* userData, OH_AudioStream_Result error);
 static int32_t OnWriteData(OH_AudioRenderer* renderer, void* userData, void* buffer, int32_t length);

taoyao-client-openharmony/taoyao/media/src/main/cpp/media/MediaManager.cpp

@@ -14,7 +14,7 @@
 #include <api/video_codecs/builtin_video_decoder_factory.h>
 #include <api/video_codecs/builtin_video_encoder_factory.h>
 
-static std::mutex mediaMutex;
+static std::recursive_mutex mediaMutex;
 
 acgist::MediaManager::MediaManager() {
 }
@@ -81,19 +81,20 @@ bool acgist::MediaManager::releasePeerConnectionFactory() {
 
 int acgist::MediaManager::newLocalClient() {
     {
-        std::lock_guard<std::mutex> mediaLock(mediaMutex);
+        std::lock_guard<std::recursive_mutex> mediaLock(mediaMutex);
         this->localClientRef++;
         if(this->localClientRef > 0) {
             this->newPeerConnectionFactory();
             this->startCapture();
         }
     }
+    OH_LOG_WARN(LOG_APP, "打开本地终端：%d", this->localClientRef);
     return this->localClientRef;
 }
 
 int acgist::MediaManager::releaseLocalClient() {
     {
-        std::lock_guard<std::mutex> mediaLock(mediaMutex);
+        std::lock_guard<std::recursive_mutex> mediaLock(mediaMutex);
         this->localClientRef--;
         if(this->localClientRef <= 0) {
             if(this->localClientRef < 0) {
@@ -104,6 +105,7 @@ int acgist::MediaManager::releaseLocalClient() {
             }
         }
     }
+    OH_LOG_WARN(LOG_APP, "关闭本地终端：%d", this->localClientRef);
     return this->localClientRef;
 }
 
@@ -114,20 +116,42 @@ bool acgist::MediaManager::startCapture() {
 }
 
 bool acgist::MediaManager::startAudioCapture() {
-    if(this->audioCapturer != nullptr) {
+    #if __TAOYAO_AUDIO_LOCAL__
-        return true;
+        if (this->audioCapturer == nullptr) {
-    }
+            OH_LOG_WARN(LOG_APP, "开始音频采集");
-    this->audioCapturer = new acgist::AudioCapturer();
+            this->audioCapturer = new acgist::AudioCapturer();
-    this->audioCapturer->start();
+            this->audioCapturer->start();
+        }
+        if(this->audioTrackSource == nullptr) {
+            OH_LOG_WARN(LOG_APP, "设置音频来源");
+            this->audioTrackSource = new rtc::RefCountedObject<acgist::TaoyaoAudioTrackSource>();
+            this->audioCapturer->source = this->audioTrackSource;
+        }
+    #else
+        if(this->audioTrackSource == nullptr) {
+            OH_LOG_WARN(LOG_APP, "设置音频来源");
+            cricket::AudioOptions options;
+            options.highpass_filter   = true;
+            options.auto_gain_control = true;
+            options.echo_cancellation = true;
+            options.noise_suppression = true;
+            this->audioTrackSource = this->peerConnectionFactory->CreateAudioSource(options);
+        }
+    #endif
     return true;
 }
 
 bool acgist::MediaManager::startVideoCapture() {
-    if(this->videoCapturer != nullptr) {
+    if(this->videoCapturer == nullptr) {
-        return true;
+        OH_LOG_WARN(LOG_APP, "开始视频采集");
+        this->videoCapturer = new acgist::VideoCapturer();
+        this->videoCapturer->start();
+    }
+    if(this->videoTrackSource == nullptr) {
+        OH_LOG_WARN(LOG_APP, "设置视频来源");
+        this->videoTrackSource = new rtc::RefCountedObject<acgist::TaoyaoVideoTrackSource>();
+        this->videoCapturer->source = this->videoTrackSource;
     }
-    this->videoCapturer = new acgist::VideoCapturer();
-    this->videoCapturer->start();
     return true;
 }
 
@@ -138,36 +162,54 @@ bool acgist::MediaManager::stopCapture() {
 }
 
 bool acgist::MediaManager::stopAudioCapture() {
-    if(this->audioCapturer == nullptr) {
+    #if __TAOYAO_AUDIO_LOCAL__
-        return true;
+    if(this->audioCapturer != nullptr) {
+        OH_LOG_WARN(LOG_APP, "停止音频采集");
+        this->audioCapturer->stop();
+        delete this->audioCapturer;
+        this->audioCapturer = nullptr;
     }
-    this->audioCapturer->stop();
+    if(this->audioTrackSource != nullptr) {
-    delete this->audioCapturer;
+        OH_LOG_WARN(LOG_APP, "释放音频来源");
-    this->audioCapturer = nullptr;
+        this->audioTrackSource->Release();
+        // delete this->AudioTrackSource;
+        this->audioTrackSource = nullptr;
+    }
+    #else
+        if(this->audioTrackSource != nullptr) {
+            OH_LOG_WARN(LOG_APP, "释放音频来源");
+            this->audioTrackSource->Release();
+            // delete this->audioTrackSource;
+            this->audioTrackSource = nullptr;
+        }
+    #endif
     return true;
 }
 
 bool acgist::MediaManager::stopVideoCapture() {
-    if(this->videoCapturer == nullptr) {
+    if(this->videoCapturer != nullptr) {
-        return true;
+        OH_LOG_WARN(LOG_APP, "停止视频采集");
+        this->videoCapturer->stop();
+        delete this->videoCapturer;
+        this->videoCapturer = nullptr;
+    }
+    if(this->videoTrackSource != nullptr) {
+        OH_LOG_WARN(LOG_APP, "释放视频来源");
+        this->videoTrackSource->Release();
+        // delete this->videoTrackSource;
+        this->videoTrackSource = nullptr;
     }
-    this->videoCapturer->stop();
-    delete this->videoCapturer;
-    this->videoCapturer = nullptr;
     return true;
 }
 
 rtc::scoped_refptr<webrtc::AudioTrackInterface> acgist::MediaManager::getAudioTrack() {
-    cricket::AudioOptions options;
+    #if __TAOYAO_AUDIO_LOCAL__
-    options.highpass_filter   = true;
+    return this->peerConnectionFactory->CreateAudioTrack("taoyao-audio", this->audioTrackSource);
-    options.auto_gain_control = true;
+    #else
-    options.echo_cancellation = true;
+    return this->peerConnectionFactory->CreateAudioTrack("taoyao-audio", this->audioTrackSource.get());
-    options.noise_suppression = true;
+    #endif
-    this->audioTrackSource = this->peerConnectionFactory->CreateAudioSource(options);
-    return this->peerConnectionFactory->CreateAudioTrack("taoyao-audio", audioTrackSource.get());
 }
 
 rtc::scoped_refptr<webrtc::VideoTrackInterface> acgist::MediaManager::getVideoTrack() {
-    this->videoTrackSource = new rtc::RefCountedObject<acgist::TaoyaoVideoTrackSource>();
     return this->peerConnectionFactory->CreateVideoTrack("taoyao-video", this->videoTrackSource);
 }

taoyao-client-openharmony/taoyao/media/src/main/cpp/media/RemoteClient.cpp

@@ -4,7 +4,7 @@
 
 #include "hilog/log.h"
 
-static std::mutex clientMutex;
+static std::recursive_mutex clientMutex;
 
 acgist::RemoteClient::RemoteClient(acgist::MediaManager* mediaManager) : RoomClient(mediaManager) {}
 
@@ -18,7 +18,7 @@ acgist::RemoteClient::~RemoteClient() {
 }
 
 bool acgist::RemoteClient::addConsumer(const std::string& consumerId, mediasoupclient::Consumer* consumer) {
-    std::lock_guard<std::mutex> clientLock(clientMutex);
+    std::lock_guard<std::recursive_mutex> clientLock(clientMutex);
     auto oldConsumer = this->consumers.find(clientId);
     if(oldConsumer != this->consumers.end()) {
         OH_LOG_INFO(LOG_APP, "关闭旧的消费者：%s", consumerId.data());
@@ -35,7 +35,7 @@ bool acgist::RemoteClient::addConsumer(const std::string& consumerId, mediasoupc
 }
 
 bool acgist::RemoteClient::closeConsumer(const std::string& consumerId) {
-    std::lock_guard<std::mutex> clientLock(clientMutex);
+    std::lock_guard<std::recursive_mutex> clientLock(clientMutex);
     auto consumer = this->consumers.find(consumerId);
     if(consumer == this->consumers.end()) {
         OH_LOG_INFO(LOG_APP, "消费者已经关闭：%s", consumerId.data());
@@ -50,7 +50,7 @@ bool acgist::RemoteClient::closeConsumer(const std::string& consumerId) {
 }
 
 bool acgist::RemoteClient::pauseConsumer(const std::string& consumerId) {
-    std::lock_guard<std::mutex> clientLock(clientMutex);
+    std::lock_guard<std::recursive_mutex> clientLock(clientMutex);
     auto consumer = this->consumers.find(consumerId);
     if(consumer == this->consumers.end()) {
         OH_LOG_INFO(LOG_APP, "无效消费者：%s", consumerId.data());
@@ -62,7 +62,7 @@ bool acgist::RemoteClient::pauseConsumer(const std::string& consumerId) {
 }
 
 bool acgist::RemoteClient::resumeConsumer(const std::string& consumerId) {
-    std::lock_guard<std::mutex> clientLock(clientMutex);
+    std::lock_guard<std::recursive_mutex> clientLock(clientMutex);
     auto consumer = this->consumers.find(consumerId);
     if(consumer == this->consumers.end()) {
         OH_LOG_INFO(LOG_APP, "无效消费者：%s", consumerId.data());

taoyao-client-openharmony/taoyao/media/src/main/cpp/media/Room.cpp

@@ -4,7 +4,7 @@
 
 #include "hilog/log.h"
 
-static std::mutex roomMutex;
+static std::recursive_mutex roomMutex;
 
 acgist::Room::Room(const std::string& roomId, acgist::MediaManager* mediaManager) : roomId(roomId), mediaManager(mediaManager) {
     this->device = new mediasoupclient::Device();
@@ -243,7 +243,7 @@ int acgist::Room::close() {
 }
 
 int acgist::Room::closeClient() {
-    std::lock_guard<std::mutex> lockRoom(roomMutex);
+    std::lock_guard<std::recursive_mutex> lockRoom(roomMutex);
     OH_LOG_INFO(LOG_APP, "关闭本地终端：%s", this->roomId.data());
     if(this->client != nullptr) {
         delete this->client;
@@ -253,7 +253,7 @@ int acgist::Room::closeClient() {
 }
 
 int acgist::Room::closeClients() {
-    std::lock_guard<std::mutex> lockRoom(roomMutex);
+    std::lock_guard<std::recursive_mutex> lockRoom(roomMutex);
     OH_LOG_INFO(LOG_APP, "关闭远程终端：%s", this->roomId.data());
     for (auto iterator = this->clients.begin(); iterator != this->clients.end(); ++iterator) {
         if (iterator->second == nullptr) {
@@ -268,7 +268,7 @@ int acgist::Room::closeClients() {
 }
 
 int acgist::Room::closeAudioProducer() {
-    std::lock_guard<std::mutex> lockRoom(roomMutex);
+    std::lock_guard<std::recursive_mutex> lockRoom(roomMutex);
     OH_LOG_INFO(LOG_APP, "关闭音频生产者：%s", this->roomId.data());
     if (this->audioProducer != nullptr) {
         this->audioProducer->Close();
@@ -277,7 +277,7 @@ int acgist::Room::closeAudioProducer() {
 }
 
 int acgist::Room::closeVideoProducer() {
-    std::lock_guard<std::mutex> lockRoom(roomMutex);
+    std::lock_guard<std::recursive_mutex> lockRoom(roomMutex);
     OH_LOG_INFO(LOG_APP, "关闭视频生产者：%s", this->roomId.data());
     if (this->videoProducer != nullptr) {
         this->videoProducer->Close();
@@ -286,7 +286,7 @@ int acgist::Room::closeVideoProducer() {
 }
 
 int acgist::Room::closeTransport() {
-    std::lock_guard<std::mutex> lockRoom(roomMutex);
+    std::lock_guard<std::recursive_mutex> lockRoom(roomMutex);
     OH_LOG_INFO(LOG_APP, "关闭通道：%s", this->roomId.data());
     if (this->sendTransport != nullptr) {
         this->sendTransport->Close();
@@ -298,7 +298,7 @@ int acgist::Room::closeTransport() {
 }
 
 int acgist::Room::newRemoteClient(const std::string& clientId, const std::string& name) {
-    std::lock_guard<std::mutex> lockRoom(roomMutex);
+    std::lock_guard<std::recursive_mutex> lockRoom(roomMutex);
     auto oldClient = this->clients.find(clientId);
     if(oldClient != this->clients.end()) {
         OH_LOG_INFO(LOG_APP, "已经存在远程终端：%s", clientId.data());
@@ -315,7 +315,7 @@ int acgist::Room::newRemoteClient(const std::string& clientId, const std::string
 }
 
 int acgist::Room::closeRemoteClient(const std::string& clientId) {
-    std::lock_guard<std::mutex> lockRoom(roomMutex);
+    std::lock_guard<std::recursive_mutex> lockRoom(roomMutex);
     auto client = this->clients.find(clientId);
     if(client == this->clients.end()) {
         OH_LOG_INFO(LOG_APP, "远程终端已经删除：%s", clientId.data());
@@ -330,7 +330,7 @@ int acgist::Room::closeRemoteClient(const std::string& clientId) {
 }
 
 int acgist::Room::newConsumer(nlohmann::json& body) {
-    std::lock_guard<std::mutex> lockRoom(roomMutex);
+    std::lock_guard<std::recursive_mutex> lockRoom(roomMutex);
     mediasoupclient::Consumer* consumer = this->recvTransport->Consume(
         this->consumerListener,
         body["consumerId"],
@@ -359,7 +359,7 @@ int acgist::Room::newConsumer(nlohmann::json& body) {
 }
 
 int acgist::Room::closeConsumer(const std::string& consumerId) {
-    std::lock_guard<std::mutex> lockRoom(roomMutex);
+    std::lock_guard<std::recursive_mutex> lockRoom(roomMutex);
     auto clientId = this->consumerIdClientId.find(consumerId);
     if(clientId == this->consumerIdClientId.end()) {
         OH_LOG_INFO(LOG_APP, "关闭消费者无效：%s", consumerId.data());
@@ -375,7 +375,7 @@ int acgist::Room::closeConsumer(const std::string& consumerId) {
 }
 
 int acgist::Room::pauseConsumer(const std::string& consumerId) {
-    std::lock_guard<std::mutex> lockRoom(roomMutex);
+    std::lock_guard<std::recursive_mutex> lockRoom(roomMutex);
     auto clientId = this->consumerIdClientId.find(consumerId);
     if(clientId == this->consumerIdClientId.end()) {
         OH_LOG_INFO(LOG_APP, "暂停消费者无效：%s", consumerId.data());
@@ -391,7 +391,7 @@ int acgist::Room::pauseConsumer(const std::string& consumerId) {
 }
 
 int acgist::Room::resumeConsumer(const std::string& consumerId) {
-    std::lock_guard<std::mutex> lockRoom(roomMutex);
+    std::lock_guard<std::recursive_mutex> lockRoom(roomMutex);
     auto clientId = this->consumerIdClientId.find(consumerId);
     if(clientId == this->consumerIdClientId.end()) {
         OH_LOG_INFO(LOG_APP, "恢复消费者无效：%s", consumerId.data());
@@ -407,7 +407,7 @@ int acgist::Room::resumeConsumer(const std::string& consumerId) {
 }
 
 int acgist::Room::closeProducer(const std::string& producerId) {
-    std::lock_guard<std::mutex> lockRoom(roomMutex);
+    std::lock_guard<std::recursive_mutex> lockRoom(roomMutex);
     if(this->audioProducer != nullptr && this->audioProducer->GetId() == producerId) {
         OH_LOG_INFO(LOG_APP, "关闭音频生产者：%s", producerId.data());
         this->audioProducer->Close();
@@ -429,7 +429,7 @@ int acgist::Room::closeProducer(const std::string& producerId) {
 }
 
 int acgist::Room::pauseProducer(const std::string& producerId) {
-    std::lock_guard<std::mutex> lockRoom(roomMutex);
+    std::lock_guard<std::recursive_mutex> lockRoom(roomMutex);
     if(this->audioProducer != nullptr && this->audioProducer->GetId() == producerId) {
         OH_LOG_INFO(LOG_APP, "暂停音频生产者：%s", producerId.data());
         this->audioProducer->Pause();
@@ -443,7 +443,7 @@ int acgist::Room::pauseProducer(const std::string& producerId) {
 }
 
 int acgist::Room::resumeProducer(const std::string& producerId) {
-    std::lock_guard<std::mutex> lockRoom(roomMutex);
+    std::lock_guard<std::recursive_mutex> lockRoom(roomMutex);
     if(this->audioProducer != nullptr && this->audioProducer->GetId() == producerId) {
         OH_LOG_INFO(LOG_APP, "恢复音频生产者：%s", producerId.data());
         this->audioProducer->Resume();

taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoEncoder.cpp

@@ -36,9 +36,6 @@ acgist::TaoyaoVideoEncoder::TaoyaoVideoEncoder() {
     // 准备就绪
     ret = OH_VideoEncoder_Prepare(this->avCodec);
     OH_LOG_INFO(LOG_APP, "视频编码准备就绪：%o", ret);
-    // 配置surface
-    ret = OH_VideoEncoder_GetSurface(this->avCodec, &this->nativeWindow);
-    OH_LOG_INFO(LOG_APP, "配置视频窗口：%o", ret);
 }
 
 acgist::TaoyaoVideoEncoder::~TaoyaoVideoEncoder() {
@@ -47,11 +44,6 @@ acgist::TaoyaoVideoEncoder::~TaoyaoVideoEncoder() {
         this->avCodec = nullptr;
         OH_LOG_INFO(LOG_APP, "释放视频编码器：%o", ret);
     }
-    if(this->nativeWindow != nullptr) {
-        OH_NativeWindow_DestroyNativeWindow(this->nativeWindow);
-        this->nativeWindow = nullptr;
-        OH_LOG_INFO(LOG_APP, "释放视频窗口");
-    }
 }
 
 void acgist::TaoyaoVideoEncoder::initFormatConfig(OH_AVFormat* format) {
@@ -123,28 +115,40 @@ void acgist::TaoyaoVideoEncoder::resetDoubleConfig(const char* key, double value
 }
 
 bool acgist::TaoyaoVideoEncoder::start() {
+    if(this->running) {
+        return true;
+    }
+    this->running = true;
     OH_AVErrCode ret = OH_VideoEncoder_Start(this->avCodec);
     OH_LOG_INFO(LOG_APP, "开始视频编码：%o", ret);
     return ret == OH_AVErrCode::AV_ERR_OK;
 }
 
 bool acgist::TaoyaoVideoEncoder::stop() {
-    OH_AVErrCode ret = OH_VideoEncoder_NotifyEndOfStream(this->avCodec);
+    if(!this->running) {
-    OH_LOG_INFO(LOG_APP, "通知视频编码结束：%o", ret);
+        return true;
-    ret = OH_VideoEncoder_Stop(this->avCodec);
+    }
+    this->running = false;
+    // Surface模式
+    // OH_AVErrCode ret = OH_VideoEncoder_NotifyEndOfStream(this->avCodec);
+    // OH_LOG_INFO(LOG_APP, "通知视频编码结束：%o", ret);
+    OH_AVErrCode ret = OH_VideoEncoder_Stop(this->avCodec);
     OH_LOG_INFO(LOG_APP, "结束视频编码：%o", ret);
     return ret == OH_AVErrCode::AV_ERR_OK;
 }
 
 int32_t acgist::TaoyaoVideoEncoder::Release() {
+    // TODO: 释放资源
     return 0;
 }
 
 int32_t acgist::TaoyaoVideoEncoder::RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback* callback) {
+    this->encodedImageCallback = callback;
     return 0;
 }
 
 void acgist::TaoyaoVideoEncoder::SetRates(const webrtc::VideoEncoder::RateControlParameters& parameters) {
+    // TODO: 动态调整编码
 }
 
 webrtc::VideoEncoder::EncoderInfo acgist::TaoyaoVideoEncoder::GetEncoderInfo() const {
@@ -153,7 +157,16 @@ webrtc::VideoEncoder::EncoderInfo acgist::TaoyaoVideoEncoder::GetEncoderInfo() c
     return info;
 }
 
-int32_t acgist::TaoyaoVideoEncoder::Encode(const webrtc::VideoFrame& frame, const std::vector<webrtc::VideoFrameType>* frame_types) {
+int32_t acgist::TaoyaoVideoEncoder::Encode(const webrtc::VideoFrame& videoFrame, const std::vector<webrtc::VideoFrameType>* frame_types) {
+//     frameSize = videoFrame.width * height * 3 / 2
+    // 配置buffer info信息
+    OH_AVCodecBufferAttr info;
+    info.size   = 0; // TODO
+    info.offset = 0;
+    info.pts    = 0;
+    info.flags  = 0; // TODO frame_types
+    OH_AVErrCode ret = OH_AVBuffer_SetBufferAttr(this->buffer, &info);
+    ret = OH_VideoEncoder_PushInputBuffer(this->avCodec, index);
     return 0;
 }
 
@@ -166,14 +179,38 @@ static void OnStreamChanged(OH_AVCodec* codec, OH_AVFormat* format, void* userDa
 }
 
 static void OnNeedInputBuffer(OH_AVCodec* codec, uint32_t index, OH_AVBuffer* buffer, void* userData) {
-    // 忽略
+    if(userData == nullptr) {
+        OH_VideoEncoder_PushInputBuffer(codec, index);
+        return;
+    }
+    acgist::TaoyaoVideoEncoder* videoEncoder = (acgist::TaoyaoVideoEncoder*) userData;
+    if(videoEncoder->running) {
+        videoEncoder->index  = index;
+        videoEncoder->buffer = buffer;
+    } else {
+        // 写入结束
+        OH_AVCodecBufferAttr info;
+        info.size   = 0;
+        info.offset = 0;
+        info.pts    = 0;
+        info.flags  = AVCODEC_BUFFER_FLAGS_EOS;
+        OH_AVErrCode ret = OH_AVBuffer_SetBufferAttr(buffer, &info);
+        ret = OH_VideoEncoder_PushInputBuffer(codec, index);
+        OH_LOG_INFO(LOG_APP, "通知视频编码结束：%o", ret);
+    }
 }
 
 static void OnNewOutputBuffer(OH_AVCodec* codec, uint32_t index, OH_AVBuffer* buffer, void* userData) {
+    if(userData == nullptr) {
+        OH_VideoEncoder_FreeOutputBuffer(codec, index);
+        return;
+    }
+    acgist::TaoyaoVideoEncoder* videoEncoder = (acgist::TaoyaoVideoEncoder*) userData;
     // TODO: 全局是否性能更好
     OH_AVCodecBufferAttr info;
     OH_AVErrCode ret = OH_AVBuffer_GetBufferAttr(buffer, &info);
     char* data = reinterpret_cast<char*>(OH_AVBuffer_GetAddr(buffer));
+//     videoEncoder->encodedImageCallback
     // TODO: 继续处理
     ret = OH_VideoEncoder_FreeOutputBuffer(codec, index);
 }

taoyao-client-openharmony/taoyao/media/src/main/cpp/media/WebRTC.cpp

@@ -16,8 +16,22 @@
 
 static const std::string h264ProfileLevelId = "42e01f";
 
+acgist::TaoyaoAudioTrackSource::TaoyaoAudioTrackSource() {
+}
+
+acgist::TaoyaoAudioTrackSource::~TaoyaoAudioTrackSource() {
+}
+
+webrtc::MediaSourceInterface::SourceState acgist::TaoyaoAudioTrackSource::state() const {
+    return webrtc::MediaSourceInterface::SourceState::kLive;
+}
+
+bool acgist::TaoyaoAudioTrackSource::remote() const {
+    return false;
+}
+
 void acgist::TaoyaoAudioTrackSource::OnData(const void* audio_data, int bits_per_sample, int sample_rate, size_t number_of_channels, size_t number_of_frames) {
-    // TODO: 数据转发
+    // TODO: 转发媒体
 }
 
 acgist::TaoyaoVideoTrackSource::TaoyaoVideoTrackSource() {
@@ -74,20 +88,18 @@ std::vector<webrtc::SdpVideoFormat> acgist::TaoyaoVideoEncoderFactory::GetSuppor
 std::unique_ptr<webrtc::VideoEncoder> acgist::TaoyaoVideoEncoderFactory::CreateVideoEncoder(const webrtc::SdpVideoFormat& format) {
     OH_LOG_DEBUG(LOG_APP, "返回WebRTC编码器：%s", format.name.data());
     // 硬编
-    // TODO: 大小写
+    if (absl::EqualsIgnoreCase(format.name.data(), "H264") == 0) {
-    if (std::strcmp(format.name.data(), "H264") == 0) {
+        return std::unique_ptr<webrtc::VideoEncoder>(new acgist::TaoyaoVideoEncoder());
-        // return std::unique_ptr<webrtc::VideoEncoder>(new acgist::TaoyaoVideoEncoder());
     }
     // 软便
-    // TODO: 大小写
+    if (absl::EqualsIgnoreCase(format.name.data(), cricket::kVp8CodecName) == 0) {
-    if (std::strcmp(format.name.data(), cricket::kVp8CodecName) == 0) {
         return webrtc::VP8Encoder::Create();
     }
-    if (std::strcmp(format.name.data(), cricket::kVp9CodecName) == 0) {
+    if (absl::EqualsIgnoreCase(format.name.data(), cricket::kVp9CodecName) == 0) {
         // return webrtc::VP9Encoder::Create();
         return webrtc::VP9Encoder::Create(cricket::CreateVideoCodec(format));
     }
-    if (std::strcmp(format.name.data(), cricket::kH264CodecName) == 0) {
+    if (absl::EqualsIgnoreCase(format.name.data(), cricket::kH264CodecName) == 0) {
         // return webrtc::H264Encoder::Create();
         return webrtc::H264Encoder::Create(cricket::CreateVideoCodec(format));
     }
@@ -122,12 +134,10 @@ std::vector<webrtc::SdpVideoFormat> acgist::TaoyaoVideoDecoderFactory::GetSuppor
 std::unique_ptr<webrtc::VideoDecoder> acgist::TaoyaoVideoDecoderFactory::CreateVideoDecoder(const webrtc::SdpVideoFormat& format) {
     OH_LOG_DEBUG(LOG_APP, "返回WebRTC解码器：%s", format.name.data());
     // 硬解
-    // TODO: 大小写
     if (absl::EqualsIgnoreCase(format.name.data(), "H264") == 0) {
-        // return std::unique_ptr<webrtc::VideoDecoder>(new acgist::TaoyaoVideoDecoder());
+        return std::unique_ptr<webrtc::VideoDecoder>(new acgist::TaoyaoVideoDecoder());
     }
     // 软解
-    // TODO: 大小写
     if (absl::EqualsIgnoreCase(format.name.data(), cricket::kVp8CodecName) == 0) {
         return webrtc::VP8Decoder::Create();
     }