From 0d446646d9d9943e5044e679a8c75290b9c9044d Mon Sep 17 00:00:00 2001 From: acgist <289547414@qq.com> Date: Sat, 11 May 2024 14:58:32 +0800 Subject: [PATCH] =?UTF-8?q?[*]=20=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../taoyao/media/audio/MixerProcesser.java | 2 +- .../taoyao/media/src/main/cpp/CMakeLists.txt | 1 + .../taoyao/media/src/main/cpp/bind.cpp | 18 ++-- .../media/src/main/cpp/include/Capturer.hpp | 14 ++- .../src/main/cpp/include/MediaManager.hpp | 4 + .../media/src/main/cpp/include/Player.hpp | 12 ++- .../media/src/main/cpp/include/WebRTC.hpp | 24 ++++- .../src/main/cpp/media/AudioCapturer.cpp | 37 ++++--- .../media/src/main/cpp/media/AudioPlayer.cpp | 8 -- .../media/src/main/cpp/media/MediaManager.cpp | 102 ++++++++++++------ .../media/src/main/cpp/media/RemoteClient.cpp | 10 +- .../taoyao/media/src/main/cpp/media/Room.cpp | 30 +++--- .../media/src/main/cpp/media/VideoEncoder.cpp | 63 ++++++++--- .../media/src/main/cpp/media/WebRTC.cpp | 32 ++++-- 14 files changed, 239 insertions(+), 118 deletions(-) diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/MixerProcesser.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/MixerProcesser.java index 899913b..078bf51 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/MixerProcesser.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/MixerProcesser.java @@ -23,7 +23,7 @@ import java.util.concurrent.TimeUnit; * WebRtcAudioTrack#AudioTrackThread ：远程音频 * WebRtcAudioRecord#AudioRecordThread：本地音频 * - * AudioFormat.ENCODING_PCM_16BIT = 2KB + * AudioFormat.ENCODING_PCM_16BIT = 2Byte * * PCM时间计算：1_000_000 microseconds / 48000 hz / 2 bytes * diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/CMakeLists.txt b/taoyao-client-openharmony/taoyao/media/src/main/cpp/CMakeLists.txt index f4f8a30..6d03422 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/CMakeLists.txt +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/CMakeLists.txt @@ -22,6 +22,7 @@ set(LIBWEBRTC_BINARY_PATH "${CMAKE_CURRENT_SOURCE_DIR}/deps/webrtc/lib/${OHOS_A set(LIBWEBRTC_INCLUDE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/deps/webrtc/src/") ADD_DEFINITIONS(-DNDEBUG) +ADD_DEFINITIONS(-DLOG_TAG="libtaoyao") ADD_DEFINITIONS(-DVK_USE_PLATFORM_OHOS=1) add_subdirectory("./deps/libmediasoupclient") diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/bind.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/bind.cpp index d67ec5b..655b2a7 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/bind.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/bind.cpp @@ -25,8 +25,8 @@ #include #include -static std::mutex roomMutex; -static std::mutex taoyaoMutex; +static std::recursive_mutex roomMutex; +static std::recursive_mutex taoyaoMutex; #ifndef TAOYAO_JSON_SIZE #define TAOYAO_JSON_SIZE 2048 @@ -74,11 +74,11 @@ static std::string clientId = ""; // 终端名称 static std::string name = ""; // ETS环境 -static napi_env env = nullptr; +static napi_env env = nullptr; // 是否加载 static bool initTaoyao = false; // PUSH方法引用 -static napi_ref pushRef = nullptr; +static napi_ref pushRef = nullptr; // REQUEST方法引用 static napi_ref requestRef = nullptr; // 媒体功能 @@ -153,7 +153,7 @@ std::string request(const std::string& signal, const std::string& body, uint64_t static napi_value init(napi_env env, napi_callback_info info) { TAOYAO_JSON_BODY(3); { - std::lock_guard taoyaoLock(taoyaoMutex); + std::lock_guard taoyaoLock(taoyaoMutex); if(initTaoyao) { napi_create_int32(env, -1, &ret); OH_LOG_WARN(LOG_APP, "libtaoyao已经加载"); @@ -183,8 +183,8 @@ static napi_value init(napi_env env, napi_callback_info info) { static napi_value shutdown(napi_env env, napi_callback_info info) { napi_value ret; { - std::lock_guard taoyaoLock(taoyaoMutex); - if(!initTaoyao) { + std::lock_guard taoyaoLock(taoyaoMutex); + if (!initTaoyao) { napi_create_int32(env, -1, &ret); OH_LOG_INFO(LOG_APP, "libtaoyao已经卸载"); return ret; @@ -221,7 +221,7 @@ static napi_value shutdown(napi_env env, napi_callback_info info) { static napi_value roomClose(napi_env env, napi_callback_info info) { TAOYAO_JSON_BODY(1); { - std::lock_guard roomLock(roomMutex); + std::lock_guard roomLock(roomMutex); std::string roomId = body["roomId"]; auto iterator = acgist::roomMap.find(roomId); if(iterator == acgist::roomMap.end()) { @@ -285,7 +285,7 @@ static napi_value roomExpel(napi_env env, napi_callback_info info) { static napi_value roomInvite(napi_env env, napi_callback_info info) { TAOYAO_JSON_BODY(1); { - std::lock_guard roomLock(roomMutex); + std::lock_guard roomLock(roomMutex); // TODO: 试试引用 std::string roomId = body["roomId"]; std::string password = body["password"]; diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Capturer.hpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Capturer.hpp index 01e54c9..9e8a63c 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Capturer.hpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Capturer.hpp @@ -1,6 +1,8 @@ /** * 采集器 * + * 音频默认使用OHOS实现：modules/audio_device/ohos/audio_device_template.h + * * @author acgist * * https://docs.openharmony.cn/pages/v4.1/zh-cn/application-dev/media/camera/camera-overview.md @@ -18,11 +20,16 @@ #define TAOYAO_CAPTURER_HPP // OpenGL ES || VULKAN -#define __VULKAN__ true -#ifndef __VULKAN__ -#define __OPENGL__ true +#define __TAOYAO_VULKAN__ true +#ifndef __TAOYAO_VULKAN__ +#define __TAOYAO_OPENGL__ true #endif +// 本地音频采集 +#define __TAOYAO_AUDIO_LOCAL__ false +// 本地视频采集 +#define __TAOYAO_VIDEO_LOCAL__ true + #include #include @@ -59,6 +66,7 @@ template class Capturer { protected: + // 是否运行 bool running = false; public: diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/MediaManager.hpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/MediaManager.hpp index 5f0c2b3..adc1fe7 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/MediaManager.hpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/MediaManager.hpp @@ -28,7 +28,11 @@ public: public: int localClientRef = 0; + #if __TAOYAO_AUDIO_LOCAL__ + acgist::TaoyaoAudioTrackSource* audioTrackSource = nullptr; + #else rtc::scoped_refptr audioTrackSource = nullptr; + #endif acgist::TaoyaoVideoTrackSource* videoTrackSource = nullptr; acgist::AudioCapturer* audioCapturer = nullptr; acgist::VideoCapturer* videoCapturer = nullptr; diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Player.hpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Player.hpp index 5f63474..0e5a759 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Player.hpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Player.hpp @@ -41,8 +41,10 @@ public: /** * 音频播放器 + * + * 默认交给音频设备播放 */ -class AudioPlayer: public Player { +class AudioPlayer : public Player { public: // 音频构造器 @@ -52,7 +54,7 @@ public: public: AudioPlayer(); - virtual ~AudioPlayer(); + virtual ~AudioPlayer() override; public: virtual bool start() override; @@ -63,13 +65,13 @@ public: /** * 视频播放器 * - * TODO: 实现留给有缘人了~.~! + * TODO: 如果需要自行实现 */ -class VideoPlayer: public Player { +class VideoPlayer : public Player { public: VideoPlayer(); - virtual ~VideoPlayer(); + virtual ~VideoPlayer() override; public: virtual bool start() override; diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/WebRTC.hpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/WebRTC.hpp index 6a442a3..7fbb40c 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/WebRTC.hpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/WebRTC.hpp @@ -43,10 +43,18 @@ namespace acgist { /** * 音频轨道来源 + * + * TODO: 媒体转发 */ -class TaoyaoAudioTrackSource : public webrtc::AudioTrackSinkInterface { +class TaoyaoAudioTrackSource : public webrtc::AudioTrackSinkInterface, public webrtc::Notifier { public: + TaoyaoAudioTrackSource(); + virtual ~TaoyaoAudioTrackSource(); + +public: + virtual webrtc::MediaSourceInterface::SourceState state() const override; + virtual bool remote() const override; virtual void OnData(const void* audio_data, int bits_per_sample, int sample_rate, size_t number_of_channels, size_t number_of_frames) override; }; @@ -55,7 +63,7 @@ public: * 视频管道 */ class VideoTrackSinkInterface { - + public: virtual void OnData(const webrtc::VideoFrame& videoFrame) = 0; @@ -85,10 +93,16 @@ public: class TaoyaoVideoEncoder : public webrtc::VideoEncoder { public: + // 是否运行 + bool running = false; // 视频编码器 OH_AVCodec* avCodec = nullptr; - // 视频窗口 - OHNativeWindow* nativeWindow = nullptr; + // 缓冲数据索引 + uint32_t index = 0; + // 缓冲数据 + OH_AVBuffer* buffer = nullptr; + // 编码回调 + webrtc::EncodedImageCallback* encodedImageCallback = nullptr; public: TaoyaoVideoEncoder(); @@ -115,7 +129,7 @@ public: virtual int32_t RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback* callback) override; virtual void SetRates(const webrtc::VideoEncoder::RateControlParameters& parameters) override; virtual webrtc::VideoEncoder::EncoderInfo GetEncoderInfo() const override; - virtual int32_t Encode(const webrtc::VideoFrame& frame, const std::vector* frame_types) override; + virtual int32_t Encode(const webrtc::VideoFrame& videoFrame, const std::vector* frame_types) override; }; diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioCapturer.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioCapturer.cpp index e19c7ac..653d77f 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioCapturer.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioCapturer.cpp @@ -4,8 +4,12 @@ */ #include "../include/Capturer.hpp" +#include + #include +static std::recursive_mutex audioMutex; + // 采集回调 static int32_t OnError(OH_AudioCapturer* capturer, void* userData, OH_AudioStream_Result error); static int32_t OnReadData(OH_AudioCapturer* capturer, void* userData, void* buffer, int32_t length); @@ -14,21 +18,21 @@ static int32_t OnInterruptEvent(OH_AudioCapturer* capturer, void* userData, OH_A acgist::AudioCapturer::AudioCapturer() { OH_AudioStream_Result ret = OH_AudioStreamBuilder_Create(&this->builder, AUDIOSTREAM_TYPE_RENDERER); - OH_LOG_INFO(LOG_APP, "构造音频构造器：%o", ret); - // 配置音频录制参数 + OH_LOG_INFO(LOG_APP, "配置音频构造器：%o", ret); + // 配置音频采集参数 OH_AudioStreamBuilder_SetSamplingRate(this->builder, acgist::samplingRate); OH_AudioStreamBuilder_SetChannelCount(this->builder, acgist::channelCount); OH_AudioStreamBuilder_SetLatencyMode(this->builder, OH_AudioStream_LatencyMode::AUDIOSTREAM_LATENCY_MODE_NORMAL); OH_AudioStreamBuilder_SetSampleFormat(this->builder, OH_AudioStream_SampleFormat::AUDIOSTREAM_SAMPLE_S16LE); - OH_LOG_DEBUG(LOG_APP, "配置音频录制参数：%d %d", acgist::samplingRate, acgist::channelCount); - // 设置回调函数 + OH_LOG_DEBUG(LOG_APP, "配置音频采集参数：%d %d", acgist::samplingRate, acgist::channelCount); + // 设置采集回调 OH_AudioCapturer_Callbacks callbacks; callbacks.OH_AudioCapturer_OnError = OnError; callbacks.OH_AudioCapturer_OnReadData = OnReadData; callbacks.OH_AudioCapturer_OnStreamEvent = OnStreamEvent; callbacks.OH_AudioCapturer_OnInterruptEvent = OnInterruptEvent; ret = OH_AudioStreamBuilder_SetCapturerCallback(this->builder, callbacks, this); - OH_LOG_DEBUG(LOG_APP, "设置录制回调函数：%o", ret); + OH_LOG_DEBUG(LOG_APP, "设置音频采集回调：%o", ret); } acgist::AudioCapturer::~AudioCapturer() { @@ -36,18 +40,19 @@ acgist::AudioCapturer::~AudioCapturer() { if(this->builder != nullptr) { OH_AudioStream_Result ret = OH_AudioStreamBuilder_Destroy(this->builder); this->builder = nullptr; - OH_LOG_INFO(LOG_APP, "释放音频采集：%o", ret); + OH_LOG_INFO(LOG_APP, "释放音频构造器：%o", ret); } } bool acgist::AudioCapturer::start() { + std::lock_guard audioLock(audioMutex); if(this->running) { return true; } this->running = true; - // 构造音频采集器 + // 配置音频采集器 OH_AudioStream_Result ret = OH_AudioStreamBuilder_GenerateCapturer(this->builder, &this->audioCapturer); - OH_LOG_DEBUG(LOG_APP, "构造音频采集器：%o", ret); + OH_LOG_DEBUG(LOG_APP, "配置音频采集器：%o", ret); // 开始音频采集 ret = OH_AudioCapturer_Start(this->audioCapturer); OH_LOG_DEBUG(LOG_APP, "开始音频采集：%o", ret); @@ -55,13 +60,11 @@ bool acgist::AudioCapturer::start() { } bool acgist::AudioCapturer::stop() { + std::lock_guard audioLock(audioMutex); if(!this->running) { return true; } this->running = false; - if(this->audioCapturer == nullptr) { - return true; - } // 停止音频采集 OH_AudioStream_Result ret = OH_AudioCapturer_Stop(this->audioCapturer); OH_LOG_DEBUG(LOG_APP, "停止音频采集：%o", ret); @@ -78,8 +81,16 @@ static int32_t OnError(OH_AudioCapturer* capturer, void* userData, OH_AudioStrea } static int32_t OnReadData(OH_AudioCapturer* capturer, void* userData, void* buffer, int32_t length) { + if(userData == nullptr) { + return -1; + } acgist::AudioCapturer* audioCapturer = (acgist::AudioCapturer*) userData; - audioCapturer->source->OnData(buffer, acgist::bitsPerSample, acgist::samplingRate, acgist::channelCount, sizeof(buffer) / 2); + if(audioCapturer->source == nullptr) { + return -2; + } + // 单声道 48000 / 1000 * 10 * 2(16bit) + // 双声道 48000 / 1000 * 10 * 2(16bit) * 2 + audioCapturer->source->OnData(buffer, acgist::bitsPerSample, acgist::samplingRate, acgist::channelCount, length / 2); return 0; } @@ -89,6 +100,6 @@ static int32_t OnStreamEvent(OH_AudioCapturer* capturer, void* userData, OH_Audi } static int32_t OnInterruptEvent(OH_AudioCapturer* capturer, void* userData, OH_AudioInterrupt_ForceType type, OH_AudioInterrupt_Hint hint) { - OH_LOG_DEBUG(LOG_APP, "音频采集打断：%o %o", type, hint); + OH_LOG_DEBUG(LOG_APP, "打断音频采集：%o %o", type, hint); return 0; } diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioPlayer.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioPlayer.cpp index f1c55ca..84886bf 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioPlayer.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioPlayer.cpp @@ -1,15 +1,7 @@ -/** - * 音频播放不用实现（系统已经实现） - * 这里只是用来学习使用 - */ - #include "../include/Player.hpp" #include -#include -#include - // 播放回调 static int32_t OnError(OH_AudioRenderer* renderer, void* userData, OH_AudioStream_Result error); static int32_t OnWriteData(OH_AudioRenderer* renderer, void* userData, void* buffer, int32_t length); diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/MediaManager.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/MediaManager.cpp index fe93f57..0c668cf 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/MediaManager.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/MediaManager.cpp @@ -14,7 +14,7 @@ #include #include -static std::mutex mediaMutex; +static std::recursive_mutex mediaMutex; acgist::MediaManager::MediaManager() { } @@ -81,19 +81,20 @@ bool acgist::MediaManager::releasePeerConnectionFactory() { int acgist::MediaManager::newLocalClient() { { - std::lock_guard mediaLock(mediaMutex); + std::lock_guard mediaLock(mediaMutex); this->localClientRef++; if(this->localClientRef > 0) { this->newPeerConnectionFactory(); this->startCapture(); } } + OH_LOG_WARN(LOG_APP, "打开本地终端：%d", this->localClientRef); return this->localClientRef; } int acgist::MediaManager::releaseLocalClient() { { - std::lock_guard mediaLock(mediaMutex); + std::lock_guard mediaLock(mediaMutex); this->localClientRef--; if(this->localClientRef <= 0) { if(this->localClientRef < 0) { @@ -104,6 +105,7 @@ int acgist::MediaManager::releaseLocalClient() { } } } + OH_LOG_WARN(LOG_APP, "关闭本地终端：%d", this->localClientRef); return this->localClientRef; } @@ -114,20 +116,42 @@ bool acgist::MediaManager::startCapture() { } bool acgist::MediaManager::startAudioCapture() { - if(this->audioCapturer != nullptr) { - return true; - } - this->audioCapturer = new acgist::AudioCapturer(); - this->audioCapturer->start(); + #if __TAOYAO_AUDIO_LOCAL__ + if (this->audioCapturer == nullptr) { + OH_LOG_WARN(LOG_APP, "开始音频采集"); + this->audioCapturer = new acgist::AudioCapturer(); + this->audioCapturer->start(); + } + if(this->audioTrackSource == nullptr) { + OH_LOG_WARN(LOG_APP, "设置音频来源"); + this->audioTrackSource = new rtc::RefCountedObject(); + this->audioCapturer->source = this->audioTrackSource; + } + #else + if(this->audioTrackSource == nullptr) { + OH_LOG_WARN(LOG_APP, "设置音频来源"); + cricket::AudioOptions options; + options.highpass_filter = true; + options.auto_gain_control = true; + options.echo_cancellation = true; + options.noise_suppression = true; + this->audioTrackSource = this->peerConnectionFactory->CreateAudioSource(options); + } + #endif return true; } bool acgist::MediaManager::startVideoCapture() { - if(this->videoCapturer != nullptr) { - return true; + if(this->videoCapturer == nullptr) { + OH_LOG_WARN(LOG_APP, "开始视频采集"); + this->videoCapturer = new acgist::VideoCapturer(); + this->videoCapturer->start(); + } + if(this->videoTrackSource == nullptr) { + OH_LOG_WARN(LOG_APP, "设置视频来源"); + this->videoTrackSource = new rtc::RefCountedObject(); + this->videoCapturer->source = this->videoTrackSource; } - this->videoCapturer = new acgist::VideoCapturer(); - this->videoCapturer->start(); return true; } @@ -138,36 +162,54 @@ bool acgist::MediaManager::stopCapture() { } bool acgist::MediaManager::stopAudioCapture() { - if(this->audioCapturer == nullptr) { - return true; + #if __TAOYAO_AUDIO_LOCAL__ + if(this->audioCapturer != nullptr) { + OH_LOG_WARN(LOG_APP, "停止音频采集"); + this->audioCapturer->stop(); + delete this->audioCapturer; + this->audioCapturer = nullptr; } - this->audioCapturer->stop(); - delete this->audioCapturer; - this->audioCapturer = nullptr; + if(this->audioTrackSource != nullptr) { + OH_LOG_WARN(LOG_APP, "释放音频来源"); + this->audioTrackSource->Release(); + // delete this->AudioTrackSource; + this->audioTrackSource = nullptr; + } + #else + if(this->audioTrackSource != nullptr) { + OH_LOG_WARN(LOG_APP, "释放音频来源"); + this->audioTrackSource->Release(); + // delete this->audioTrackSource; + this->audioTrackSource = nullptr; + } + #endif return true; } bool acgist::MediaManager::stopVideoCapture() { - if(this->videoCapturer == nullptr) { - return true; + if(this->videoCapturer != nullptr) { + OH_LOG_WARN(LOG_APP, "停止视频采集"); + this->videoCapturer->stop(); + delete this->videoCapturer; + this->videoCapturer = nullptr; + } + if(this->videoTrackSource != nullptr) { + OH_LOG_WARN(LOG_APP, "释放视频来源"); + this->videoTrackSource->Release(); + // delete this->videoTrackSource; + this->videoTrackSource = nullptr; } - this->videoCapturer->stop(); - delete this->videoCapturer; - this->videoCapturer = nullptr; return true; } rtc::scoped_refptr acgist::MediaManager::getAudioTrack() { - cricket::AudioOptions options; - options.highpass_filter = true; - options.auto_gain_control = true; - options.echo_cancellation = true; - options.noise_suppression = true; - this->audioTrackSource = this->peerConnectionFactory->CreateAudioSource(options); - return this->peerConnectionFactory->CreateAudioTrack("taoyao-audio", audioTrackSource.get()); + #if __TAOYAO_AUDIO_LOCAL__ + return this->peerConnectionFactory->CreateAudioTrack("taoyao-audio", this->audioTrackSource); + #else + return this->peerConnectionFactory->CreateAudioTrack("taoyao-audio", this->audioTrackSource.get()); + #endif } rtc::scoped_refptr acgist::MediaManager::getVideoTrack() { - this->videoTrackSource = new rtc::RefCountedObject(); return this->peerConnectionFactory->CreateVideoTrack("taoyao-video", this->videoTrackSource); } diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/RemoteClient.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/RemoteClient.cpp index 28e23fd..a21d4cf 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/RemoteClient.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/RemoteClient.cpp @@ -4,7 +4,7 @@ #include "hilog/log.h" -static std::mutex clientMutex; +static std::recursive_mutex clientMutex; acgist::RemoteClient::RemoteClient(acgist::MediaManager* mediaManager) : RoomClient(mediaManager) {} @@ -18,7 +18,7 @@ acgist::RemoteClient::~RemoteClient() { } bool acgist::RemoteClient::addConsumer(const std::string& consumerId, mediasoupclient::Consumer* consumer) { - std::lock_guard clientLock(clientMutex); + std::lock_guard clientLock(clientMutex); auto oldConsumer = this->consumers.find(clientId); if(oldConsumer != this->consumers.end()) { OH_LOG_INFO(LOG_APP, "关闭旧的消费者：%s", consumerId.data()); @@ -35,7 +35,7 @@ bool acgist::RemoteClient::addConsumer(const std::string& consumerId, mediasoupc } bool acgist::RemoteClient::closeConsumer(const std::string& consumerId) { - std::lock_guard clientLock(clientMutex); + std::lock_guard clientLock(clientMutex); auto consumer = this->consumers.find(consumerId); if(consumer == this->consumers.end()) { OH_LOG_INFO(LOG_APP, "消费者已经关闭：%s", consumerId.data()); @@ -50,7 +50,7 @@ bool acgist::RemoteClient::closeConsumer(const std::string& consumerId) { } bool acgist::RemoteClient::pauseConsumer(const std::string& consumerId) { - std::lock_guard clientLock(clientMutex); + std::lock_guard clientLock(clientMutex); auto consumer = this->consumers.find(consumerId); if(consumer == this->consumers.end()) { OH_LOG_INFO(LOG_APP, "无效消费者：%s", consumerId.data()); @@ -62,7 +62,7 @@ bool acgist::RemoteClient::pauseConsumer(const std::string& consumerId) { } bool acgist::RemoteClient::resumeConsumer(const std::string& consumerId) { - std::lock_guard clientLock(clientMutex); + std::lock_guard clientLock(clientMutex); auto consumer = this->consumers.find(consumerId); if(consumer == this->consumers.end()) { OH_LOG_INFO(LOG_APP, "无效消费者：%s", consumerId.data()); diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/Room.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/Room.cpp index 6a60e62..d54e4f4 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/Room.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/Room.cpp @@ -4,7 +4,7 @@ #include "hilog/log.h" -static std::mutex roomMutex; +static std::recursive_mutex roomMutex; acgist::Room::Room(const std::string& roomId, acgist::MediaManager* mediaManager) : roomId(roomId), mediaManager(mediaManager) { this->device = new mediasoupclient::Device(); @@ -243,7 +243,7 @@ int acgist::Room::close() { } int acgist::Room::closeClient() { - std::lock_guard lockRoom(roomMutex); + std::lock_guard lockRoom(roomMutex); OH_LOG_INFO(LOG_APP, "关闭本地终端：%s", this->roomId.data()); if(this->client != nullptr) { delete this->client; @@ -253,7 +253,7 @@ int acgist::Room::closeClient() { } int acgist::Room::closeClients() { - std::lock_guard lockRoom(roomMutex); + std::lock_guard lockRoom(roomMutex); OH_LOG_INFO(LOG_APP, "关闭远程终端：%s", this->roomId.data()); for (auto iterator = this->clients.begin(); iterator != this->clients.end(); ++iterator) { if (iterator->second == nullptr) { @@ -268,7 +268,7 @@ int acgist::Room::closeClients() { } int acgist::Room::closeAudioProducer() { - std::lock_guard lockRoom(roomMutex); + std::lock_guard lockRoom(roomMutex); OH_LOG_INFO(LOG_APP, "关闭音频生产者：%s", this->roomId.data()); if (this->audioProducer != nullptr) { this->audioProducer->Close(); @@ -277,7 +277,7 @@ int acgist::Room::closeAudioProducer() { } int acgist::Room::closeVideoProducer() { - std::lock_guard lockRoom(roomMutex); + std::lock_guard lockRoom(roomMutex); OH_LOG_INFO(LOG_APP, "关闭视频生产者：%s", this->roomId.data()); if (this->videoProducer != nullptr) { this->videoProducer->Close(); @@ -286,7 +286,7 @@ int acgist::Room::closeVideoProducer() { } int acgist::Room::closeTransport() { - std::lock_guard lockRoom(roomMutex); + std::lock_guard lockRoom(roomMutex); OH_LOG_INFO(LOG_APP, "关闭通道：%s", this->roomId.data()); if (this->sendTransport != nullptr) { this->sendTransport->Close(); @@ -298,7 +298,7 @@ int acgist::Room::closeTransport() { } int acgist::Room::newRemoteClient(const std::string& clientId, const std::string& name) { - std::lock_guard lockRoom(roomMutex); + std::lock_guard lockRoom(roomMutex); auto oldClient = this->clients.find(clientId); if(oldClient != this->clients.end()) { OH_LOG_INFO(LOG_APP, "已经存在远程终端：%s", clientId.data()); @@ -315,7 +315,7 @@ int acgist::Room::newRemoteClient(const std::string& clientId, const std::string } int acgist::Room::closeRemoteClient(const std::string& clientId) { - std::lock_guard lockRoom(roomMutex); + std::lock_guard lockRoom(roomMutex); auto client = this->clients.find(clientId); if(client == this->clients.end()) { OH_LOG_INFO(LOG_APP, "远程终端已经删除：%s", clientId.data()); @@ -330,7 +330,7 @@ int acgist::Room::closeRemoteClient(const std::string& clientId) { } int acgist::Room::newConsumer(nlohmann::json& body) { - std::lock_guard lockRoom(roomMutex); + std::lock_guard lockRoom(roomMutex); mediasoupclient::Consumer* consumer = this->recvTransport->Consume( this->consumerListener, body["consumerId"], @@ -359,7 +359,7 @@ int acgist::Room::newConsumer(nlohmann::json& body) { } int acgist::Room::closeConsumer(const std::string& consumerId) { - std::lock_guard lockRoom(roomMutex); + std::lock_guard lockRoom(roomMutex); auto clientId = this->consumerIdClientId.find(consumerId); if(clientId == this->consumerIdClientId.end()) { OH_LOG_INFO(LOG_APP, "关闭消费者无效：%s", consumerId.data()); @@ -375,7 +375,7 @@ int acgist::Room::closeConsumer(const std::string& consumerId) { } int acgist::Room::pauseConsumer(const std::string& consumerId) { - std::lock_guard lockRoom(roomMutex); + std::lock_guard lockRoom(roomMutex); auto clientId = this->consumerIdClientId.find(consumerId); if(clientId == this->consumerIdClientId.end()) { OH_LOG_INFO(LOG_APP, "暂停消费者无效：%s", consumerId.data()); @@ -391,7 +391,7 @@ int acgist::Room::pauseConsumer(const std::string& consumerId) { } int acgist::Room::resumeConsumer(const std::string& consumerId) { - std::lock_guard lockRoom(roomMutex); + std::lock_guard lockRoom(roomMutex); auto clientId = this->consumerIdClientId.find(consumerId); if(clientId == this->consumerIdClientId.end()) { OH_LOG_INFO(LOG_APP, "恢复消费者无效：%s", consumerId.data()); @@ -407,7 +407,7 @@ int acgist::Room::resumeConsumer(const std::string& consumerId) { } int acgist::Room::closeProducer(const std::string& producerId) { - std::lock_guard lockRoom(roomMutex); + std::lock_guard lockRoom(roomMutex); if(this->audioProducer != nullptr && this->audioProducer->GetId() == producerId) { OH_LOG_INFO(LOG_APP, "关闭音频生产者：%s", producerId.data()); this->audioProducer->Close(); @@ -429,7 +429,7 @@ int acgist::Room::closeProducer(const std::string& producerId) { } int acgist::Room::pauseProducer(const std::string& producerId) { - std::lock_guard lockRoom(roomMutex); + std::lock_guard lockRoom(roomMutex); if(this->audioProducer != nullptr && this->audioProducer->GetId() == producerId) { OH_LOG_INFO(LOG_APP, "暂停音频生产者：%s", producerId.data()); this->audioProducer->Pause(); @@ -443,7 +443,7 @@ int acgist::Room::pauseProducer(const std::string& producerId) { } int acgist::Room::resumeProducer(const std::string& producerId) { - std::lock_guard lockRoom(roomMutex); + std::lock_guard lockRoom(roomMutex); if(this->audioProducer != nullptr && this->audioProducer->GetId() == producerId) { OH_LOG_INFO(LOG_APP, "恢复音频生产者：%s", producerId.data()); this->audioProducer->Resume(); diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoEncoder.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoEncoder.cpp index b345a56..d91a0d3 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoEncoder.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoEncoder.cpp @@ -36,9 +36,6 @@ acgist::TaoyaoVideoEncoder::TaoyaoVideoEncoder() { // 准备就绪 ret = OH_VideoEncoder_Prepare(this->avCodec); OH_LOG_INFO(LOG_APP, "视频编码准备就绪：%o", ret); - // 配置surface - ret = OH_VideoEncoder_GetSurface(this->avCodec, &this->nativeWindow); - OH_LOG_INFO(LOG_APP, "配置视频窗口：%o", ret); } acgist::TaoyaoVideoEncoder::~TaoyaoVideoEncoder() { @@ -47,11 +44,6 @@ acgist::TaoyaoVideoEncoder::~TaoyaoVideoEncoder() { this->avCodec = nullptr; OH_LOG_INFO(LOG_APP, "释放视频编码器：%o", ret); } - if(this->nativeWindow != nullptr) { - OH_NativeWindow_DestroyNativeWindow(this->nativeWindow); - this->nativeWindow = nullptr; - OH_LOG_INFO(LOG_APP, "释放视频窗口"); - } } void acgist::TaoyaoVideoEncoder::initFormatConfig(OH_AVFormat* format) { @@ -123,28 +115,40 @@ void acgist::TaoyaoVideoEncoder::resetDoubleConfig(const char* key, double value } bool acgist::TaoyaoVideoEncoder::start() { + if(this->running) { + return true; + } + this->running = true; OH_AVErrCode ret = OH_VideoEncoder_Start(this->avCodec); OH_LOG_INFO(LOG_APP, "开始视频编码：%o", ret); return ret == OH_AVErrCode::AV_ERR_OK; } bool acgist::TaoyaoVideoEncoder::stop() { - OH_AVErrCode ret = OH_VideoEncoder_NotifyEndOfStream(this->avCodec); - OH_LOG_INFO(LOG_APP, "通知视频编码结束：%o", ret); - ret = OH_VideoEncoder_Stop(this->avCodec); + if(!this->running) { + return true; + } + this->running = false; + // Surface模式 + // OH_AVErrCode ret = OH_VideoEncoder_NotifyEndOfStream(this->avCodec); + // OH_LOG_INFO(LOG_APP, "通知视频编码结束：%o", ret); + OH_AVErrCode ret = OH_VideoEncoder_Stop(this->avCodec); OH_LOG_INFO(LOG_APP, "结束视频编码：%o", ret); return ret == OH_AVErrCode::AV_ERR_OK; } int32_t acgist::TaoyaoVideoEncoder::Release() { + // TODO: 释放资源 return 0; } int32_t acgist::TaoyaoVideoEncoder::RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback* callback) { + this->encodedImageCallback = callback; return 0; } void acgist::TaoyaoVideoEncoder::SetRates(const webrtc::VideoEncoder::RateControlParameters& parameters) { + // TODO: 动态调整编码 } webrtc::VideoEncoder::EncoderInfo acgist::TaoyaoVideoEncoder::GetEncoderInfo() const { @@ -153,7 +157,16 @@ webrtc::VideoEncoder::EncoderInfo acgist::TaoyaoVideoEncoder::GetEncoderInfo() c return info; } -int32_t acgist::TaoyaoVideoEncoder::Encode(const webrtc::VideoFrame& frame, const std::vector* frame_types) { +int32_t acgist::TaoyaoVideoEncoder::Encode(const webrtc::VideoFrame& videoFrame, const std::vector* frame_types) { +// frameSize = videoFrame.width * height * 3 / 2 + // 配置buffer info信息 + OH_AVCodecBufferAttr info; + info.size = 0; // TODO + info.offset = 0; + info.pts = 0; + info.flags = 0; // TODO frame_types + OH_AVErrCode ret = OH_AVBuffer_SetBufferAttr(this->buffer, &info); + ret = OH_VideoEncoder_PushInputBuffer(this->avCodec, index); return 0; } @@ -166,14 +179,38 @@ static void OnStreamChanged(OH_AVCodec* codec, OH_AVFormat* format, void* userDa } static void OnNeedInputBuffer(OH_AVCodec* codec, uint32_t index, OH_AVBuffer* buffer, void* userData) { - // 忽略 + if(userData == nullptr) { + OH_VideoEncoder_PushInputBuffer(codec, index); + return; + } + acgist::TaoyaoVideoEncoder* videoEncoder = (acgist::TaoyaoVideoEncoder*) userData; + if(videoEncoder->running) { + videoEncoder->index = index; + videoEncoder->buffer = buffer; + } else { + // 写入结束 + OH_AVCodecBufferAttr info; + info.size = 0; + info.offset = 0; + info.pts = 0; + info.flags = AVCODEC_BUFFER_FLAGS_EOS; + OH_AVErrCode ret = OH_AVBuffer_SetBufferAttr(buffer, &info); + ret = OH_VideoEncoder_PushInputBuffer(codec, index); + OH_LOG_INFO(LOG_APP, "通知视频编码结束：%o", ret); + } } static void OnNewOutputBuffer(OH_AVCodec* codec, uint32_t index, OH_AVBuffer* buffer, void* userData) { + if(userData == nullptr) { + OH_VideoEncoder_FreeOutputBuffer(codec, index); + return; + } + acgist::TaoyaoVideoEncoder* videoEncoder = (acgist::TaoyaoVideoEncoder*) userData; // TODO: 全局是否性能更好 OH_AVCodecBufferAttr info; OH_AVErrCode ret = OH_AVBuffer_GetBufferAttr(buffer, &info); char* data = reinterpret_cast(OH_AVBuffer_GetAddr(buffer)); +// videoEncoder->encodedImageCallback // TODO: 继续处理 ret = OH_VideoEncoder_FreeOutputBuffer(codec, index); } diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/WebRTC.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/WebRTC.cpp index 359c9e0..e79504e 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/WebRTC.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/WebRTC.cpp @@ -16,8 +16,22 @@ static const std::string h264ProfileLevelId = "42e01f"; +acgist::TaoyaoAudioTrackSource::TaoyaoAudioTrackSource() { +} + +acgist::TaoyaoAudioTrackSource::~TaoyaoAudioTrackSource() { +} + +webrtc::MediaSourceInterface::SourceState acgist::TaoyaoAudioTrackSource::state() const { + return webrtc::MediaSourceInterface::SourceState::kLive; +} + +bool acgist::TaoyaoAudioTrackSource::remote() const { + return false; +} + void acgist::TaoyaoAudioTrackSource::OnData(const void* audio_data, int bits_per_sample, int sample_rate, size_t number_of_channels, size_t number_of_frames) { - // TODO: 数据转发 + // TODO: 转发媒体 } acgist::TaoyaoVideoTrackSource::TaoyaoVideoTrackSource() { @@ -74,20 +88,18 @@ std::vector acgist::TaoyaoVideoEncoderFactory::GetSuppor std::unique_ptr acgist::TaoyaoVideoEncoderFactory::CreateVideoEncoder(const webrtc::SdpVideoFormat& format) { OH_LOG_DEBUG(LOG_APP, "返回WebRTC编码器：%s", format.name.data()); // 硬编 - // TODO: 大小写 - if (std::strcmp(format.name.data(), "H264") == 0) { - // return std::unique_ptr(new acgist::TaoyaoVideoEncoder()); + if (absl::EqualsIgnoreCase(format.name.data(), "H264") == 0) { + return std::unique_ptr(new acgist::TaoyaoVideoEncoder()); } // 软便 - // TODO: 大小写 - if (std::strcmp(format.name.data(), cricket::kVp8CodecName) == 0) { + if (absl::EqualsIgnoreCase(format.name.data(), cricket::kVp8CodecName) == 0) { return webrtc::VP8Encoder::Create(); } - if (std::strcmp(format.name.data(), cricket::kVp9CodecName) == 0) { + if (absl::EqualsIgnoreCase(format.name.data(), cricket::kVp9CodecName) == 0) { // return webrtc::VP9Encoder::Create(); return webrtc::VP9Encoder::Create(cricket::CreateVideoCodec(format)); } - if (std::strcmp(format.name.data(), cricket::kH264CodecName) == 0) { + if (absl::EqualsIgnoreCase(format.name.data(), cricket::kH264CodecName) == 0) { // return webrtc::H264Encoder::Create(); return webrtc::H264Encoder::Create(cricket::CreateVideoCodec(format)); } @@ -122,12 +134,10 @@ std::vector acgist::TaoyaoVideoDecoderFactory::GetSuppor std::unique_ptr acgist::TaoyaoVideoDecoderFactory::CreateVideoDecoder(const webrtc::SdpVideoFormat& format) { OH_LOG_DEBUG(LOG_APP, "返回WebRTC解码器：%s", format.name.data()); // 硬解 - // TODO: 大小写 if (absl::EqualsIgnoreCase(format.name.data(), "H264") == 0) { - // return std::unique_ptr(new acgist::TaoyaoVideoDecoder()); + return std::unique_ptr(new acgist::TaoyaoVideoDecoder()); } // 软解 - // TODO: 大小写 if (absl::EqualsIgnoreCase(format.name.data(), cricket::kVp8CodecName) == 0) { return webrtc::VP8Decoder::Create(); }