diff --git a/taoyao-client-media/.env b/taoyao-client-media/.env index ad21851..1766951 100644 --- a/taoyao-client-media/.env +++ b/taoyao-client-media/.env @@ -25,4 +25,4 @@ MEDIASOUP_LISTEN_IP=0.0.0.0 # Mediasoup监听端口（起始端口） MEDIASOUP_LISTEN_PORT=44444 # Mediasoup声明地址（不能配置环回地址） -MEDIASOUP_ANNOUNCED_IP=192.168.1.239 \ No newline at end of file +MEDIASOUP_ANNOUNCED_IP=192.168.1.100 \ No newline at end of file diff --git a/taoyao-client-openharmony/README.md b/taoyao-client-openharmony/README.md index 0738fa4..e9fc2d8 100644 --- a/taoyao-client-openharmony/README.md +++ b/taoyao-client-openharmony/README.md @@ -47,3 +47,15 @@ ninja -C ./out/arm64-v8a -j 32 * https://gitee.com/openharmony-tpc * https://gitee.com/openharmony-tpc/chromium_third_party_webrtc * https://gitee.com/openharmony-tpc/chromium_third_party_ohos_prebuilts + +## 参考项目 + +* https://gitee.com/han_jin_fei/oh_web-rtc + +## 源码修改 + +``` +vim modules/audio_device/audio_device_impl.cc +vim modules/audio_device/ohos/ohaudio_recorder_wrapper.cc +OH_AudioStreamBuilder_SetLatencyMode(builder, AUDIOSTREAM_LATENCY_MODE_NORMAL); +``` diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/CMakeLists.txt b/taoyao-client-openharmony/taoyao/media/src/main/cpp/CMakeLists.txt index f77aa9a..b4e9466 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/CMakeLists.txt +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/CMakeLists.txt @@ -23,7 +23,6 @@ set(LIBWEBRTC_INCLUDE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/deps/webrtc/src/") #ADD_DEFINITIONS(-DNDEBUG) ADD_DEFINITIONS(-DLOG_TAG="libtaoyao") -ADD_DEFINITIONS(-DVK_USE_PLATFORM_OHOS=1) add_subdirectory("./deps/libmediasoupclient") diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Capturer.hpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Capturer.hpp index 1d1616c..8213d36 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Capturer.hpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Capturer.hpp @@ -19,11 +19,6 @@ #ifndef TAOYAO_CAPTURER_HPP #define TAOYAO_CAPTURER_HPP -// 本地音频采集 -#define __TAOYAO_AUDIO_LOCAL__ false -// 本地视频采集 -#define __TAOYAO_VIDEO_LOCAL__ true - #include #include diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/MediaManager.hpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/MediaManager.hpp index 2ac75d1..9974236 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/MediaManager.hpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/MediaManager.hpp @@ -18,8 +18,14 @@ #include "api/peer_connection_interface.h" -#define TAOYAO_VIDEO_SOURCE_SCREEN true -#define TAOYAO_VIDEO_SOURCE_CAMERA false +// 本地音频采集 +#define TAOYAO_AUDIO_LOCAL false +// 本地视频采集 +#define TAOYAO_VIDEO_LOCAL true +// 视频来源屏幕 +#define TAOYAO_VIDEO_SOURCE_SCREEN false +// 视频来源相机 +#define TAOYAO_VIDEO_SOURCE_CAMERA true namespace acgist { diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioCapturer.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioCapturer.cpp index ab85924..f272c52 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioCapturer.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioCapturer.cpp @@ -1,3 +1,8 @@ +/** + * https://gitee.com/openharmony/multimedia_audio_framework/blob/master/services/audio_service/client/src/fast_audio_stream.cpp + * https://gitee.com/openharmony/multimedia_audio_framework/blob/master/frameworks/js/napi/audiocapturer/napi_audio_capturer.cpp + * https://gitee.com/openharmony/multimedia_audio_framework/blob/master/services/audio_service/client/src/audio_manager_proxy.cpp + */ #include "../include/Capturer.hpp" #include @@ -20,6 +25,7 @@ acgist::AudioCapturer::AudioCapturer() { OH_AudioStreamBuilder_SetChannelCount(this->builder, acgist::channelCount); OH_AudioStreamBuilder_SetLatencyMode(this->builder, OH_AudioStream_LatencyMode::AUDIOSTREAM_LATENCY_MODE_NORMAL); OH_AudioStreamBuilder_SetSampleFormat(this->builder, OH_AudioStream_SampleFormat::AUDIOSTREAM_SAMPLE_S16LE); + // OH_AudioStreamBuilder_SetRendererInfo(this->builder, OH_AudioStream_Usage::AUDIOSTREAM_USAGE_VOICE_COMMUNICATION); OH_LOG_DEBUG(LOG_APP, "配置音频采集参数：%{public}d %{public}d", acgist::samplingRate, acgist::channelCount); // 设置音频采集回调 OH_AudioCapturer_Callbacks callbacks; diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/CameraCapturer.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/CameraCapturer.cpp index 80c95dc..29a88c8 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/CameraCapturer.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/CameraCapturer.cpp @@ -453,6 +453,25 @@ if (eglRet == EGL_FALSE) { // } // // TODO: 释放webrtc // videoFrameBuffer->Release(); + +// height = (height > 0) ? height : -height; // abs +// width = stride; +// size_t size = stride * height + stride * height / 2; +// if (bufferSize < size) { +// return false; +// } +// +// rtc::scoped_refptr i420_buffer = webrtc::I420Buffer::Create(width, height); +// libyuv::NV21ToI420(buffer, width, buffer + width * height, width, i420_buffer.get()->MutableDataY(), +// i420_buffer.get()->StrideY(), i420_buffer.get()->MutableDataU(), i420_buffer.get()->StrideU(), +// i420_buffer.get()->MutableDataV(), i420_buffer.get()->StrideV(), width, height); +// +// webrtc::VideoFrame video_frame = webrtc::VideoFrame::Builder() +// .set_video_frame_buffer(i420_buffer) +// .set_timestamp_rtp(0) +// .set_timestamp_ms(rtc::TimeMillis()) +// .set_rotation(webrtc::kVideoRotation_90) +// .build(); } static bool CheckEglExtension(const char* extensions, const char* extension) { diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/LocalClient.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/LocalClient.cpp index 29a8119..caa8ce5 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/LocalClient.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/LocalClient.cpp @@ -2,7 +2,8 @@ acgist::LocalClient::LocalClient(acgist::MediaManager* mediaManager) : acgist::RoomClient(mediaManager) { this->mediaManager->newLocalClient(); - // this->audioTrack = this->mediaManager->getAudioTrack(); + this->audioTrack = this->mediaManager->getAudioTrack(); + this->audioTrack->set_enabled(true); this->videoTrack = this->mediaManager->getVideoTrack(); } diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/MediaManager.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/MediaManager.cpp index bc4d16d..b89af0d 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/MediaManager.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/MediaManager.cpp @@ -34,9 +34,9 @@ bool acgist::MediaManager::newPeerConnectionFactory() { this->workerThread->SetName("worker_thread", nullptr); this->signalingThread->SetName("signaling_thread", nullptr); if ( - !this->networkThread->Start() || - !this->signalingThread->Start() || - !this->workerThread->Start() + !this->networkThread->Start() || + !this->workerThread->Start() || + !this->signalingThread->Start() ) { OH_LOG_WARN(LOG_APP, "WebRTC线程启动失败"); this->networkThread = nullptr; @@ -44,6 +44,18 @@ bool acgist::MediaManager::newPeerConnectionFactory() { this->signalingThread = nullptr; return false; } + // std::make_unique>() + // std::make_unique>() this->peerConnectionFactory = webrtc::CreatePeerConnectionFactory( // 网络线程 this->networkThread.get(), @@ -109,33 +121,33 @@ int acgist::MediaManager::releaseLocalClient() { } bool acgist::MediaManager::startCapture() { - // this->startAudioCapture(); + this->startAudioCapture(); this->startVideoCapture(); return true; } bool acgist::MediaManager::startAudioCapture() { - #if __TAOYAO_AUDIO_LOCAL__ - if (this->audioCapturer == nullptr) { - OH_LOG_INFO(LOG_APP, "开始音频采集"); - this->audioCapturer = new acgist::AudioCapturer(); - this->audioCapturer->start(); - } - if(this->audioTrackSource == nullptr) { - OH_LOG_INFO(LOG_APP, "设置音频来源"); - this->audioTrackSource = new rtc::RefCountedObject(); - this->audioCapturer->source = this->audioTrackSource; - } + #if TAOYAO_AUDIO_LOCAL + if (this->audioCapturer == nullptr) { + OH_LOG_INFO(LOG_APP, "开始音频采集"); + this->audioCapturer = new acgist::AudioCapturer(); + this->audioCapturer->start(); + } + if(this->audioTrackSource == nullptr) { + OH_LOG_INFO(LOG_APP, "设置音频来源"); + this->audioTrackSource = new rtc::RefCountedObject(); + this->audioCapturer->source = this->audioTrackSource; + } #else - if(this->audioTrackSource == nullptr) { - OH_LOG_INFO(LOG_APP, "设置音频来源"); - cricket::AudioOptions options; -// options.highpass_filter = true; -// options.auto_gain_control = true; -// options.echo_cancellation = true; -// options.noise_suppression = true; - this->audioTrackSource = this->peerConnectionFactory->CreateAudioSource(options); - } + if(this->audioTrackSource == nullptr) { + OH_LOG_INFO(LOG_APP, "设置音频来源"); + cricket::AudioOptions options; +// options.highpass_filter = true; +// options.auto_gain_control = true; +// options.echo_cancellation = true; +// options.noise_suppression = true; + this->audioTrackSource = this->peerConnectionFactory->CreateAudioSource(options); + } #endif return true; } @@ -168,26 +180,26 @@ bool acgist::MediaManager::stopCapture() { } bool acgist::MediaManager::stopAudioCapture() { - #if __TAOYAO_AUDIO_LOCAL__ - if(this->audioCapturer != nullptr) { - OH_LOG_INFO(LOG_APP, "停止音频采集"); - this->audioCapturer->stop(); - delete this->audioCapturer; - this->audioCapturer = nullptr; - } - if(this->audioTrackSource != nullptr) { - OH_LOG_INFO(LOG_APP, "释放音频来源"); - this->audioTrackSource->Release(); - // delete this->AudioTrackSource; - this->audioTrackSource = nullptr; - } + #if TAOYAO_AUDIO_LOCAL + if(this->audioCapturer != nullptr) { + OH_LOG_INFO(LOG_APP, "停止音频采集"); + this->audioCapturer->stop(); + delete this->audioCapturer; + this->audioCapturer = nullptr; + } + if(this->audioTrackSource != nullptr) { + OH_LOG_INFO(LOG_APP, "释放音频来源"); + this->audioTrackSource->Release(); + // delete this->AudioTrackSource; + this->audioTrackSource = nullptr; + } #else - if(this->audioTrackSource != nullptr) { - OH_LOG_INFO(LOG_APP, "释放音频来源"); - this->audioTrackSource->Release(); - // delete this->audioTrackSource; - this->audioTrackSource = nullptr; - } + if(this->audioTrackSource != nullptr) { + OH_LOG_INFO(LOG_APP, "释放音频来源"); + this->audioTrackSource->Release(); + // delete this->audioTrackSource; + this->audioTrackSource = nullptr; + } #endif return true; } @@ -209,10 +221,10 @@ bool acgist::MediaManager::stopVideoCapture() { } rtc::scoped_refptr acgist::MediaManager::getAudioTrack() { - #if __TAOYAO_AUDIO_LOCAL__ - return this->peerConnectionFactory->CreateAudioTrack("taoyao-audio", this->audioTrackSource); + #if TAOYAO_AUDIO_LOCAL + return this->peerConnectionFactory->CreateAudioTrack("audio_label", this->audioTrackSource); #else - return this->peerConnectionFactory->CreateAudioTrack("taoyao-audio", this->audioTrackSource.get()); + return this->peerConnectionFactory->CreateAudioTrack("audio_label", this->audioTrackSource.get()); #endif } diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/Room.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/Room.cpp index a23b0e8..9fa0f1a 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/Room.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/Room.cpp @@ -118,7 +118,7 @@ int acgist::Room::produceMedia() { this->createRecvTransport(); } if(this->audioProduce) { - // this->produceAudio(); + this->produceAudio(); } if(this->videoProduce) { this->produceVideo(); @@ -210,7 +210,7 @@ int acgist::Room::produceAudio() { OH_LOG_INFO(LOG_APP, "音频媒体状态错误：%{public}s", this->roomId.data()); return -2; } - OH_LOG_INFO(LOG_APP, "生产音频媒体：%s", this->roomId.data()); + OH_LOG_INFO(LOG_APP, "生产音频媒体：%{public}s", this->roomId.data()); nlohmann::json codecOptions = { { "opusStereo", true }, { "opusDtx", true } @@ -228,18 +228,18 @@ int acgist::Room::produceAudio() { int acgist::Room::produceVideo() { std::lock_guard lockRoom(roomMutex); if(this->videoProducer != nullptr) { - OH_LOG_INFO(LOG_APP, "视频媒体已经生产：%s", this->roomId.data()); + OH_LOG_INFO(LOG_APP, "视频媒体已经生产：%{public}s", this->roomId.data()); return -1; } if(!this->device->CanProduce("video")) { - OH_LOG_INFO(LOG_APP, "不能生产视频媒体：%s", this->roomId.data()); + OH_LOG_INFO(LOG_APP, "不能生产视频媒体：%{public}s", this->roomId.data()); return -1; } if(this->client->videoTrack->state() == webrtc::MediaStreamTrackInterface::TrackState::kEnded) { - OH_LOG_INFO(LOG_APP, "视频媒体状态错误：%s", this->roomId.data()); + OH_LOG_INFO(LOG_APP, "视频媒体状态错误：%{public}s", this->roomId.data()); return -2; } - OH_LOG_INFO(LOG_APP, "生产视频媒体：%s", this->roomId.data()); + OH_LOG_INFO(LOG_APP, "生产视频媒体：%{public}s", this->roomId.data()); nlohmann::json codecOptions = { // x-google-start-bitrate { "videoGoogleStartBitrate", 1200 }, @@ -278,7 +278,7 @@ int acgist::Room::close() { return 0; } this->closed = true; - OH_LOG_INFO(LOG_APP, "关闭房间：%s", this->roomId.data()); + OH_LOG_INFO(LOG_APP, "关闭房间：%{public}s", this->roomId.data()); this->closeClient(); this->closeClients(); this->closeAudioProducer(); @@ -289,7 +289,7 @@ int acgist::Room::close() { int acgist::Room::closeClient() { std::lock_guard lockRoom(roomMutex); - OH_LOG_INFO(LOG_APP, "关闭本地终端：%s", this->roomId.data()); + OH_LOG_INFO(LOG_APP, "关闭本地终端：%{public}s", this->roomId.data()); if(this->client != nullptr) { delete this->client; this->client = nullptr; @@ -299,12 +299,12 @@ int acgist::Room::closeClient() { int acgist::Room::closeClients() { std::lock_guard lockRoom(roomMutex); - OH_LOG_INFO(LOG_APP, "关闭远程终端：%s", this->roomId.data()); + OH_LOG_INFO(LOG_APP, "关闭远程终端：%{public}s", this->roomId.data()); for (auto iterator = this->clients.begin(); iterator != this->clients.end(); ++iterator) { if (iterator->second == nullptr) { continue; } - OH_LOG_INFO(LOG_APP, "关闭消费者：%s", iterator->first.data()); + OH_LOG_INFO(LOG_APP, "关闭消费者：%{public}s", iterator->first.data()); delete iterator->second; iterator->second = nullptr; } @@ -315,7 +315,7 @@ int acgist::Room::closeClients() { int acgist::Room::closeAudioProducer() { std::lock_guard lockRoom(roomMutex); - OH_LOG_INFO(LOG_APP, "关闭音频生产者：%s", this->roomId.data()); + OH_LOG_INFO(LOG_APP, "关闭音频生产者：%{public}s", this->roomId.data()); if (this->audioProducer != nullptr) { this->audioProducer->Close(); delete this->audioProducer; @@ -326,7 +326,7 @@ int acgist::Room::closeAudioProducer() { int acgist::Room::closeVideoProducer() { std::lock_guard lockRoom(roomMutex); - OH_LOG_INFO(LOG_APP, "关闭视频生产者：%s", this->roomId.data()); + OH_LOG_INFO(LOG_APP, "关闭视频生产者：%{public}s", this->roomId.data()); if (this->videoProducer != nullptr) { this->videoProducer->Close(); delete this->videoProducer; @@ -337,7 +337,7 @@ int acgist::Room::closeVideoProducer() { int acgist::Room::closeTransport() { std::lock_guard lockRoom(roomMutex); - OH_LOG_INFO(LOG_APP, "关闭通道：%s", this->roomId.data()); + OH_LOG_INFO(LOG_APP, "关闭通道：%{public}s", this->roomId.data()); if (this->sendTransport != nullptr) { this->sendTransport->Close(); delete this->sendTransport; @@ -367,13 +367,13 @@ int acgist::Room::newRemoteClient(const std::string& clientId, const std::string std::lock_guard lockRoom(roomMutex); auto oldClient = this->clients.find(clientId); if(oldClient != this->clients.end()) { - OH_LOG_INFO(LOG_APP, "已经存在远程终端：%s", clientId.data()); + OH_LOG_INFO(LOG_APP, "已经存在远程终端：%{public}s", clientId.data()); delete oldClient->second; oldClient->second = nullptr; this->clients.erase(oldClient); removeConsumerIdByClientId(clientId, this->consumerIdClientId); } - OH_LOG_INFO(LOG_APP, "新增远程终端：%s", clientId.data()); + OH_LOG_INFO(LOG_APP, "新增远程终端：%{public}s", clientId.data()); acgist::RemoteClient* client = new acgist::RemoteClient(this->mediaManager); client->clientId = clientId; client->name = name; @@ -386,10 +386,10 @@ int acgist::Room::closeRemoteClient(const std::string& clientId) { std::lock_guard lockRoom(roomMutex); auto client = this->clients.find(clientId); if(client == this->clients.end()) { - OH_LOG_INFO(LOG_APP, "远程终端已经删除：%s", clientId.data()); + OH_LOG_INFO(LOG_APP, "远程终端已经删除：%{public}s", clientId.data()); return -1; } - OH_LOG_INFO(LOG_APP, "删除远程终端：%s", clientId.data()); + OH_LOG_INFO(LOG_APP, "删除远程终端：%{public}s", clientId.data()); delete client->second; client->second = nullptr; this->clients.erase(client); @@ -409,7 +409,7 @@ int acgist::Room::newConsumer(nlohmann::json& body) { std::string kind = body["kind"]; std::string sourceId = body["sourceId"]; std::string consumerId = body["consumerId"]; - OH_LOG_INFO(LOG_APP, "新增媒体消费：%s %s %s", kind.data(), sourceId.data(), consumerId.data()); + OH_LOG_INFO(LOG_APP, "新增媒体消费：%{public}s %{public}s %{public}s", kind.data(), sourceId.data(), consumerId.data()); auto oldClient = this->clients.find(sourceId); acgist::RemoteClient* client = nullptr; if(oldClient == this->clients.end()) { @@ -431,16 +431,16 @@ int acgist::Room::closeConsumer(const std::string& consumerId) { std::lock_guard lockRoom(roomMutex); auto clientId = this->consumerIdClientId.find(consumerId); if(clientId == this->consumerIdClientId.end()) { - OH_LOG_INFO(LOG_APP, "关闭消费者无效：%s", consumerId.data()); + OH_LOG_INFO(LOG_APP, "关闭消费者无效：%{public}s", consumerId.data()); return -1; } this->consumerIdClientId.erase(clientId); auto client = this->clients.find(clientId->second); if(client == this->clients.end()) { - OH_LOG_INFO(LOG_APP, "关闭消费者无效：%s %s", consumerId.data(), clientId->second.data()); + OH_LOG_INFO(LOG_APP, "关闭消费者无效：%{public}s %{public}s", consumerId.data(), clientId->second.data()); return -2; } - OH_LOG_INFO(LOG_APP, "关闭消费者：%s %s", consumerId.data(), clientId->second.data()); + OH_LOG_INFO(LOG_APP, "关闭消费者：%{public}s %{public}s", consumerId.data(), clientId->second.data()); client->second->closeConsumer(consumerId); return 0; } @@ -449,15 +449,15 @@ int acgist::Room::pauseConsumer(const std::string& consumerId) { std::lock_guard lockRoom(roomMutex); auto clientId = this->consumerIdClientId.find(consumerId); if(clientId == this->consumerIdClientId.end()) { - OH_LOG_INFO(LOG_APP, "暂停消费者无效：%s", consumerId.data()); + OH_LOG_INFO(LOG_APP, "暂停消费者无效：%{public}s", consumerId.data()); return -1; } auto client = this->clients.find(clientId->second); if(client == this->clients.end()) { - OH_LOG_INFO(LOG_APP, "暂停消费者无效：%s %s", consumerId.data(), clientId->second.data()); + OH_LOG_INFO(LOG_APP, "暂停消费者无效：%{public}s %{public}s", consumerId.data(), clientId->second.data()); return -2; } - OH_LOG_INFO(LOG_APP, "暂停消费者：%s %s", consumerId.data(), clientId->second.data()); + OH_LOG_INFO(LOG_APP, "暂停消费者：%{public}s %{public}s", consumerId.data(), clientId->second.data()); client->second->pauseConsumer(consumerId); return 0; } @@ -466,15 +466,15 @@ int acgist::Room::resumeConsumer(const std::string& consumerId) { std::lock_guard lockRoom(roomMutex); auto clientId = this->consumerIdClientId.find(consumerId); if(clientId == this->consumerIdClientId.end()) { - OH_LOG_INFO(LOG_APP, "恢复消费者无效：%s", consumerId.data()); + OH_LOG_INFO(LOG_APP, "恢复消费者无效：%{public}s", consumerId.data()); return -1; } auto client = this->clients.find(clientId->second); if(client == this->clients.end()) { - OH_LOG_INFO(LOG_APP, "恢复消费者无效：%s %s", consumerId.data(), clientId->second.data()); + OH_LOG_INFO(LOG_APP, "恢复消费者无效：%{public}s %{public}s", consumerId.data(), clientId->second.data()); return -2; } - OH_LOG_INFO(LOG_APP, "恢复消费者：%s %s", consumerId.data(), clientId->second.data()); + OH_LOG_INFO(LOG_APP, "恢复消费者：%{public}s %{public}s", consumerId.data(), clientId->second.data()); client->second->resumeConsumer(consumerId); return 0; } @@ -482,12 +482,12 @@ int acgist::Room::resumeConsumer(const std::string& consumerId) { int acgist::Room::closeProducer(const std::string& producerId) { std::lock_guard lockRoom(roomMutex); if(this->audioProducer != nullptr && this->audioProducer->GetId() == producerId) { - OH_LOG_INFO(LOG_APP, "关闭音频生产者：%s", producerId.data()); + OH_LOG_INFO(LOG_APP, "关闭音频生产者：%{public}s", producerId.data()); this->audioProducer->Close(); delete this->audioProducer; this->audioProducer = nullptr; } else if(this->videoProducer != nullptr && this->videoProducer->GetId() == producerId) { - OH_LOG_INFO(LOG_APP, "关闭视频生产者：%s", producerId.data()); + OH_LOG_INFO(LOG_APP, "关闭视频生产者：%{public}s", producerId.data()); this->videoProducer->Close(); delete this->videoProducer; this->videoProducer = nullptr; @@ -500,10 +500,10 @@ int acgist::Room::closeProducer(const std::string& producerId) { int acgist::Room::pauseProducer(const std::string& producerId) { std::lock_guard lockRoom(roomMutex); if(this->audioProducer != nullptr && this->audioProducer->GetId() == producerId) { - OH_LOG_INFO(LOG_APP, "暂停音频生产者：%s", producerId.data()); + OH_LOG_INFO(LOG_APP, "暂停音频生产者：%{public}s", producerId.data()); this->audioProducer->Pause(); } else if(this->videoProducer != nullptr && this->videoProducer->GetId() == producerId) { - OH_LOG_INFO(LOG_APP, "暂停视频生产者：%s", producerId.data()); + OH_LOG_INFO(LOG_APP, "暂停视频生产者：%{public}s", producerId.data()); this->videoProducer->Pause(); } else { return -1; @@ -514,10 +514,10 @@ int acgist::Room::pauseProducer(const std::string& producerId) { int acgist::Room::resumeProducer(const std::string& producerId) { std::lock_guard lockRoom(roomMutex); if(this->audioProducer != nullptr && this->audioProducer->GetId() == producerId) { - OH_LOG_INFO(LOG_APP, "恢复音频生产者：%s", producerId.data()); + OH_LOG_INFO(LOG_APP, "恢复音频生产者：%{public}s", producerId.data()); this->audioProducer->Resume(); } else if(this->videoProducer != nullptr && this->videoProducer->GetId() == producerId) { - OH_LOG_INFO(LOG_APP, "恢复视频生产者：%s", producerId.data()); + OH_LOG_INFO(LOG_APP, "恢复视频生产者：%{public}s", producerId.data()); this->videoProducer->Resume(); } else { return -1; @@ -571,7 +571,7 @@ std::future acgist::SendListener::OnProduce(mediasoupclient::SendTr } std::future acgist::SendListener::OnProduceData(mediasoupclient::SendTransport* transport, const nlohmann::json& sctpStreamParameters, const std::string& label, const std::string& protocol, const nlohmann::json& appData) { - OH_LOG_INFO(LOG_APP, "生产数据：%s - %s - %s - %s", this->room->roomId.data(), transport->GetId().data(), label.data(), protocol.data()); + OH_LOG_INFO(LOG_APP, "生产数据：%{public}s - %{public}s - %{public}s - %{public}s", this->room->roomId.data(), transport->GetId().data(), label.data(), protocol.data()); // TODO: 自行实现实现逻辑 std::promise promise; // promise.set_value(producerId); @@ -585,7 +585,7 @@ acgist::RecvListener::~RecvListener() { } std::future acgist::RecvListener::OnConnect(mediasoupclient::Transport* transport, const nlohmann::json& dtlsParameters) { - OH_LOG_INFO(LOG_APP, "连接接收通道：%s - %s", this->room->roomId.data(), transport->GetId().data()); + OH_LOG_INFO(LOG_APP, "连接接收通道：%{public}s - %{public}s", this->room->roomId.data(), transport->GetId().data()); nlohmann::json requestBody = { { "roomId", this->room->roomId }, { "transportId", transport->GetId() }, @@ -598,7 +598,7 @@ std::future acgist::RecvListener::OnConnect(mediasoupclient::Transport* tr } void acgist::RecvListener::OnConnectionStateChange(mediasoupclient::Transport* transport, const std::string& connectionState) { - OH_LOG_INFO(LOG_APP, "接收通道状态改变：%s - %s - %s", this->room->roomId.data(), transport->GetId().data(), connectionState.data()); + OH_LOG_INFO(LOG_APP, "接收通道状态改变：%{public}s - %{public}s - %{public}s", this->room->roomId.data(), transport->GetId().data(), connectionState.data()); // TODO: 自行实现异常逻辑 } @@ -609,7 +609,7 @@ acgist::ProducerListener::~ProducerListener() { } void acgist::ProducerListener::OnTransportClose(mediasoupclient::Producer* producer) { - OH_LOG_INFO(LOG_APP, "生产者通道关闭：%s - %s", this->room->roomId.data(), producer->GetId().data()); + OH_LOG_INFO(LOG_APP, "生产者通道关闭：%{public}s - %{public}s", this->room->roomId.data(), producer->GetId().data()); producer->Close(); // TODO: 自行实现异常逻辑 } @@ -621,7 +621,7 @@ acgist::ConsumerListener::~ConsumerListener() { } void acgist::ConsumerListener::OnTransportClose(mediasoupclient::Consumer* consumer) { - OH_LOG_INFO(LOG_APP, "消费者通道关闭：%s - %s", this->room->roomId.data(), consumer->GetId().data()); + OH_LOG_INFO(LOG_APP, "消费者通道关闭：%{public}s - %{public}s", this->room->roomId.data(), consumer->GetId().data()); consumer->Close(); // TODO: 自行实现异常逻辑 } diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/ScreenCapturer.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/ScreenCapturer.cpp index 70b4d4a..3094f05 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/ScreenCapturer.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/ScreenCapturer.cpp @@ -41,6 +41,26 @@ static void OnVideoBufferAvailable(OH_AVScreenCapture* capture, bool isReady) { // webrtc::VideoFrame::Builder builder; // webrtc::VideoFrame videoFrame = builder.set_timestamp_ms(timestamp).set_video_frame_buffer(videoFrameBuffer).build(); // screenCapturer->source->OnData(videoFrame); + +// height = (height > 0) ? height : -height; // abs +// width = stride; +// size_t size = stride * height + stride * height / 2; +// if (bufferSize < size) { +// return false; +// } +// +// rtc::scoped_refptr i420_buffer = webrtc::I420Buffer::Create(width, height); +// libyuv::NV21ToI420(buffer, width, buffer + width * height, width, i420_buffer.get()->MutableDataY(), +// i420_buffer.get()->StrideY(), i420_buffer.get()->MutableDataU(), i420_buffer.get()->StrideU(), +// i420_buffer.get()->MutableDataV(), i420_buffer.get()->StrideV(), width, height); +// +// webrtc::VideoFrame video_frame = webrtc::VideoFrame::Builder() +// .set_video_frame_buffer(i420_buffer) +// .set_timestamp_rtp(0) +// .set_timestamp_ms(rtc::TimeMillis()) +// .set_rotation(webrtc::kVideoRotation_90) +// .build(); + OH_NativeBuffer_Unmap(buffer); OH_AVScreenCapture_ReleaseVideoBuffer(capture); } diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoEncoder.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoEncoder.cpp index 55ab313..bfd1de6 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoEncoder.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoEncoder.cpp @@ -59,7 +59,7 @@ void acgist::TaoyaoVideoEncoder::initFormatConfig(OH_AVFormat* format) { OH_AVFormat_SetIntValue(format, OH_MD_KEY_I_FRAME_INTERVAL, acgist::iFrameInterval); OH_AVFormat_SetIntValue(format, OH_MD_KEY_QUALITY, 0); OH_AVFormat_SetIntValue(format, OH_MD_KEY_RANGE_FLAG, false); - OH_AVFormat_SetIntValue(format, OH_MD_KEY_PIXEL_FORMAT, AV_PIXEL_FORMAT_YUVI420); + OH_AVFormat_SetIntValue(format, OH_MD_KEY_PIXEL_FORMAT, AV_PIXEL_FORMAT_YUVI420); // AV_PIXEL_FORMAT_NV21 OH_AVFormat_SetIntValue(format, OH_MD_KEY_PROFILE, static_cast(OH_AVCProfile::AVC_PROFILE_BASELINE)); OH_AVFormat_SetIntValue(format, OH_MD_KEY_COLOR_PRIMARIES, static_cast(OH_ColorPrimary::COLOR_PRIMARY_BT709)); OH_AVFormat_SetIntValue(format, OH_MD_KEY_MATRIX_COEFFICIENTS, static_cast(OH_MatrixCoefficient::MATRIX_COEFFICIENT_IDENTITY)); diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/WebRTC.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/WebRTC.cpp index c2fc8a0..a0e58d9 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/WebRTC.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/WebRTC.cpp @@ -71,6 +71,27 @@ absl::optional acgist::TaoyaoVideoTrackSource::needs_denoising() const { void acgist::TaoyaoVideoTrackSource::OnData(const webrtc::VideoFrame& videoFrame) { OH_LOG_DEBUG(LOG_APP, "视频数据：%{public}d %{public}d", videoFrame.width(), videoFrame.height()); +// if (out_height != frame.height() || out_width != frame.width()) { +// rtc::scoped_refptr scaled_buffer = I420Buffer::Create(out_width, out_height); +// scaled_buffer->ScaleFrom(*frame.video_frame_buffer()->ToI420()); +// VideoFrame::Builder new_frame_builder = VideoFrame::Builder() +// .set_video_frame_buffer(scaled_buffer) +// .set_rotation(kVideoRotation_0) +// .set_timestamp_us(frame.timestamp_us()) +// .set_id(frame.id()); +// if (frame.has_update_rect()) { +// VideoFrame::UpdateRect new_rect = frame.update_rect().ScaleWithFrame( +// frame.width(), frame.height(), 0, 0, frame.width(), frame.height(), out_width, out_height); +// new_frame_builder.set_update_rect(new_rect); +// } +// OH_LOG_Print(LOG_APP, LOG_INFO, LOG_DOMAIN, "OhosDesktopCapture", "OhosDesktopCapture::OnFrame 2"); +// broadcaster_.OnFrame(new_frame_builder.build()); +// +// } else { +// // No adaptations needed, just return the frame as is. +// OH_LOG_Print(LOG_APP, LOG_INFO, LOG_DOMAIN, "OhosDesktopCapture", "OhosDesktopCapture::OnFrame 3"); +// broadcaster_.OnFrame(frame); +// } this->OnFrame(videoFrame); } diff --git a/taoyao-client-openharmony/taoyao/media/src/main/ets/pages/Index.ets b/taoyao-client-openharmony/taoyao/media/src/main/ets/pages/Index.ets index 4939df9..2f205ab 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/ets/pages/Index.ets +++ b/taoyao-client-openharmony/taoyao/media/src/main/ets/pages/Index.ets @@ -1,6 +1,7 @@ import { taoyaoSignal } from "../taoyao/TaoyaoSignal"; import fs from '@ohos.file.fs'; +import image from "@ohos.multimedia.image"; import { BusinessError } from "@ohos.base"; import common from "@ohos.app.ability.common"; import abilityAccessCtrl, { Permissions } from "@ohos.abilityAccessCtrl"; @@ -69,6 +70,7 @@ struct Index { .fontSize(20) .fontWeight(FontWeight.Bold) .onClick(() => { + // const imageReceiver = image.createImageReceiver({ width: 640, height: 480 }, image.ImageFormat.JPEG, 8); taoyaoSignal.init(); }); } diff --git a/taoyao-client-openharmony/taoyao/media/src/main/ets/taoyao/Setting.ets b/taoyao-client-openharmony/taoyao/media/src/main/ets/taoyao/Setting.ets index c162913..4c4108b 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/ets/taoyao/Setting.ets +++ b/taoyao-client-openharmony/taoyao/media/src/main/ets/taoyao/Setting.ets @@ -30,7 +30,7 @@ class Setting { // 信令版本 version: string = "1.0.0"; // 信令地址 - signalAddress: string = "wss://192.168.8.20:8888/websocket.signal"; + signalAddress: string = "wss://192.168.8.60:8888/websocket.signal"; // signalAddress: string = "wss://192.168.1.100:8888/websocket.signal"; // 启动自动加载系统 initOnLoad : boolean = true;