diff --git a/taoyao-client-media/.env b/taoyao-client-media/.env index 1766951..ad21851 100644 --- a/taoyao-client-media/.env +++ b/taoyao-client-media/.env @@ -25,4 +25,4 @@ MEDIASOUP_LISTEN_IP=0.0.0.0 # Mediasoup监听端口(起始端口) MEDIASOUP_LISTEN_PORT=44444 # Mediasoup声明地址(不能配置环回地址) -MEDIASOUP_ANNOUNCED_IP=192.168.1.100 \ No newline at end of file +MEDIASOUP_ANNOUNCED_IP=192.168.1.239 \ No newline at end of file diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/CMakeLists.txt b/taoyao-client-openharmony/taoyao/media/src/main/cpp/CMakeLists.txt index 67cd3cd..b566a3b 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/CMakeLists.txt +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/CMakeLists.txt @@ -30,6 +30,7 @@ add_subdirectory("./deps/libmediasoupclient") add_library( ${PROJECT_NAME} SHARED bind.cpp + media/Signal.cpp media/WebRTC.cpp media/Room.cpp media/Client.cpp @@ -69,6 +70,8 @@ target_link_libraries( libohaudio.so # 相机 libohcamera.so + # ACE + libace_ndk.z.so # NAPI libace_napi.z.so # LOG diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/bind.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/bind.cpp index e3b1cd3..5839027 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/bind.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/bind.cpp @@ -77,17 +77,6 @@ static std::recursive_mutex taoyaoMutex; namespace acgist { -uint32_t width = 720; -uint32_t height = 480; -uint64_t bitrate = 3'000'000L; -uint32_t iFrameInterval = 5'000; -double frameRate = 30.0; -int32_t samplingRate = 48'000; -int32_t channelCount = 2; -int32_t bitsPerSample = 16; -std::string clientId = ""; -std::string name = ""; - // 索引:667-999 static uint32_t index = 667; // 最小索引 diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Capturer.hpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Capturer.hpp index b918c28..dccff47 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Capturer.hpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Capturer.hpp @@ -44,6 +44,7 @@ #include #include +#include "ohcamera/camera_input.h" #include #include @@ -154,6 +155,8 @@ public: uint32_t cameraSize = 0; // 摄像头索引 uint32_t cameraIndex = 0; + // 摄像头输入 + Camera_Input* cameraInput = nullptr; // 摄像头设备列表 Camera_Device* cameraDevice = nullptr; // 摄像头管理器 diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Signal.hpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Signal.hpp index bee6939..9a9c4a8 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Signal.hpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Signal.hpp @@ -7,6 +7,33 @@ #ifndef TAOYAO_SIGNAL_HPP #define TAOYAO_SIGNAL_HPP +#ifndef TAOYAO_AUDIO_RET_LOG +#define TAOYAO_AUDIO_RET_LOG(format, ret, ...) \ +if(ret == OH_AudioStream_Result::AUDIOSTREAM_SUCCESS) { \ + OH_LOG_INFO(LOG_APP, format, ret, ##__VA_ARGS__); \ +} else { \ + OH_LOG_ERROR(LOG_APP, format, ret, ##__VA_ARGS__); \ +} +#endif + +#ifndef TAOYAO_VIDEO_RET_LOG +#define TAOYAO_VIDEO_RET_LOG(format, ret, ...) \ +if(ret == Camera_ErrorCode::CAMERA_OK) { \ + OH_LOG_INFO(LOG_APP, format, ret, ##__VA_ARGS__); \ +} else { \ + OH_LOG_ERROR(LOG_APP, format, ret, ##__VA_ARGS__); \ +} +#endif + +#ifndef TAOYAO_OPENGL_RET_LOG +#define TAOYAO_OPENGL_RET_LOG(format, ret, ...) \ +if(ret) { \ + OH_LOG_INFO(LOG_APP, format, ret, ##__VA_ARGS__); \ +} else { \ + OH_LOG_ERROR(LOG_APP, format, ret, ##__VA_ARGS__); \ +} +#endif + #include #include diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioCapturer.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioCapturer.cpp index 1fb840a..8cf138e 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioCapturer.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioCapturer.cpp @@ -14,13 +14,13 @@ static int32_t OnInterruptEvent(OH_AudioCapturer* capturer, void* userData, OH_A acgist::AudioCapturer::AudioCapturer() { OH_AudioStream_Result ret = OH_AudioStreamBuilder_Create(&this->builder, AUDIOSTREAM_TYPE_CAPTURER); - OH_LOG_INFO(LOG_APP, "配置音频构造器:%o", ret); + TAOYAO_AUDIO_RET_LOG("配置音频构造器:%{public}d", ret); // 配置音频采集参数 OH_AudioStreamBuilder_SetSamplingRate(this->builder, acgist::samplingRate); OH_AudioStreamBuilder_SetChannelCount(this->builder, acgist::channelCount); OH_AudioStreamBuilder_SetLatencyMode(this->builder, OH_AudioStream_LatencyMode::AUDIOSTREAM_LATENCY_MODE_NORMAL); OH_AudioStreamBuilder_SetSampleFormat(this->builder, OH_AudioStream_SampleFormat::AUDIOSTREAM_SAMPLE_S16LE); - OH_LOG_DEBUG(LOG_APP, "配置音频采集参数:%d %d", acgist::samplingRate, acgist::channelCount); + OH_LOG_DEBUG(LOG_APP, "配置音频采集参数:%{public}d %{public}d", acgist::samplingRate, acgist::channelCount); // 设置音频采集回调 OH_AudioCapturer_Callbacks callbacks; callbacks.OH_AudioCapturer_OnError = OnError; @@ -28,7 +28,7 @@ acgist::AudioCapturer::AudioCapturer() { callbacks.OH_AudioCapturer_OnStreamEvent = OnStreamEvent; callbacks.OH_AudioCapturer_OnInterruptEvent = OnInterruptEvent; ret = OH_AudioStreamBuilder_SetCapturerCallback(this->builder, callbacks, this); - OH_LOG_DEBUG(LOG_APP, "设置音频采集回调:%o", ret); + TAOYAO_AUDIO_RET_LOG("设置音频采集回调:%{public}d", ret); } acgist::AudioCapturer::~AudioCapturer() { @@ -36,7 +36,7 @@ acgist::AudioCapturer::~AudioCapturer() { if(this->builder != nullptr) { OH_AudioStream_Result ret = OH_AudioStreamBuilder_Destroy(this->builder); this->builder = nullptr; - OH_LOG_INFO(LOG_APP, "释放音频构造器:%o", ret); + TAOYAO_AUDIO_RET_LOG("释放音频构造器:%{public}d", ret); } } @@ -48,10 +48,10 @@ bool acgist::AudioCapturer::start() { this->running = true; // 配置音频采集器 OH_AudioStream_Result ret = OH_AudioStreamBuilder_GenerateCapturer(this->builder, &this->audioCapturer); - OH_LOG_DEBUG(LOG_APP, "配置音频采集器:%o", ret); + TAOYAO_AUDIO_RET_LOG("配置音频采集器:%{public}d", ret); // 开始音频采集 ret = OH_AudioCapturer_Start(this->audioCapturer); - OH_LOG_DEBUG(LOG_APP, "开始音频采集:%o", ret); + TAOYAO_AUDIO_RET_LOG("开始音频采集:%{public}d", ret); return ret == OH_AudioStream_Result::AUDIOSTREAM_SUCCESS; } @@ -63,11 +63,11 @@ bool acgist::AudioCapturer::stop() { this->running = false; // 停止音频采集 OH_AudioStream_Result ret = OH_AudioCapturer_Stop(this->audioCapturer); - OH_LOG_DEBUG(LOG_APP, "停止音频采集:%o", ret); + TAOYAO_AUDIO_RET_LOG("停止音频采集:%{public}d", ret); // 释放音频采集器 ret = OH_AudioCapturer_Release(this->audioCapturer); this->audioCapturer = nullptr; - OH_LOG_DEBUG(LOG_APP, "释放音频采集器:%o", ret); + TAOYAO_AUDIO_RET_LOG("释放音频采集器:%{public}d", ret); return ret == OH_AudioStream_Result::AUDIOSTREAM_SUCCESS; } @@ -77,6 +77,7 @@ static int32_t OnError(OH_AudioCapturer* capturer, void* userData, OH_AudioStrea } static int32_t OnReadData(OH_AudioCapturer* capturer, void* userData, void* buffer, int32_t length) { + // OH_LOG_ERROR(LOG_APP, "音频数据采集回调:%{public}d", length); if(userData == nullptr) { return -1; } diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioPlayer.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioPlayer.cpp index 96fb67b..a213c46 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioPlayer.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioPlayer.cpp @@ -14,7 +14,7 @@ static int32_t OnInterruptEvent(OH_AudioRenderer* renderer, void* userData, OH_A acgist::AudioPlayer::AudioPlayer() { OH_AudioStream_Result ret = OH_AudioStreamBuilder_Create(&this->builder, AUDIOSTREAM_TYPE_RENDERER); - OH_LOG_INFO(LOG_APP, "配置音频构造器:%o", ret); + TAOYAO_AUDIO_RET_LOG("配置音频构造器:%{public}d", ret); // 配置音频播放参数 OH_AudioStreamBuilder_SetSamplingRate(this->builder, acgist::samplingRate); OH_AudioStreamBuilder_SetChannelCount(this->builder, acgist::channelCount); @@ -29,7 +29,7 @@ acgist::AudioPlayer::AudioPlayer() { callbacks.OH_AudioRenderer_OnStreamEvent = OnStreamEvent; callbacks.OH_AudioRenderer_OnInterruptEvent = OnInterruptEvent; ret = OH_AudioStreamBuilder_SetRendererCallback(this->builder, callbacks, this); - OH_LOG_DEBUG(LOG_APP, "设置音频播放回调:%o", ret); + TAOYAO_AUDIO_RET_LOG("设置音频播放回调:%{public}d", ret); } acgist::AudioPlayer::~AudioPlayer() { @@ -37,7 +37,7 @@ acgist::AudioPlayer::~AudioPlayer() { if(this->builder != nullptr) { OH_AudioStream_Result ret = OH_AudioStreamBuilder_Destroy(this->builder); this->builder = nullptr; - OH_LOG_INFO(LOG_APP, "释放音频构造器:%o", ret); + TAOYAO_AUDIO_RET_LOG("释放音频构造器:%{public}d", ret); } } @@ -49,10 +49,10 @@ bool acgist::AudioPlayer::start() { this->running = true; // 配置音频播放器 OH_AudioStream_Result ret = OH_AudioStreamBuilder_GenerateRenderer(this->builder, &this->audioRenderer); - OH_LOG_DEBUG(LOG_APP, "配置音频播放器:%o", ret); + TAOYAO_AUDIO_RET_LOG("配置音频播放器:%{public}d", ret); // 开始音频播放 ret = OH_AudioRenderer_Start(this->audioRenderer); - OH_LOG_DEBUG(LOG_APP, "开始音频播放:%o", ret); + TAOYAO_AUDIO_RET_LOG("开始音频播放:%{public}d", ret); return ret == OH_AudioStream_Result::AUDIOSTREAM_SUCCESS; } @@ -64,11 +64,11 @@ bool acgist::AudioPlayer::stop() { this->running = false; // 停止音频播放 OH_AudioStream_Result ret = OH_AudioRenderer_Stop(this->audioRenderer); - OH_LOG_DEBUG(LOG_APP, "停止音频播放:%o", ret); + TAOYAO_AUDIO_RET_LOG("停止音频播放:%{public}d", ret); // 释放音频播放器 ret = OH_AudioRenderer_Release(this->audioRenderer); this->audioRenderer = nullptr; - OH_LOG_DEBUG(LOG_APP, "释放音频播放器:%o", ret); + TAOYAO_AUDIO_RET_LOG("释放音频播放器:%{public}d", ret); return ret == OH_AudioStream_Result::AUDIOSTREAM_SUCCESS; } diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/LocalClient.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/LocalClient.cpp index dd5482b..29a8119 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/LocalClient.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/LocalClient.cpp @@ -2,8 +2,8 @@ acgist::LocalClient::LocalClient(acgist::MediaManager* mediaManager) : acgist::RoomClient(mediaManager) { this->mediaManager->newLocalClient(); - this->audioTrack = this->mediaManager->getAudioTrack(); -// this->videoTrack = this->mediaManager->getVideoTrack(); + // this->audioTrack = this->mediaManager->getAudioTrack(); + this->videoTrack = this->mediaManager->getVideoTrack(); } acgist::LocalClient::~LocalClient() { diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/MediaManager.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/MediaManager.cpp index 1612d5a..96b1e0f 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/MediaManager.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/MediaManager.cpp @@ -109,8 +109,8 @@ int acgist::MediaManager::releaseLocalClient() { } bool acgist::MediaManager::startCapture() { - this->startAudioCapture(); -// this->startVideoCapture(); + // this->startAudioCapture(); + this->startVideoCapture(); return true; } diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/Room.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/Room.cpp index a9dd8c4..6357c6e 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/Room.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/Room.cpp @@ -118,10 +118,10 @@ int acgist::Room::produceMedia() { this->createRecvTransport(); } if(this->audioProduce) { - this->produceAudio(); + // this->produceAudio(); } if(this->videoProduce) { -// this->produceVideo(); + this->produceVideo(); } return 0; } diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/Signal.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/Signal.cpp new file mode 100644 index 0000000..3527ebc --- /dev/null +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/Signal.cpp @@ -0,0 +1,16 @@ +#include "../include/Signal.hpp" + +namespace acgist { + +uint32_t width = 720; +uint32_t height = 480; +uint64_t bitrate = 3'000'000L; +uint32_t iFrameInterval = 5'000; +double frameRate = 30.0; +int32_t samplingRate = 48'000; +int32_t channelCount = 2; +int32_t bitsPerSample = 16; +std::string clientId = ""; +std::string name = ""; + +} diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoCapturer.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoCapturer.cpp index 6f5496d..c937d96 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoCapturer.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoCapturer.cpp @@ -7,6 +7,14 @@ * https://docs.openharmony.cn/pages/v4.1/zh-cn/application-dev/graphics/native-window-guidelines.md * https://docs.openharmony.cn/pages/v4.1/zh-cn/application-dev/reference/native-lib/vulkan-guidelines.md * https://docs.openharmony.cn/pages/v4.1/zh-cn/application-dev/reference/native-lib/openglesv3-symbol.md + * + * https://gitee.com/openharmony/third_party_vulkan-headers/blob/master/include/vulkan/vulkan.hpp + * https://gitee.com/openharmony/graphic_graphic_surface/blob/master/surface/src/native_window.cpp + * https://gitee.com/openharmony/graphic_graphic_2d/blob/master/frameworks/surfaceimage/src/native_image.cpp + * https://gitee.com/openharmony/arkui_ace_engine/blob/master/interfaces/native/native_interface_xcomponent.cpp + * https://gitee.com/openharmony/multimedia_camera_framework/blob/master/frameworks/native/ndk/camera_manager.cpp + * https://gitee.com/openharmony/multimedia_camera_framework/blob/master/frameworks/native/ndk/impl/camera_manager_impl.cpp + * https://gitee.com/openharmony/multimedia_camera_framework/blob/master/frameworks/native/camera/src/session/capture_session.cpp */ #include "../include/Capturer.hpp" @@ -15,6 +23,8 @@ #include "hilog/log.h" +#include + #include "rtc_base/time_utils.h" #include "api/video/nv12_buffer.h" @@ -44,20 +54,21 @@ acgist::VideoCapturer::VideoCapturer() { initOpenGLES(); #endif Camera_ErrorCode ret = OH_Camera_GetCameraManager(&this->cameraManager); - OH_LOG_INFO(LOG_APP, "配置摄像头管理器:%o", ret); + TAOYAO_VIDEO_RET_LOG("配置摄像头管理器:%{public}d", ret); ret = OH_CameraManager_GetSupportedCameras(this->cameraManager, &this->cameraDevice, &this->cameraSize); - OH_LOG_INFO(LOG_APP, "获取摄像头设备列表:%o %d", ret, this->cameraSize); + TAOYAO_VIDEO_RET_LOG("摄像头设备列表数量:%{public}d %{public}d", ret, this->cameraSize); + ret = OH_CameraManager_CreateCameraInput(this->cameraManager, &this->cameraDevice[this->cameraIndex], &this->cameraInput); + TAOYAO_VIDEO_RET_LOG("配置摄像头输入:%{public}d %{public}d", ret, this->cameraIndex); + ret = OH_CameraInput_Open(cameraInput); + TAOYAO_VIDEO_RET_LOG("打开摄像头:%{public}d", ret); ret = OH_CameraManager_GetSupportedCameraOutputCapability(this->cameraManager, &this->cameraDevice[this->cameraIndex], &this->cameraOutputCapability); - OH_LOG_INFO(LOG_APP, "获取摄像头输出功能:%o %d %d", ret, this->cameraIndex, this->cameraOutputCapability->videoProfilesSize); + TAOYAO_VIDEO_RET_LOG("摄像头输出功能:%{public}d %{public}d %{public}d", ret, this->cameraIndex, this->cameraOutputCapability->videoProfilesSize); // 注册相机状态回调:可以取消注册 // OH_CameraManager_RegisterCallback(this->cameraManager, CameraManager_Callbacks* callback); -// char surfaceId[OH_XCOMPONENT_ID_LEN_MAX + 1] = {}; -// uint64_t idSize = OH_XCOMPONENT_ID_LEN_MAX + 1; -// OH_NativeXComponent_GetXComponentId((OH_NativeXComponent*) this->nativeWindow, surfaceId, idSize); - ret = OH_CameraManager_CreateVideoOutput(this->cameraManager, this->cameraOutputCapability->videoProfiles[0], (char*) this->surfaceId, &this->cameraVideoOutput); - OH_LOG_INFO(LOG_APP, "创建摄像头视频输出:%o", ret); + ret = OH_CameraManager_CreateVideoOutput(this->cameraManager, this->cameraOutputCapability->videoProfiles[0], std::to_string(this->surfaceId).data(), &this->cameraVideoOutput); + TAOYAO_VIDEO_RET_LOG("创建摄像头视频输出:%{public}d %{public}lld %{public}s", ret, this->surfaceId, std::to_string(this->surfaceId).data()); ret = OH_CameraManager_CreateCaptureSession(this->cameraManager, &this->cameraCaptureSession); - OH_LOG_INFO(LOG_APP, "创建摄像头视频会话:%o", ret); + TAOYAO_VIDEO_RET_LOG("创建摄像头视频会话:%{public}d", ret); // 设置相机:闪光、变焦、质量、高宽、补光、防抖 // 设置缩放比例 // OH_CaptureSession_SetZoomRatio(this->cameraCaptureSession, 0.5F); @@ -72,19 +83,22 @@ acgist::VideoCapturer::~VideoCapturer() { #endif Camera_ErrorCode ret = OH_CaptureSession_Release(this->cameraCaptureSession); this->cameraCaptureSession = nullptr; - OH_LOG_INFO(LOG_APP, "释放摄像头视频会话:%o", ret); + TAOYAO_VIDEO_RET_LOG("释放摄像头视频会话:%{public}o", ret); ret = OH_VideoOutput_Release(this->cameraVideoOutput); this->cameraVideoOutput = nullptr; - OH_LOG_INFO(LOG_APP, "释放摄像头视频输出:%o", ret); + ret = OH_CameraInput_Release(this->cameraInput); + this->cameraInput = nullptr; + TAOYAO_VIDEO_RET_LOG("释放摄像头:%{public}o", ret); + TAOYAO_VIDEO_RET_LOG("释放摄像头视频输出:%{public}o", ret); ret = OH_CameraManager_DeleteSupportedCameraOutputCapability(this->cameraManager, this->cameraOutputCapability); this->cameraOutputCapability = nullptr; - OH_LOG_INFO(LOG_APP, "释放摄像头输出能力:%o", ret); + TAOYAO_VIDEO_RET_LOG("释放摄像头输出能力:%{public}d", ret); ret = OH_CameraManager_DeleteSupportedCameras(this->cameraManager, this->cameraDevice, this->cameraSize); this->cameraDevice = nullptr; - OH_LOG_INFO(LOG_APP, "释放摄像头设备列表:%o", ret); + TAOYAO_VIDEO_RET_LOG("释放摄像头设备列表:%{public}d", ret); ret = OH_Camera_DeleteCameraManager(this->cameraManager); this->cameraManager = nullptr; - OH_LOG_INFO(LOG_APP, "释放摄像头管理器:%o", ret); + TAOYAO_VIDEO_RET_LOG("释放摄像头管理器:%{public}d", ret); } bool acgist::VideoCapturer::start() { @@ -93,22 +107,26 @@ bool acgist::VideoCapturer::start() { return true; } this->running = true; - OH_CaptureSession_BeginConfig(this->cameraCaptureSession); - Camera_ErrorCode ret = OH_CaptureSession_AddVideoOutput(this->cameraCaptureSession, this->cameraVideoOutput); - OH_LOG_INFO(LOG_APP, "视频捕获绑定会话:%o", ret); - OH_CaptureSession_CommitConfig(this->cameraCaptureSession); + OH_NativeImage_AttachContext(this->nativeImage, this->textureId); + Camera_ErrorCode ret = OH_CaptureSession_BeginConfig(this->cameraCaptureSession); + TAOYAO_VIDEO_RET_LOG("开始配置视频会话:%{public}o", ret); + ret = OH_CaptureSession_AddInput(this->cameraCaptureSession, this->cameraInput); + TAOYAO_VIDEO_RET_LOG("视频输入绑定会话:%{public}d", ret); + ret = OH_CaptureSession_AddVideoOutput(this->cameraCaptureSession, this->cameraVideoOutput); + TAOYAO_VIDEO_RET_LOG("视频捕获绑定会话:%{public}d", ret); + ret = OH_CaptureSession_CommitConfig(this->cameraCaptureSession); + TAOYAO_VIDEO_RET_LOG("结束配置视频会话:%{public}o", ret); ret = OH_CaptureSession_Start(this->cameraCaptureSession); - OH_LOG_INFO(LOG_APP, "开始视频会话:%o", ret); + TAOYAO_VIDEO_RET_LOG("开始视频会话:%{public}d", ret); ret = OH_VideoOutput_Start(this->cameraVideoOutput); - OH_LOG_INFO(LOG_APP, "开始视频捕获:%o", ret); + TAOYAO_VIDEO_RET_LOG("开始视频捕获:%{public}d", ret); VideoOutput_Callbacks callbacks; callbacks.onError = onError; callbacks.onFrameEnd = onFrameEnd; callbacks.onFrameStart = onFrameStart; - // :可以取消注册 + // 可以取消注册 ret = OH_VideoOutput_RegisterCallback(this->cameraVideoOutput, &callbacks); - OH_LOG_INFO(LOG_APP, "视频捕获回调:%o", ret); - OH_NativeImage_AttachContext(this->nativeImage, this->textureId); + TAOYAO_VIDEO_RET_LOG("视频捕获回调:%{public}d", ret); return ret = Camera_ErrorCode::CAMERA_OK; } @@ -118,16 +136,22 @@ bool acgist::VideoCapturer::stop() { return true; } this->running = false; - OH_NativeImage_DetachContext(this->nativeImage); - OH_CaptureSession_BeginConfig(this->cameraCaptureSession); - Camera_ErrorCode ret = OH_CaptureSession_RemoveVideoOutput(this->cameraCaptureSession, this->cameraVideoOutput); - OH_LOG_INFO(LOG_APP, "视频捕获取消绑定会话:%o", ret); - OH_CaptureSession_CommitConfig(this->cameraCaptureSession); + Camera_ErrorCode ret = OH_CaptureSession_BeginConfig(this->cameraCaptureSession); + TAOYAO_VIDEO_RET_LOG("开始配置视频会话:%{public}o", ret); + ret = OH_CaptureSession_RemoveInput(this->cameraCaptureSession, this->cameraInput); + TAOYAO_VIDEO_RET_LOG("视频输入取消绑定会话:%{public}o", ret); + ret = OH_CaptureSession_RemoveVideoOutput(this->cameraCaptureSession, this->cameraVideoOutput); + TAOYAO_VIDEO_RET_LOG("视频捕获取消绑定会话:%{public}o", ret); + ret = OH_CaptureSession_CommitConfig(this->cameraCaptureSession); + TAOYAO_VIDEO_RET_LOG("结束配置视频会话:%{public}o", ret); ret = OH_CaptureSession_Stop(this->cameraCaptureSession); - OH_LOG_INFO(LOG_APP, "结束视频会话:%o", ret); + TAOYAO_VIDEO_RET_LOG("结束视频会话:%{public}o", ret); ret = OH_VideoOutput_Stop(this->cameraVideoOutput); // OH_VideoOutput_UnregisterCallback(this->cameraVideoOutput, callback); - OH_LOG_INFO(LOG_APP, "结束视频捕获:%o", ret); + TAOYAO_VIDEO_RET_LOG("结束视频捕获:%{public}o", ret); + ret = OH_CameraInput_Close(this->cameraInput); + TAOYAO_VIDEO_RET_LOG("关闭摄像头:%{public}o", ret); + OH_NativeImage_DetachContext(this->nativeImage); return ret = Camera_ErrorCode::CAMERA_OK; } @@ -152,12 +176,12 @@ void acgist::VideoCapturer::initVulkan() { vkInstanceCreateInfo.ppEnabledExtensionNames = instanceExtensions.data(); OH_LOG_INFO(LOG_APP, "配置vkInstanceCreateInfo"); VkResult ret = vkCreateInstance(&this->vkInstanceCreateInfo, nullptr, &this->vkInstance); - OH_LOG_INFO(LOG_APP, "加载vkInstance:%o", ret); + OH_LOG_INFO(LOG_APP, "加载vkInstance:%d", ret); this->vkSurfaceCreateInfoOHOS.sType = VK_STRUCTURE_TYPE_SURFACE_CREATE_INFO_OHOS; this->vkSurfaceCreateInfoOHOS.window = this->nativeWindow; OH_LOG_INFO(LOG_APP, "配置vkSurfaceCreateInfoOHOS"); ret = vkCreateSurfaceOHOS(this->vkInstance, &this->vkSurfaceCreateInfoOHOS, nullptr, &this->vkSurfaceKHR); - OH_LOG_INFO(LOG_APP, "加载vkSurfaceKHR:%o", ret); + OH_LOG_INFO(LOG_APP, "加载vkSurfaceKHR:%d", ret); } void acgist::VideoCapturer::releaseVulkan() { @@ -172,16 +196,6 @@ void acgist::VideoCapturer::releaseVulkan() { } void acgist::VideoCapturer::initOpenGLES() { - // IMAGE WINDOW - glGenTextures(this->textureSize, &this->textureId); - this->nativeImage = OH_NativeImage_Create(this->textureId, GL_TEXTURE_2D); - // TODO: 验证是否需要window -// this->nativeWindow = OH_NativeImage_AcquireNativeWindow(this->nativeImage); -// OH_NativeWindow_NativeWindowHandleOpt(this->nativeWindow, SET_BUFFER_GEOMETRY, acgist::width, acgist::height); -// OH_NativeWindow_NativeWindowHandleOpt(this->nativeWindow, SET_TIMEOUT, 100'000); -// OH_NativeWindow_NativeWindowHandleOpt(this->nativeWindow, GET_TIMEOUT, 100'000); - OH_OnFrameAvailableListener listener = { this, onFrame }; - OH_NativeImage_SetOnFrameAvailableListener(this->nativeImage, listener); // EGL static const char* EGL_EXT_PLATFORM_WAYLAND = "EGL_EXT_platform_wayland"; static const char* EGL_KHR_PLATFORM_WAYLAND = "EGL_KHR_platform_wayland"; @@ -194,29 +208,34 @@ void acgist::VideoCapturer::initOpenGLES() { if (extensions && (CheckEglExtension(extensions, EGL_EXT_PLATFORM_WAYLAND) || CheckEglExtension(extensions, EGL_KHR_PLATFORM_WAYLAND))) { PFNEGLGETPLATFORMDISPLAYEXTPROC eglGetPlatformDisplayExt = (PFNEGLGETPLATFORMDISPLAYEXTPROC) eglGetProcAddress(EGL_GET_PLATFORM_DISPLAY_EXT); this->eglDisplay = eglGetPlatformDisplayExt(EGL_PLATFORM_OHOS_KHR, EGL_DEFAULT_DISPLAY, nullptr); + OH_LOG_INFO(LOG_APP, "扩展EGLDisplay"); } } // 新建 if(this->eglDisplay == EGL_NO_DISPLAY) { this->eglDisplay = eglGetDisplay(EGL_DEFAULT_DISPLAY); + OH_LOG_INFO(LOG_APP, "新建EGLDisplay"); } + // 配置 EGLint count; EGLConfig config; EGLint config_attribs[] = { - EGL_SURFACE_TYPE, EGL_WINDOW_BIT, + EGL_RENDERABLE_TYPE, EGL_OPENGL_BIT, + EGL_SURFACE_TYPE, EGL_PBUFFER_BIT, EGL_RED_SIZE, 8, EGL_GREEN_SIZE, 8, EGL_BLUE_SIZE, 8, EGL_ALPHA_SIZE, 8, - EGL_RENDERABLE_TYPE, EGL_OPENGL_ES3_BIT, + EGL_DEPTH_SIZE, 0, + EGL_STENCIL_SIZE, 0, EGL_NONE }; EGLint context_attribs[] = { - EGL_CONTEXT_CLIENT_VERSION, 2, + EGL_CONTEXT_CLIENT_VERSION, 3, EGL_NONE }; EGLBoolean ret = eglChooseConfig(this->eglDisplay, config_attribs, &config, 1, &count); - OH_LOG_INFO(LOG_APP, "EGL选择配置:%d", ret); + TAOYAO_OPENGL_RET_LOG("EGL选择配置:%{public}d", ret); // 当前 this->eglContext = eglGetCurrentContext(); // 新建 @@ -224,16 +243,41 @@ void acgist::VideoCapturer::initOpenGLES() { EGLint major; EGLint minor; ret = eglInitialize(this->eglDisplay, &major, &minor); - OH_LOG_INFO(LOG_APP, "加载EGL:%d", ret); + TAOYAO_OPENGL_RET_LOG("加载EGL:%{public}d", ret); ret = eglBindAPI(EGL_OPENGL_ES_API); - OH_LOG_INFO(LOG_APP, "绑定EGL:%d", ret); + TAOYAO_OPENGL_RET_LOG("绑定EGL:%{public}d", ret); this->eglContext = eglCreateContext(this->eglDisplay, config, EGL_NO_CONTEXT, context_attribs); } - this->eglSurface = eglCreatePbufferSurface(this->eglDisplay, config, context_attribs); - // TODO: 验证强转 + const EGLint surfaceAttr[] = { + EGL_WIDTH, 512, + EGL_HEIGHT, 512, + EGL_LARGEST_PBUFFER, EGL_TRUE, + EGL_NONE + }; + // TODO: 验证强转 + this->eglSurface = eglCreatePbufferSurface(this->eglDisplay, config, surfaceAttr); + // this->eglSurface = eglCreateWindowSurface(this->eglDisplay, config, (EGLNativeWindowType) this->nativeWindow, context_attribs); + + eglMakeCurrent(this->eglDisplay, this->eglSurface, this->eglSurface, this->eglContext); + + // IMAGE WINDOW + glGenTextures(this->textureSize, &this->textureId); + this->nativeImage = OH_NativeImage_Create(this->textureId, GL_TEXTURE_2D); + OH_LOG_INFO(LOG_APP, "创建NativeImage:%{public}d", this->textureId); + // TODO: 验证是否需要window +// this->nativeWindow = OH_NativeImage_AcquireNativeWindow(this->nativeImage); +// OH_LOG_INFO(LOG_APP, "视频采集Window:%{public}d", this->nativeWindow); + // this->eglSurface = eglCreatePlatformWindowSurface(this->eglDisplay, config, this->nativeWindow, context_attribs); +// OH_NativeWindow_NativeWindowHandleOpt(this->nativeWindow, SET_BUFFER_GEOMETRY, acgist::width, acgist::height); + // OH_NativeWindow_NativeWindowHandleOpt(this->nativeWindow, SET_TIMEOUT, 10'000); + // OH_NativeWindow_NativeWindowHandleOpt(this->nativeWindow, GET_TIMEOUT, 10'000); + OH_OnFrameAvailableListener listener = { this, onFrame }; + OH_NativeImage_SetOnFrameAvailableListener(this->nativeImage, listener); + OH_NativeImage_GetSurfaceId(this->nativeImage, &this->surfaceId); + OH_LOG_INFO(LOG_APP, "视频采集SurfaceId:%{public}lld", this->surfaceId); } void acgist::VideoCapturer::releaseOpenGLES() { @@ -244,12 +288,12 @@ void acgist::VideoCapturer::releaseOpenGLES() { if(this->eglSurface != EGL_NO_SURFACE) { EGLBoolean ret = eglDestroySurface(this->eglDisplay, this->eglSurface); this->eglSurface = EGL_NO_SURFACE; - OH_LOG_INFO(LOG_APP, "销毁EGLSurface:%d", ret); + TAOYAO_OPENGL_RET_LOG("销毁EGLSurface:%d", ret); } if(this->eglContext != EGL_NO_CONTEXT) { EGLBoolean ret = eglDestroyContext(this->eglDisplay, this->eglContext); this->eglContext = EGL_NO_CONTEXT; - OH_LOG_INFO(LOG_APP, "销毁EGLContext:%d", ret); + TAOYAO_OPENGL_RET_LOG("销毁EGLContext:%d", ret); } if(this->nativeImage != nullptr) { OH_NativeImage_Destroy(&this->nativeImage); @@ -276,6 +320,7 @@ static void onFrameStart(Camera_VideoOutput* videoOutput) { } static void onFrame(void* context) { + OH_LOG_ERROR(LOG_APP, "视频数据采集回调:%{public}d", 1); // // TODO: 解析 // rtc::scoped_refptr videoFrameBuffer = // webrtc::I420Buffer::Copy(width, height, (uint8_t *)data, 0, (uint8_t *)data, 0, (uint8_t *)data, 0); diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/WebRTC.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/WebRTC.cpp index bda26b5..118adf4 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/WebRTC.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/WebRTC.cpp @@ -31,7 +31,7 @@ bool acgist::TaoyaoAudioTrackSource::remote() const { } void acgist::TaoyaoAudioTrackSource::OnData(const void* audio_data, int bits_per_sample, int sample_rate, size_t number_of_channels, size_t number_of_frames) { - OH_LOG_DEBUG(LOG_APP, "音频数据:%{public}d %{public}d %{public}d %{public}d", bits_per_sample, sample_rate, number_of_channels, number_of_frames); + // OH_LOG_DEBUG(LOG_APP, "音频数据:%{public}d %{public}d %{public}d %{public}d", bits_per_sample, sample_rate, number_of_channels, number_of_frames); for(auto iterator = this->vector.begin(); iterator != this->vector.end(); ++iterator) { (*iterator)->OnData(audio_data, bits_per_sample, sample_rate, number_of_channels, number_of_frames); } @@ -39,7 +39,7 @@ void acgist::TaoyaoAudioTrackSource::OnData(const void* audio_data, int bits_per void acgist::TaoyaoAudioTrackSource::AddSink(webrtc::AudioTrackSinkInterface* sink) { OH_LOG_DEBUG(LOG_APP, "添加本地音频管道"); -// this->vector.push_back(sink); + this->vector.push_back(sink); } void acgist::TaoyaoAudioTrackSource::RemoveSink(webrtc::AudioTrackSinkInterface* sink) { @@ -70,6 +70,7 @@ absl::optional acgist::TaoyaoVideoTrackSource::needs_denoising() const { } void acgist::TaoyaoVideoTrackSource::OnData(const webrtc::VideoFrame& videoFrame) { + OH_LOG_DEBUG(LOG_APP, "视频数据:%{public}d %{public}d", videoFrame.width(), videoFrame.height()); this->OnFrame(videoFrame); } diff --git a/taoyao-client-openharmony/taoyao/media/src/main/ets/pages/Index.ets b/taoyao-client-openharmony/taoyao/media/src/main/ets/pages/Index.ets index 073e1ee..f0d034f 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/ets/pages/Index.ets +++ b/taoyao-client-openharmony/taoyao/media/src/main/ets/pages/Index.ets @@ -9,6 +9,7 @@ import { setting } from "../taoyao/Setting"; const permissions: Array = [ "ohos.permission.CAMERA", + "ohos.permission.INTERNET", "ohos.permission.MICROPHONE", "ohos.permission.READ_MEDIA", "ohos.permission.WRITE_MEDIA", @@ -22,6 +23,10 @@ function reqPermissionsFromUser(permissions: Array, context: common const caText = context.resourceManager.getRawFileContentSync("cacert.pem").buffer; caFile.writeSync(caText); caFile.close(); + // 自动连接信令 + if(setting.initOnLoad) { + taoyaoSignal.connect(); + } }).catch((err: BusinessError) => { }); } diff --git a/taoyao-client-openharmony/taoyao/media/src/main/ets/taoyao/Setting.ets b/taoyao-client-openharmony/taoyao/media/src/main/ets/taoyao/Setting.ets index 2910149..8a802bb 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/ets/taoyao/Setting.ets +++ b/taoyao-client-openharmony/taoyao/media/src/main/ets/taoyao/Setting.ets @@ -30,8 +30,8 @@ class Setting { // 信令版本 version: string = "1.0.0"; // 信令地址 - // signalAddress: string = "wss://192.168.8.57:8888/websocket.signal"; - signalAddress: string = "wss://192.168.1.100:8888/websocket.signal"; + signalAddress: string = "wss://192.168.8.239:8888/websocket.signal"; + // signalAddress: string = "wss://192.168.1.100:8888/websocket.signal"; // 启动自动加载系统 initOnLoad : boolean = true; diff --git a/taoyao-client-openharmony/taoyao/media/src/main/ets/taoyao/TaoyaoSignal.ets b/taoyao-client-openharmony/taoyao/media/src/main/ets/taoyao/TaoyaoSignal.ets index ade9e1b..5e26210 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/ets/taoyao/TaoyaoSignal.ets +++ b/taoyao-client-openharmony/taoyao/media/src/main/ets/taoyao/TaoyaoSignal.ets @@ -92,7 +92,7 @@ class TaoyaoSignal { this.reconnect(); }); // 配置CA证书 - const caPath = getContext(this).cacheDir + setting.caPath; + const caPath = getContext(this).filesDir + setting.caPath; hilog.info(0x0000, "TaoyaoSignal", "配置证书:%{public}s", caPath); const options: webSocket.WebSocketRequestOptions = { caPath: caPath @@ -150,6 +150,7 @@ class TaoyaoSignal { setting.config.clientIndex = index; hilog.info(0x0000, "TaoyaoSignal", "信令注册成功:%{public}d", index); this.heartbeat(); + // 自动加载媒体 if(setting.initOnLoad) { taoyaoSignal.init(); } @@ -369,10 +370,6 @@ class TaoyaoSignal { const taoyaoSignal = new TaoyaoSignal(); -if(setting.initOnLoad) { - taoyaoSignal.connect(); -} - export { taoyaoSignal }