diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/CMakeLists.txt b/taoyao-client-openharmony/taoyao/media/src/main/cpp/CMakeLists.txt index b566a3b..9015f0d 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/CMakeLists.txt +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/CMakeLists.txt @@ -21,7 +21,7 @@ set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -fno-rtti -std=c++17 -O3 set(LIBWEBRTC_BINARY_PATH "${CMAKE_CURRENT_SOURCE_DIR}/deps/webrtc/lib/${OHOS_ARCH}/") set(LIBWEBRTC_INCLUDE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/deps/webrtc/src/") -ADD_DEFINITIONS(-DNDEBUG) +#ADD_DEFINITIONS(-DNDEBUG) ADD_DEFINITIONS(-DLOG_TAG="libtaoyao") ADD_DEFINITIONS(-DVK_USE_PLATFORM_OHOS=1) @@ -64,14 +64,10 @@ target_link_libraries( libEGL.so # OpenGL ES libGLESv3.so - # Vulkan - libvulkan.so # 音频 libohaudio.so # 相机 libohcamera.so - # ACE - libace_ndk.z.so # NAPI libace_napi.z.so # LOG diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Capturer.hpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Capturer.hpp index dccff47..74502ad 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Capturer.hpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Capturer.hpp @@ -19,10 +19,6 @@ #ifndef TAOYAO_CAPTURER_HPP #define TAOYAO_CAPTURER_HPP -// OpenGL ES || VULKAN -#define __TAOYAO_VULKAN__ false -#define __TAOYAO_OPENGL__ true - // 本地音频采集 #define __TAOYAO_AUDIO_LOCAL__ true // 本地视频采集 @@ -32,13 +28,12 @@ #include #include +#include #include #include "./Signal.hpp" #include "./WebRTC.hpp" -#include - #include #include #include @@ -122,17 +117,6 @@ public: class VideoCapturer: public Capturer { public: - // ================ Vulkan ================ - // VK实例 - VkInstance vkInstance = VK_NULL_HANDLE; - // VK Surface - VkSurfaceKHR vkSurfaceKHR = VK_NULL_HANDLE; - // VK应用 - VkApplicationInfo vkApplicationInfo = {}; - // VK实例配置 - VkInstanceCreateInfo vkInstanceCreateInfo = {}; - // VK Surface配置 - VkSurfaceCreateInfoOHOS vkSurfaceCreateInfoOHOS = {}; // ================ OpenGL ES ================ // OpenGL ES SurfaceId uint64_t surfaceId = 0; @@ -147,36 +131,32 @@ public: // EGL Surface EGLSurface eglSurface = EGL_NO_SURFACE; // ================ Camera ================ + // 相机设备数量 + uint32_t cameraSize = 0; + // 相机索引 + uint32_t cameraIndex = 0; // NativeImage OH_NativeImage* nativeImage = nullptr; - // OHNativeWindow - OHNativeWindow* nativeWindow = nullptr; - // 摄像头设备数量 - uint32_t cameraSize = 0; - // 摄像头索引 - uint32_t cameraIndex = 0; - // 摄像头输入 + // 相机输入 Camera_Input* cameraInput = nullptr; - // 摄像头设备列表 + // 相机设备列表 Camera_Device* cameraDevice = nullptr; - // 摄像头管理器 + // 相机管理 Camera_Manager* cameraManager = nullptr; - // 摄像头视频输出 + // 相机视频输出 Camera_VideoOutput* cameraVideoOutput = nullptr; - // 摄像头视频会话 + // 相机预览输出 + Camera_PreviewOutput* cameraPreviewOutput = nullptr; + // 相机视频会话 Camera_CaptureSession* cameraCaptureSession = nullptr; - // 摄像头输出能力 - Camera_OutputCapability *cameraOutputCapability = nullptr; + // 相机输出能力 + Camera_OutputCapability* cameraOutputCapability = nullptr; public: VideoCapturer(); virtual ~VideoCapturer() override; public: - // 加载VK - void initVulkan(); - // 释放VK - void releaseVulkan(); // 加载OpenGL ES void initOpenGLES(); // 释放OpenGL ES diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioCapturer.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioCapturer.cpp index 8cf138e..738124d 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioCapturer.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioCapturer.cpp @@ -76,8 +76,14 @@ static int32_t OnError(OH_AudioCapturer* capturer, void* userData, OH_AudioStrea return 0; } +#include +#include + + // TODO: 删除 + static std::ofstream outfile("/data/storage/el2/base/haps/media/files/audio.pcm"); + static int32_t OnReadData(OH_AudioCapturer* capturer, void* userData, void* buffer, int32_t length) { - // OH_LOG_ERROR(LOG_APP, "音频数据采集回调：%{public}d", length); + OH_LOG_DEBUG(LOG_APP, "音频帧数据采集回调：%{public}d", length); if(userData == nullptr) { return -1; } @@ -87,7 +93,11 @@ static int32_t OnReadData(OH_AudioCapturer* capturer, void* userData, void* buff } // 单声道 48000 / 1000 * 10 * 2(16bit) // 双声道 48000 / 1000 * 10 * 2(16bit) * 2 - audioCapturer->source->OnData(buffer, acgist::bitsPerSample, acgist::samplingRate, acgist::channelCount, length / 2); + // 字节数量 * 8 / 位深 / 通道数量 + size_t number_of_frames = length / 2; + // size_t number_of_frames = length * 8 / 16 / 2; + outfile.write((char*) buffer, length); + audioCapturer->source->OnData((uint16_t*) buffer, acgist::bitsPerSample, acgist::samplingRate, acgist::channelCount, number_of_frames); return 0; } diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/LocalClient.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/LocalClient.cpp index 29a8119..90a0b67 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/LocalClient.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/LocalClient.cpp @@ -2,8 +2,8 @@ acgist::LocalClient::LocalClient(acgist::MediaManager* mediaManager) : acgist::RoomClient(mediaManager) { this->mediaManager->newLocalClient(); - // this->audioTrack = this->mediaManager->getAudioTrack(); - this->videoTrack = this->mediaManager->getVideoTrack(); + this->audioTrack = this->mediaManager->getAudioTrack(); + // this->videoTrack = this->mediaManager->getVideoTrack(); } acgist::LocalClient::~LocalClient() { diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/MediaManager.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/MediaManager.cpp index 96b1e0f..63bd43a 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/MediaManager.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/MediaManager.cpp @@ -87,7 +87,7 @@ int acgist::MediaManager::newLocalClient() { this->newPeerConnectionFactory(); this->startCapture(); } - OH_LOG_INFO(LOG_APP, "打开本地终端：%d", this->localClientRef); + OH_LOG_INFO(LOG_APP, "打开本地终端：%{public}d", this->localClientRef); return this->localClientRef; } @@ -109,8 +109,8 @@ int acgist::MediaManager::releaseLocalClient() { } bool acgist::MediaManager::startCapture() { - // this->startAudioCapture(); - this->startVideoCapture(); + this->startAudioCapture(); + // this->startVideoCapture(); return true; } diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/Room.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/Room.cpp index 6357c6e..9ec2382 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/Room.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/Room.cpp @@ -118,10 +118,10 @@ int acgist::Room::produceMedia() { this->createRecvTransport(); } if(this->audioProduce) { - // this->produceAudio(); + this->produceAudio(); } if(this->videoProduce) { - this->produceVideo(); + // this->produceVideo(); } return 0; } @@ -532,7 +532,7 @@ acgist::SendListener::~SendListener() { } std::future acgist::SendListener::OnConnect(mediasoupclient::Transport* transport, const nlohmann::json& dtlsParameters) { - OH_LOG_INFO(LOG_APP, "连接发送通道：%s - %s", this->room->roomId.data(), transport->GetId().data()); + OH_LOG_INFO(LOG_APP, "连接发送通道：%{public}s - %{public}s", this->room->roomId.data(), transport->GetId().data()); nlohmann::json requestBody = { { "roomId", this->room->roomId }, { "transportId", transport->GetId() }, @@ -545,12 +545,12 @@ std::future acgist::SendListener::OnConnect(mediasoupclient::Transport* tr } void acgist::SendListener::OnConnectionStateChange(mediasoupclient::Transport* transport, const std::string& connectionState) { - OH_LOG_INFO(LOG_APP, "发送通道状态改变：%s - %s - %s", this->room->roomId.data(), transport->GetId().data(), connectionState.data()); + OH_LOG_INFO(LOG_APP, "发送通道状态改变：%{public}s - %{public}s - %{public}s", this->room->roomId.data(), transport->GetId().data(), connectionState.data()); // TODO: 自行实现异常逻辑 } std::future acgist::SendListener::OnProduce(mediasoupclient::SendTransport* transport, const std::string& kind, nlohmann::json rtpParameters, const nlohmann::json& appData) { - OH_LOG_INFO(LOG_APP, "生产媒体：%s - %s - %s", this->room->roomId.data(), transport->GetId().data(), kind.data()); + OH_LOG_INFO(LOG_APP, "生产媒体：%{public}s - %{public}s - %{public}s", this->room->roomId.data(), transport->GetId().data(), kind.data()); nlohmann::json requestBody = { { "kind", kind }, { "roomId", this->room->roomId }, diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoCapturer.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoCapturer.cpp index c937d96..6142443 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoCapturer.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoCapturer.cpp @@ -35,10 +35,25 @@ static std::recursive_mutex videoMutex; -// 采集回调 -static void onError(Camera_VideoOutput* videoOutput, Camera_ErrorCode errorCode); -static void onFrameEnd(Camera_VideoOutput* videoOutput, int32_t frameCount); -static void onFrameStart(Camera_VideoOutput* videoOutput); +// 相机回调 +static void onCameraStatus(Camera_Manager* cameraManager, Camera_StatusInfo* status); + +// 相机输入回调 +static void onInputOnError(const Camera_Input* cameraInput, Camera_ErrorCode errorCode); + +// 预览采集回调 +static void onPreviewError(Camera_PreviewOutput* previewOutput, Camera_ErrorCode errorCode); +static void onPreviewFrameEnd(Camera_PreviewOutput* previewOutput, int32_t frameCount); +static void onPreviewFrameStart(Camera_PreviewOutput* previewOutput); + +// 视频采集回调 +static void onVideoError(Camera_VideoOutput* videoOutput, Camera_ErrorCode errorCode); +static void onVideoFrameEnd(Camera_VideoOutput* videoOutput, int32_t frameCount); +static void onVideoFrameStart(Camera_VideoOutput* videoOutput); + +// 相机会话回调 +static void onSessionError(Camera_CaptureSession* session, Camera_ErrorCode errorCode); +static void onSessionFocusStateChange(Camera_CaptureSession* session, Camera_FocusState focusState); // 数据回调 static void onFrame(void* context); @@ -47,58 +62,90 @@ static void onFrame(void* context); static bool CheckEglExtension(const char* extensions, const char* extension); acgist::VideoCapturer::VideoCapturer() { - #if __TAOYAO_VULKAN__ - initVulkan(); - #endif - #if __TAOYAO_OPENGL__ initOpenGLES(); - #endif Camera_ErrorCode ret = OH_Camera_GetCameraManager(&this->cameraManager); - TAOYAO_VIDEO_RET_LOG("配置摄像头管理器：%{public}d", ret); + TAOYAO_VIDEO_RET_LOG("配置相机管理：%{public}d", ret); + CameraManager_Callbacks cameraManager_Callbacks = { onCameraStatus }; + ret = OH_CameraManager_RegisterCallback(this->cameraManager, &cameraManager_Callbacks); + TAOYAO_VIDEO_RET_LOG("注册相机管理回调：%{public}d", ret); ret = OH_CameraManager_GetSupportedCameras(this->cameraManager, &this->cameraDevice, &this->cameraSize); - TAOYAO_VIDEO_RET_LOG("摄像头设备列表数量：%{public}d %{public}d", ret, this->cameraSize); - ret = OH_CameraManager_CreateCameraInput(this->cameraManager, &this->cameraDevice[this->cameraIndex], &this->cameraInput); - TAOYAO_VIDEO_RET_LOG("配置摄像头输入：%{public}d %{public}d", ret, this->cameraIndex); - ret = OH_CameraInput_Open(cameraInput); - TAOYAO_VIDEO_RET_LOG("打开摄像头：%{public}d", ret); + TAOYAO_VIDEO_RET_LOG("相机设备列表：%{public}d %{public}d", ret, this->cameraSize); + for (int index = 0; index < this->cameraSize; ++index) { + OH_LOG_DEBUG(LOG_APP, "相机cameraId = %{public}s", this->cameraDevice[index].cameraId); + OH_LOG_DEBUG(LOG_APP, "相机cameraPosition = %{public}d", this->cameraDevice[index].cameraPosition); + OH_LOG_DEBUG(LOG_APP, "相机cameraType = %{public}d", this->cameraDevice[index].cameraType); + OH_LOG_DEBUG(LOG_APP, "相机connectionType = %{public}d", this->cameraDevice[index].connectionType); + } ret = OH_CameraManager_GetSupportedCameraOutputCapability(this->cameraManager, &this->cameraDevice[this->cameraIndex], &this->cameraOutputCapability); - TAOYAO_VIDEO_RET_LOG("摄像头输出功能：%{public}d %{public}d %{public}d", ret, this->cameraIndex, this->cameraOutputCapability->videoProfilesSize); - // 注册相机状态回调：可以取消注册 - // OH_CameraManager_RegisterCallback(this->cameraManager, CameraManager_Callbacks* callback); - ret = OH_CameraManager_CreateVideoOutput(this->cameraManager, this->cameraOutputCapability->videoProfiles[0], std::to_string(this->surfaceId).data(), &this->cameraVideoOutput); - TAOYAO_VIDEO_RET_LOG("创建摄像头视频输出：%{public}d %{public}lld %{public}s", ret, this->surfaceId, std::to_string(this->surfaceId).data()); + TAOYAO_VIDEO_RET_LOG("相机输出能力：%{public}d %{public}d %{public}d %{public}d", ret, this->cameraIndex, this->cameraOutputCapability->previewProfilesSize, this->cameraOutputCapability->videoProfilesSize); + ret = OH_CameraManager_CreateCameraInput(this->cameraManager, &this->cameraDevice[this->cameraIndex], &this->cameraInput); + TAOYAO_VIDEO_RET_LOG("配置相机输入：%{public}d %{public}d", ret, this->cameraIndex); + CameraInput_Callbacks cameraInput_Callbacks = { onInputOnError }; + ret = OH_CameraInput_RegisterCallback(this->cameraInput, &cameraInput_Callbacks); + TAOYAO_VIDEO_RET_LOG("注册相机输入回调：%{public}d %{public}d", ret, this->cameraIndex); +// auto& previewProfile = this->cameraOutputCapability->previewProfiles[0]; +// previewProfile->format = CAMERA_FORMAT_YUV_420_SP; + ret = OH_CameraManager_CreatePreviewOutput(this->cameraManager, this->cameraOutputCapability->previewProfiles[0], std::to_string(this->surfaceId).data(), &this->cameraPreviewOutput); + TAOYAO_VIDEO_RET_LOG("创建相机预览输出：%{public}d %{public}lld %{public}s", ret, this->surfaceId, std::to_string(this->surfaceId).data()); + PreviewOutput_Callbacks previewOutput_Callbacks = { onPreviewFrameStart, onPreviewFrameEnd, onPreviewError }; + ret = OH_PreviewOutput_RegisterCallback(this->cameraPreviewOutput, &previewOutput_Callbacks); + TAOYAO_VIDEO_RET_LOG("注册相机预览输出回调：%{public}d", ret); +// auto& videoProfile = this->cameraOutputCapability->videoProfiles[0]; +// videoProfile->format = CAMERA_FORMAT_YUV_420_SP; +// OH_LOG_DEBUG(LOG_APP, "相机视频配置：%{public}d %{public}d %{public}d %{public}d %{public}d", videoProfile->format, videoProfile->size.width, videoProfile->size.height, videoProfile->range.min, videoProfile->range.max); +// ret = OH_CameraManager_CreateVideoOutput(this->cameraManager, this->cameraOutputCapability->videoProfiles[0], std::to_string(this->surfaceId).data(), &this->cameraVideoOutput); +// TAOYAO_VIDEO_RET_LOG("创建相机视频输出：%{public}d %{public}lld %{public}s", ret, this->surfaceId, std::to_string(this->surfaceId).data()); +// VideoOutput_Callbacks videoOutput_Callbacks = { onVideoFrameStart, onVideoFrameEnd, onVideoError }; +// ret = OH_VideoOutput_RegisterCallback(this->cameraVideoOutput, &videoOutput_Callbacks); + TAOYAO_VIDEO_RET_LOG("注册相机视频输出回调：%{public}d", ret); ret = OH_CameraManager_CreateCaptureSession(this->cameraManager, &this->cameraCaptureSession); - TAOYAO_VIDEO_RET_LOG("创建摄像头视频会话：%{public}d", ret); + TAOYAO_VIDEO_RET_LOG("创建相机视频会话：%{public}d", ret); + CaptureSession_Callbacks captureSession_Callbacks = { onSessionFocusStateChange, onSessionError }; + OH_CaptureSession_RegisterCallback(this->cameraCaptureSession, &captureSession_Callbacks); + TAOYAO_VIDEO_RET_LOG("注册相机视频会话：%{public}d", ret); // 设置相机：闪光、变焦、质量、高宽、补光、防抖 // 设置缩放比例 // OH_CaptureSession_SetZoomRatio(this->cameraCaptureSession, 0.5F); } acgist::VideoCapturer::~VideoCapturer() { - #if __TAOYAO_VULKAN__ - releaseVulkan(); - #endif - #if __TAOYAO_OPENGL__ releaseOpenGLES(); - #endif - Camera_ErrorCode ret = OH_CaptureSession_Release(this->cameraCaptureSession); + CaptureSession_Callbacks captureSession_Callbacks = { onSessionFocusStateChange, onSessionError }; + Camera_ErrorCode ret = OH_CaptureSession_UnregisterCallback(this->cameraCaptureSession, &captureSession_Callbacks); + TAOYAO_VIDEO_RET_LOG("取消相机视频会话：%{public}d", ret); + ret = OH_CaptureSession_Release(this->cameraCaptureSession); this->cameraCaptureSession = nullptr; - TAOYAO_VIDEO_RET_LOG("释放摄像头视频会话：%{public}o", ret); + TAOYAO_VIDEO_RET_LOG("释放相机视频会话：%{public}d", ret); + PreviewOutput_Callbacks previewOutput_Callbacks = { onPreviewFrameStart, onPreviewFrameEnd, onPreviewError }; + ret = OH_PreviewOutput_UnregisterCallback(this->cameraPreviewOutput, &previewOutput_Callbacks); + TAOYAO_VIDEO_RET_LOG("取消相机预览输出回调：%{public}d", ret); + ret = OH_PreviewOutput_Release(this->cameraPreviewOutput); + this->cameraPreviewOutput = nullptr; + TAOYAO_VIDEO_RET_LOG("释放相机预览输出：%{public}d", ret); + VideoOutput_Callbacks videoOutput_Callbacks = { onVideoFrameStart, onVideoFrameEnd, onVideoError }; + ret = OH_VideoOutput_UnregisterCallback(this->cameraVideoOutput, &videoOutput_Callbacks); + TAOYAO_VIDEO_RET_LOG("取消相机视频输出回调：%{public}d", ret); ret = OH_VideoOutput_Release(this->cameraVideoOutput); this->cameraVideoOutput = nullptr; + TAOYAO_VIDEO_RET_LOG("释放相机视频输出：%{public}d", ret); + CameraInput_Callbacks cameraInput_Callbacks = { onInputOnError }; + ret = OH_CameraInput_UnregisterCallback(this->cameraInput, &cameraInput_Callbacks); + TAOYAO_VIDEO_RET_LOG("取消相机输入回调：%{public}d", ret); ret = OH_CameraInput_Release(this->cameraInput); this->cameraInput = nullptr; - TAOYAO_VIDEO_RET_LOG("释放摄像头：%{public}o", ret); - TAOYAO_VIDEO_RET_LOG("释放摄像头视频输出：%{public}o", ret); + TAOYAO_VIDEO_RET_LOG("释放相机输入：%{public}d", ret); ret = OH_CameraManager_DeleteSupportedCameraOutputCapability(this->cameraManager, this->cameraOutputCapability); this->cameraOutputCapability = nullptr; - TAOYAO_VIDEO_RET_LOG("释放摄像头输出能力：%{public}d", ret); + TAOYAO_VIDEO_RET_LOG("释放相机输出能力：%{public}d", ret); ret = OH_CameraManager_DeleteSupportedCameras(this->cameraManager, this->cameraDevice, this->cameraSize); this->cameraDevice = nullptr; - TAOYAO_VIDEO_RET_LOG("释放摄像头设备列表：%{public}d", ret); + TAOYAO_VIDEO_RET_LOG("释放相机设备列表：%{public}d", ret); + CameraManager_Callbacks cameraManager_Callbacks = { onCameraStatus }; + ret = OH_CameraManager_UnregisterCallback(this->cameraManager, &cameraManager_Callbacks); + TAOYAO_VIDEO_RET_LOG("取消相机管理回调：%{public}d", ret); ret = OH_Camera_DeleteCameraManager(this->cameraManager); this->cameraManager = nullptr; - TAOYAO_VIDEO_RET_LOG("释放摄像头管理器：%{public}d", ret); + TAOYAO_VIDEO_RET_LOG("释放相机管理：%{public}d", ret); } bool acgist::VideoCapturer::start() { @@ -107,26 +154,25 @@ bool acgist::VideoCapturer::start() { return true; } this->running = true; + Camera_ErrorCode ret = OH_CameraInput_Open(this->cameraInput); + TAOYAO_VIDEO_RET_LOG("打开相机输入：%{public}d", ret); OH_NativeImage_AttachContext(this->nativeImage, this->textureId); - Camera_ErrorCode ret = OH_CaptureSession_BeginConfig(this->cameraCaptureSession); - TAOYAO_VIDEO_RET_LOG("开始配置视频会话：%{public}o", ret); + ret = OH_CaptureSession_BeginConfig(this->cameraCaptureSession); + TAOYAO_VIDEO_RET_LOG("开始配置视频会话：%{public}d", ret); ret = OH_CaptureSession_AddInput(this->cameraCaptureSession, this->cameraInput); - TAOYAO_VIDEO_RET_LOG("视频输入绑定会话：%{public}d", ret); - ret = OH_CaptureSession_AddVideoOutput(this->cameraCaptureSession, this->cameraVideoOutput); - TAOYAO_VIDEO_RET_LOG("视频捕获绑定会话：%{public}d", ret); + TAOYAO_VIDEO_RET_LOG("绑定视频输入会话：%{public}d", ret); + ret = OH_CaptureSession_AddPreviewOutput(cameraCaptureSession, this->cameraPreviewOutput); + TAOYAO_VIDEO_RET_LOG("绑定相机预览输出会话：%{public}d", ret); +// ret = OH_CaptureSession_AddVideoOutput(this->cameraCaptureSession, this->cameraVideoOutput); +// TAOYAO_VIDEO_RET_LOG("绑定相机视频输出会话：%{public}d", ret); ret = OH_CaptureSession_CommitConfig(this->cameraCaptureSession); - TAOYAO_VIDEO_RET_LOG("结束配置视频会话：%{public}o", ret); + TAOYAO_VIDEO_RET_LOG("开始相机视频会话：%{public}d", ret); ret = OH_CaptureSession_Start(this->cameraCaptureSession); - TAOYAO_VIDEO_RET_LOG("开始视频会话：%{public}d", ret); + TAOYAO_VIDEO_RET_LOG("开始相机视频输出：%{public}d", ret); +// ret = OH_PreviewOutput_Start(this->cameraPreviewOutput); +// TAOYAO_VIDEO_RET_LOG("开始相机预览输出：%{public}d", ret); ret = OH_VideoOutput_Start(this->cameraVideoOutput); - TAOYAO_VIDEO_RET_LOG("开始视频捕获：%{public}d", ret); - VideoOutput_Callbacks callbacks; - callbacks.onError = onError; - callbacks.onFrameEnd = onFrameEnd; - callbacks.onFrameStart = onFrameStart; - // 可以取消注册 - ret = OH_VideoOutput_RegisterCallback(this->cameraVideoOutput, &callbacks); - TAOYAO_VIDEO_RET_LOG("视频捕获回调：%{public}d", ret); + TAOYAO_VIDEO_RET_LOG("结束配置视频会话：%{public}d", ret); return ret = Camera_ErrorCode::CAMERA_OK; } @@ -137,64 +183,27 @@ bool acgist::VideoCapturer::stop() { } this->running = false; Camera_ErrorCode ret = OH_CaptureSession_BeginConfig(this->cameraCaptureSession); - TAOYAO_VIDEO_RET_LOG("开始配置视频会话：%{public}o", ret); + TAOYAO_VIDEO_RET_LOG("开始配置视频会话：%{public}d", ret); ret = OH_CaptureSession_RemoveInput(this->cameraCaptureSession, this->cameraInput); - TAOYAO_VIDEO_RET_LOG("视频输入取消绑定会话：%{public}o", ret); + TAOYAO_VIDEO_RET_LOG("取消绑定视频输入会话：%{public}d", ret); + ret = OH_CaptureSession_RemovePreviewOutput(this->cameraCaptureSession, this->cameraPreviewOutput); + TAOYAO_VIDEO_RET_LOG("绑定相机预览输出会话：%{public}d", ret); ret = OH_CaptureSession_RemoveVideoOutput(this->cameraCaptureSession, this->cameraVideoOutput); - TAOYAO_VIDEO_RET_LOG("视频捕获取消绑定会话：%{public}o", ret); + TAOYAO_VIDEO_RET_LOG("绑定相机视频输出会话：%{public}d", ret); ret = OH_CaptureSession_CommitConfig(this->cameraCaptureSession); - TAOYAO_VIDEO_RET_LOG("结束配置视频会话：%{public}o", ret); + TAOYAO_VIDEO_RET_LOG("结束配置视频会话：%{public}d", ret); ret = OH_CaptureSession_Stop(this->cameraCaptureSession); - TAOYAO_VIDEO_RET_LOG("结束视频会话：%{public}o", ret); + TAOYAO_VIDEO_RET_LOG("结束视频会话：%{public}d", ret); + ret = OH_PreviewOutput_Stop(this->cameraPreviewOutput); + TAOYAO_VIDEO_RET_LOG("结束相机预览输出：%{public}d", ret); ret = OH_VideoOutput_Stop(this->cameraVideoOutput); - // OH_VideoOutput_UnregisterCallback(this->cameraVideoOutput, callback); - TAOYAO_VIDEO_RET_LOG("结束视频捕获：%{public}o", ret); + TAOYAO_VIDEO_RET_LOG("结束相机视频输出：%{public}d", ret); ret = OH_CameraInput_Close(this->cameraInput); - TAOYAO_VIDEO_RET_LOG("关闭摄像头：%{public}o", ret); + TAOYAO_VIDEO_RET_LOG("关闭相机输入：%{public}d", ret); OH_NativeImage_DetachContext(this->nativeImage); return ret = Camera_ErrorCode::CAMERA_OK; } -void acgist::VideoCapturer::initVulkan() { - // vkApplicationInfo - this->vkApplicationInfo.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO; - this->vkApplicationInfo.apiVersion = VK_API_VERSION_1_3; - this->vkApplicationInfo.pEngineName = "vulkan-taoyao"; - this->vkApplicationInfo.pApplicationName = "vulkan-taoyao"; - OH_LOG_INFO(LOG_APP, "配置vkApplicationInfo"); - // vkInstanceCreateInfo - this->vkInstanceCreateInfo.sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO; - this->vkInstanceCreateInfo.pNext = NULL; - this->vkInstanceCreateInfo.pApplicationInfo = &this->vkApplicationInfo; - OH_LOG_INFO(LOG_APP, "配置vkInstanceCreateInfo"); - // vkInstanceCreateInfo - std::vector instanceExtensions = { - VK_KHR_SURFACE_EXTENSION_NAME, - VK_OHOS_SURFACE_EXTENSION_NAME - }; - vkInstanceCreateInfo.enabledExtensionCount = static_cast(instanceExtensions.size()); - vkInstanceCreateInfo.ppEnabledExtensionNames = instanceExtensions.data(); - OH_LOG_INFO(LOG_APP, "配置vkInstanceCreateInfo"); - VkResult ret = vkCreateInstance(&this->vkInstanceCreateInfo, nullptr, &this->vkInstance); - OH_LOG_INFO(LOG_APP, "加载vkInstance：%d", ret); - this->vkSurfaceCreateInfoOHOS.sType = VK_STRUCTURE_TYPE_SURFACE_CREATE_INFO_OHOS; - this->vkSurfaceCreateInfoOHOS.window = this->nativeWindow; - OH_LOG_INFO(LOG_APP, "配置vkSurfaceCreateInfoOHOS"); - ret = vkCreateSurfaceOHOS(this->vkInstance, &this->vkSurfaceCreateInfoOHOS, nullptr, &this->vkSurfaceKHR); - OH_LOG_INFO(LOG_APP, "加载vkSurfaceKHR：%d", ret); -} - -void acgist::VideoCapturer::releaseVulkan() { - if(this->vkSurfaceKHR != VK_NULL_HANDLE) { - vkDestroySurfaceKHR(this->vkInstance, this->vkSurfaceKHR, nullptr); - this->vkSurfaceKHR = VK_NULL_HANDLE; - } - if(this->vkInstance != VK_NULL_HANDLE) { - vkDestroyInstance(this->vkInstance, nullptr); - this->vkInstance = VK_NULL_HANDLE; - } -} - void acgist::VideoCapturer::initOpenGLES() { // EGL static const char* EGL_EXT_PLATFORM_WAYLAND = "EGL_EXT_platform_wayland"; @@ -220,14 +229,13 @@ void acgist::VideoCapturer::initOpenGLES() { EGLint count; EGLConfig config; EGLint config_attribs[] = { - EGL_RENDERABLE_TYPE, EGL_OPENGL_BIT, + EGL_RENDERABLE_TYPE, EGL_OPENGL_ES3_BIT, +// EGL_SURFACE_TYPE, EGL_PIXMAP_BIT, EGL_SURFACE_TYPE, EGL_PBUFFER_BIT, EGL_RED_SIZE, 8, EGL_GREEN_SIZE, 8, EGL_BLUE_SIZE, 8, EGL_ALPHA_SIZE, 8, - EGL_DEPTH_SIZE, 0, - EGL_STENCIL_SIZE, 0, EGL_NONE }; EGLint context_attribs[] = { @@ -249,16 +257,13 @@ void acgist::VideoCapturer::initOpenGLES() { this->eglContext = eglCreateContext(this->eglDisplay, config, EGL_NO_CONTEXT, context_attribs); } const EGLint surfaceAttr[] = { - EGL_WIDTH, 512, - EGL_HEIGHT, 512, + EGL_WIDTH, 1980, + EGL_HEIGHT, 1080, EGL_LARGEST_PBUFFER, EGL_TRUE, EGL_NONE }; - // TODO: 验证强转 - this->eglSurface = eglCreatePbufferSurface(this->eglDisplay, config, surfaceAttr); -// this->eglSurface = eglCreateWindowSurface(this->eglDisplay, config, (EGLNativeWindowType) this->nativeWindow, context_attribs); - + this->eglSurface = eglCreatePbufferSurface(this->eglDisplay, config, surfaceAttr); eglMakeCurrent(this->eglDisplay, this->eglSurface, this->eglSurface, this->eglContext); @@ -266,13 +271,6 @@ void acgist::VideoCapturer::initOpenGLES() { glGenTextures(this->textureSize, &this->textureId); this->nativeImage = OH_NativeImage_Create(this->textureId, GL_TEXTURE_2D); OH_LOG_INFO(LOG_APP, "创建NativeImage：%{public}d", this->textureId); - // TODO: 验证是否需要window -// this->nativeWindow = OH_NativeImage_AcquireNativeWindow(this->nativeImage); -// OH_LOG_INFO(LOG_APP, "视频采集Window：%{public}d", this->nativeWindow); - // this->eglSurface = eglCreatePlatformWindowSurface(this->eglDisplay, config, this->nativeWindow, context_attribs); -// OH_NativeWindow_NativeWindowHandleOpt(this->nativeWindow, SET_BUFFER_GEOMETRY, acgist::width, acgist::height); - // OH_NativeWindow_NativeWindowHandleOpt(this->nativeWindow, SET_TIMEOUT, 10'000); - // OH_NativeWindow_NativeWindowHandleOpt(this->nativeWindow, GET_TIMEOUT, 10'000); OH_OnFrameAvailableListener listener = { this, onFrame }; OH_NativeImage_SetOnFrameAvailableListener(this->nativeImage, listener); @@ -300,27 +298,148 @@ void acgist::VideoCapturer::releaseOpenGLES() { this->nativeImage = nullptr; OH_LOG_INFO(LOG_APP, "销毁nativeImage"); } - if(this->nativeWindow != nullptr) { - OH_NativeWindow_DestroyNativeWindow(this->nativeWindow); - this->nativeWindow = nullptr; - OH_LOG_INFO(LOG_APP, "销毁nativeWindow"); - } } -static void onError(Camera_VideoOutput* videoOutput, Camera_ErrorCode errorCode) { +static void onCameraStatus(Camera_Manager* cameraManager, Camera_StatusInfo* status) { + OH_LOG_INFO(LOG_APP, "相机状态变化：%{public}d", status->status); +} + +static void onInputOnError(const Camera_Input* cameraInput, Camera_ErrorCode errorCode) { + OH_LOG_ERROR(LOG_APP, "相机输入错误：%{public}d", errorCode); +} + +static void onVideoError(Camera_VideoOutput* videoOutput, Camera_ErrorCode errorCode) { OH_LOG_ERROR(LOG_APP, "视频捕获数据帧失败：%d", errorCode); } -static void onFrameEnd(Camera_VideoOutput* videoOutput, int32_t frameCount) { +static void onVideoFrameEnd(Camera_VideoOutput* videoOutput, int32_t frameCount) { OH_LOG_DEBUG(LOG_APP, "结束视频捕获数据帧"); } -static void onFrameStart(Camera_VideoOutput* videoOutput) { +static void onVideoFrameStart(Camera_VideoOutput* videoOutput) { OH_LOG_DEBUG(LOG_APP, "开始视频捕获数据帧"); } +static void onPreviewError(Camera_PreviewOutput* previewOutput, Camera_ErrorCode errorCode) { + OH_LOG_ERROR(LOG_APP, "预览捕获数据帧失败：%d", errorCode); +} + +static void onPreviewFrameEnd(Camera_PreviewOutput* previewOutput, int32_t frameCount) { + OH_LOG_DEBUG(LOG_APP, "结束预览捕获数据帧"); +} + +static void onPreviewFrameStart(Camera_PreviewOutput* previewOutput) { + OH_LOG_DEBUG(LOG_APP, "开始预览捕获数据帧"); +} + +static void onSessionError(Camera_CaptureSession* session, Camera_ErrorCode errorCode) { + OH_LOG_ERROR(LOG_APP, "打开相机会话失败：%{public}o", errorCode); +} + +static void onSessionFocusStateChange(Camera_CaptureSession* session, Camera_FocusState focusState) { + OH_LOG_DEBUG(LOG_APP, "相机会话焦点改变：%{public}o", focusState); +} + static void onFrame(void* context) { - OH_LOG_ERROR(LOG_APP, "视频数据采集回调：%{public}d", 1); + OH_LOG_DEBUG(LOG_APP, "视频帧数据采集回调：%{public}d", 1); + acgist::VideoCapturer* videoCapturer = (acgist::VideoCapturer*) context; +// OH_NativeImage_UpdateSurfaceImage(videoCapturer->nativeImage); + // 更新内容到OpenGL纹理。 +uint32_t ret = OH_NativeImage_UpdateSurfaceImage(videoCapturer->nativeImage); +if (ret != 0) { +} +// 获取最近调用OH_NativeImage_UpdateSurfaceImage的纹理图像的时间戳和变化矩阵。 +int64_t timeStamp = OH_NativeImage_GetTimestamp(videoCapturer->nativeImage); +float matrix[16]; +ret = OH_NativeImage_GetTransformMatrix(videoCapturer->nativeImage, matrix); +if (ret != 0) { +} + +// 对update绑定到对应textureId的纹理做对应的opengl后处理后，将纹理上屏 +EGLBoolean eglRet = eglSwapBuffers(videoCapturer->eglDisplay, videoCapturer->eglSurface); +if (eglRet == EGL_FALSE) { +} +//HWTEST_F(NativeImageTest, OHNativeImageSetOnFrameAvailableListener001, Function | MediumTest | Level1) +//{ +// if (image == nullptr) { +// image = OH_NativeImage_Create(textureId, GL_TEXTURE_2D); +// ASSERT_NE(image, nullptr); +// } +// +// if (nativeWindow == nullptr) { +// nativeWindow = OH_NativeImage_AcquireNativeWindow(image); +// ASSERT_NE(nativeWindow, nullptr); +// } +// +// OH_OnFrameAvailableListener listener; +// listener.context = this; +// listener.onFrameAvailable = NativeImageTest::OnFrameAvailable; +// int32_t ret = OH_NativeImage_SetOnFrameAvailableListener(image, listener); +// ASSERT_EQ(ret, GSERROR_OK); +// +// NativeWindowBuffer* nativeWindowBuffer = nullptr; +// int fenceFd = -1; +// ret = OH_NativeWindow_NativeWindowRequestBuffer(nativeWindow, &nativeWindowBuffer, &fenceFd); +// ASSERT_EQ(ret, GSERROR_OK); +// +// struct Region *region = new Region(); +// struct Region::Rect *rect = new Region::Rect(); +// rect->x = 0x100; +// rect->y = 0x100; +// rect->w = 0x100; +// rect->h = 0x100; +// region->rects = rect; +// ret = OH_NativeWindow_NativeWindowFlushBuffer(nativeWindow, nativeWindowBuffer, fenceFd, *region); +// ASSERT_EQ(ret, GSERROR_OK); +// delete region; +// +// ret = OH_NativeImage_UpdateSurfaceImage(image); +// ASSERT_EQ(ret, SURFACE_ERROR_OK); +//} +// OHNativeWindow* nativeWindow = OH_NativeImage_AcquireNativeWindow(videoCapturer->nativeImage); +// int code = SET_BUFFER_GEOMETRY; +// int32_t width = 800; +// int32_t height = 600; +// int32_t ret = OH_NativeWindow_NativeWindowHandleOpt(nativeWindow, code, width, height); +// OHNativeWindowBuffer *buffer = nullptr; +// int fenceFd; +// // 通过 OH_NativeWindow_NativeWindowRequestBuffer 获取 OHNativeWindowBuffer 实例 +// OH_NativeWindow_NativeWindowRequestBuffer(nativeWindow, &buffer, &fenceFd); +// BufferHandle *handle = OH_NativeWindow_GetBufferHandleFromNative(buffer); +// // 设置刷新区域，如果Region中的Rect为nullptr,或者rectNumber为0，则认为NativeWindowBuffer全部有内容更改。 +//Region region{nullptr, 0}; +//// 通过OH_NativeWindow_NativeWindowFlushBuffer 提交给消费者使用，例如：显示在屏幕上。 +//OH_NativeWindow_NativeWindowFlushBuffer(nativeWindow, buffer, fenceFd, region); +// // 更新内容到OpenGL纹理。 +// ret = OH_NativeImage_UpdateSurfaceImage(videoCapturer->nativeImage); +//if (ret != 0) { +//} +//// 获取最近调用OH_NativeImage_UpdateSurfaceImage的纹理图像的时间戳和变化矩阵。 +//int64_t timeStamp = OH_NativeImage_GetTimestamp(videoCapturer->nativeImage); +//float matrix[16]; +//ret = OH_NativeImage_GetTransformMatrix(videoCapturer->nativeImage, matrix); +//if (ret != 0) { +//} +// +//// 对update绑定到对应textureId的纹理做对应的opengl后处理后，将纹理上屏 +//EGLBoolean eglRet = eglSwapBuffers(videoCapturer->eglDisplay, videoCapturer->eglSurface); +//if (eglRet == EGL_FALSE) { +//} + +// eglMakeCurrent(videoCapturer->eglDisplay, videoCapturer->eglSurface, videoCapturer->eglSurface, videoCapturer->eglContext); +// OH_NativeImage_UpdateSurfaceImage(videoCapturer->nativeImage); +// OH_NativeImage_GetTimestamp(videoCapturer->nativeImage); +// float matrix[16]; +// OH_NativeImage_GetTransformMatrix(videoCapturer->nativeImage, matrix); +// eglSwapBuffers(videoCapturer->eglDisplay, videoCapturer->eglSurface); +// char* chars = new char[1024]; +// eglQueryNativePixmapNV(videoCapturer->eglDisplay, videoCapturer->eglSurface, &pixmap); +// EGLBoolean ret = eglCopyBuffers(videoCapturer->eglDisplay, videoCapturer->eglSurface, (EGLNativePixmapType) chars); +// char buffer[1024]; +// glReadPixels(0, 0, 10, 10, GL_RGBA, GL_UNSIGNED_BYTE, buffer); +// OH_LOG_DEBUG(LOG_APP, "视频帧数据采集回调读取：%{public}d", ret); +// glFlush(); +// glFinish(); // // TODO: 解析 // rtc::scoped_refptr videoFrameBuffer = // webrtc::I420Buffer::Copy(width, height, (uint8_t *)data, 0, (uint8_t *)data, 0, (uint8_t *)data, 0); diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/WebRTC.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/WebRTC.cpp index 118adf4..c2fc8a0 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/WebRTC.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/WebRTC.cpp @@ -31,7 +31,7 @@ bool acgist::TaoyaoAudioTrackSource::remote() const { } void acgist::TaoyaoAudioTrackSource::OnData(const void* audio_data, int bits_per_sample, int sample_rate, size_t number_of_channels, size_t number_of_frames) { - // OH_LOG_DEBUG(LOG_APP, "音频数据：%{public}d %{public}d %{public}d %{public}d", bits_per_sample, sample_rate, number_of_channels, number_of_frames); + OH_LOG_DEBUG(LOG_APP, "音频数据：%{public}d %{public}d %{public}d %{public}d", bits_per_sample, sample_rate, number_of_channels, number_of_frames); for(auto iterator = this->vector.begin(); iterator != this->vector.end(); ++iterator) { (*iterator)->OnData(audio_data, bits_per_sample, sample_rate, number_of_channels, number_of_frames); } diff --git a/taoyao-client-openharmony/taoyao/media/src/main/ets/taoyao/Setting.ets b/taoyao-client-openharmony/taoyao/media/src/main/ets/taoyao/Setting.ets index 8a802bb..c162913 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/ets/taoyao/Setting.ets +++ b/taoyao-client-openharmony/taoyao/media/src/main/ets/taoyao/Setting.ets @@ -30,7 +30,7 @@ class Setting { // 信令版本 version: string = "1.0.0"; // 信令地址 - signalAddress: string = "wss://192.168.8.239:8888/websocket.signal"; + signalAddress: string = "wss://192.168.8.20:8888/websocket.signal"; // signalAddress: string = "wss://192.168.1.100:8888/websocket.signal"; // 启动自动加载系统 initOnLoad : boolean = true;