diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/CMakeLists.txt b/taoyao-client-openharmony/taoyao/media/src/main/cpp/CMakeLists.txt index 2166adc..191bec4 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/CMakeLists.txt +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/CMakeLists.txt @@ -8,14 +8,14 @@ project(taoyao VERSION 1.0.0 LANGUAGES C CXX) # C编译选项 set(CMAKE_C_STANDARD 17) -#set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c17 -O3") -set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -std=c17 -O0 -g") +#set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c17 -O3") +set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -std=c17 -O0 -g") set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -std=c17 -O3") # C++编译选项 set(CMAKE_CXX_STANDARD 17) -#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -O3") -set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -std=c++17 -O0 -g") +#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -O3") +set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -std=c++17 -O0 -g") set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -std=c++17 -O3") option(MEDIASOUPCLIENT_LOG_DEV OFF) @@ -24,6 +24,8 @@ option(MEDIASOUPCLIENT_BUILD_TESTS OFF) set(LIBWEBRTC_BINARY_PATH "${CMAKE_CURRENT_SOURCE_DIR}/deps/webrtc/lib/") set(LIBWEBRTC_INCLUDE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/deps/webrtc/src/") +ADD_DEFINITIONS(-DVK_USE_PLATFORM_OHOS=1) + add_subdirectory("./deps/libmediasoupclient") add_library( @@ -38,6 +40,7 @@ add_library( media/Player.cpp media/AudioPlayer.cpp media/VideoPlayer.cpp + media/VideoDecoder.cpp media/Capturer.cpp media/AudioCapturer.cpp media/VideoCapturer.cpp @@ -61,21 +64,26 @@ target_link_libraries( libace_napi.z.so # LOG libhilog_ndk.z.so + # EGL + libEGL.so + # OpenGL ES + libGLESv3.so + # Vulkan + libvulkan.so # 音频 libohaudio.so # 相机 libohcamera.so + # 图片 + libnative_image.so # 编码解码 libnative_media_aenc.so libnative_media_venc.so libnative_media_core.so libnative_media_codecbase.so - # EGL - libEGL.so - libGLESv3.so - # 图片 - libnative_image.so + # NativeBuffer libnative_buffer.so + # NativeWindow libnative_window.so ) diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Capturer.hpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Capturer.hpp index 8544b65..68f8df6 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Capturer.hpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Capturer.hpp @@ -19,6 +19,11 @@ #ifndef TAOYAO_CAPTURER_HPP #define TAOYAO_CAPTURER_HPP +#define __VULKAN__ true +#ifndef __VULKAN__ +#define __OPENGL__ true +#endif + #include #include @@ -27,6 +32,8 @@ #include "./Signal.hpp" +#include + #include #include #include @@ -40,8 +47,6 @@ #include #include -#include - namespace acgist { /** @@ -128,63 +133,36 @@ public: }; -/** - * 视频编码 - */ -class VideoEncoder { - -public: - // 视频编码器 - OH_AVCodec* avCodec = nullptr; - // 视频窗口 - OHNativeWindow* nativeWindow = nullptr; - -public: - VideoEncoder(); - virtual ~VideoEncoder(); - -public: - // 初始配置 - void initFormatConfig(OH_AVFormat* format); - // 重新开始 - void restart(); - // 动态配置 - void reset(OH_AVFormat* format); - // 动态配置 - void resetIntConfig(const char* key, int32_t value); - // 动态配置 - void resetLongConfig(const char* key, int64_t value); - // 动态配置 - void resetDoubleConfig(const char* key, double value); - // 开始编码 - virtual bool start(); - // 结束编码 - virtual bool stop(); - -}; - /** * 视频采集器 */ class VideoCapturer: public Capturer> { public: + // ================ Vulkan ================ + VkInstance vkInstance = VK_NULL_HANDLE; + VkSurfaceKHR vkSurfaceKHR = VK_NULL_HANDLE; + VkApplicationInfo vkApplicationInfo = {}; + VkInstanceCreateInfo vkInstanceCreateInfo = {}; + VkSurfaceCreateInfoOHOS vkSurfaceCreateInfoOHOS = {}; + // ================ OpenGL ES ================ // SurfaceId uint64_t surfaceId = 0; // OpenGL纹理指针 GLuint textureId = 0; // OpenGL纹理数量 GLsizei textureSize = 1; - // NativeImage - OH_NativeImage* nativeImage = nullptr; - // OHNativeWindow - OHNativeWindow* nativeWindow = nullptr; // EGL显示设备 EGLDisplay eglDisplay = EGL_NO_DISPLAY; // EGL上下文 EGLContext eglContext = EGL_NO_CONTEXT; // EGL Surface EGLSurface eglSurface = EGL_NO_SURFACE; + // ================ Camera ================ + // NativeImage + OH_NativeImage* nativeImage = nullptr; + // OHNativeWindow + OHNativeWindow* nativeWindow = nullptr; // 摄像头设备数量 uint32_t cameraSize = 0; // 摄像头索引 @@ -205,6 +183,8 @@ public: virtual ~VideoCapturer(); public: + void initVulkan(); + void releaseVulkan(); void initOpenGLES(); void releaseOpenGLES(); virtual bool start() override; diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/WebRTC.hpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/WebRTC.hpp new file mode 100644 index 0000000..0ae564e --- /dev/null +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/WebRTC.hpp @@ -0,0 +1,105 @@ +#ifndef TAOYAO_WEBRTC_HPP +#define TAOYAO_WEBRTC_HPP + +#include "./Signal.hpp" + +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "api/rtp_sender_interface.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/audio_codecs/builtin_audio_decoder_factory.h" +#include "api/audio_codecs/builtin_audio_encoder_factory.h" +#include "api/video_codecs/builtin_video_decoder_factory.h" +#include "api/video_codecs/builtin_video_encoder_factory.h" + +#include "modules/video_coding/codecs/vp8/include/vp8.h" +#include "modules/video_coding/codecs/vp9/include/vp9.h" +#include "modules/video_coding/codecs/h264/include/h264.h" + +namespace acgist { + +/** + * 视频编码 + */ +class VideoEncoder : public webrtc::VideoEncoder { + +public: + // 视频编码器 + OH_AVCodec* avCodec = nullptr; + // 视频窗口 + OHNativeWindow* nativeWindow = nullptr; + +public: + VideoEncoder(); + virtual ~VideoEncoder(); + +public: + // 初始配置 + void initFormatConfig(OH_AVFormat* format); + // 重新开始 + void restart(); + // 动态配置 + void reset(OH_AVFormat* format); + // 动态配置 + void resetIntConfig(const char* key, int32_t value); + // 动态配置 + void resetLongConfig(const char* key, int64_t value); + // 动态配置 + void resetDoubleConfig(const char* key, double value); + // 开始编码 + virtual bool start(); + // 结束编码 + virtual bool stop(); + +}; + +/** + * 视频解码器 + */ +class VideoDecoder : public webrtc::VideoDecoder { + +public: + VideoDecoder(); + virtual ~VideoDecoder(); + +public: + virtual bool start(); + virtual bool stop(); + +}; + +class TaoyaoVideoEncoderFactory : webrtc::VideoEncoderFactory { + +public: + TaoyaoVideoEncoderFactory(); + virtual ~TaoyaoVideoEncoderFactory(); + +public: + virtual std::vector GetSupportedFormats() const override; + virtual std::unique_ptr CreateVideoEncoder(const webrtc::SdpVideoFormat& format) override; + +}; + +class TaoyaoVideoDecoderFactory : webrtc::VideoDecoderFactory { + +public: + TaoyaoVideoDecoderFactory(); + virtual ~TaoyaoVideoDecoderFactory(); + +public: + virtual std::vector GetSupportedFormats() const override; + virtual std::unique_ptr CreateVideoDecoder(const webrtc::SdpVideoFormat& format) override; + +}; + +} + +#endif //TAOYAO_WEBRTC_HPP diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioPlayer.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioPlayer.cpp index aa877d5..296fc6c 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioPlayer.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioPlayer.cpp @@ -1,10 +1,10 @@ #include "../include/Player.hpp" +#include + #include #include -#include - // 播放回调 static int32_t OnError(OH_AudioRenderer* renderer, void* userData, OH_AudioStream_Result error); static int32_t OnWriteData(OH_AudioRenderer* renderer, void* userData, void* buffer, int32_t length); diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoCapturer.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoCapturer.cpp index 0db28e1..40943ed 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoCapturer.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoCapturer.cpp @@ -42,6 +42,9 @@ acgist::VideoCapturer::VideoCapturer() { OH_LOG_INFO(LOG_APP, "获取摄像头输出功能：%o %d %d", ret, this->cameraIndex, this->cameraOutputCapability->videoProfilesSize); // 注册相机状态回调 // OH_CameraManager_RegisterCallback(this->cameraManager, CameraManager_Callbacks* callback); +// char surfaceId[OH_XCOMPONENT_ID_LEN_MAX + 1] = {}; +// uint64_t idSize = OH_XCOMPONENT_ID_LEN_MAX + 1; +// OH_NativeXComponent_GetXComponentId((OH_NativeXComponent*) this->nativeWindow, surfaceId, idSize); ret = OH_CameraManager_CreateVideoOutput(this->cameraManager, this->cameraOutputCapability->videoProfiles[0], (char*) this->surfaceId, &this->cameraVideoOutput); OH_LOG_INFO(LOG_APP, "创建摄像头视频输出：%o", ret); ret = OH_CameraManager_CreateCaptureSession(this->cameraManager, &this->cameraCaptureSession); @@ -111,6 +114,46 @@ bool acgist::VideoCapturer::stop() { return ret = Camera_ErrorCode::CAMERA_OK; } +void acgist::VideoCapturer::initVulkan() { + // vkApplicationInfo + this->vkApplicationInfo.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO; + this->vkApplicationInfo.apiVersion = VK_API_VERSION_1_3; + this->vkApplicationInfo.pEngineName = "vulkan-taoyao"; + this->vkApplicationInfo.pApplicationName = "vulkan-taoyao"; + OH_LOG_INFO(LOG_APP, "配置vkApplicationInfo"); + // vkInstanceCreateInfo + this->vkInstanceCreateInfo.sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO; + this->vkInstanceCreateInfo.pNext = NULL; + this->vkInstanceCreateInfo.pApplicationInfo = &this->vkApplicationInfo; + OH_LOG_INFO(LOG_APP, "配置vkInstanceCreateInfo"); + // vkInstanceCreateInfo + std::vector instanceExtensions = { + VK_KHR_SURFACE_EXTENSION_NAME, + VK_OHOS_SURFACE_EXTENSION_NAME + }; + vkInstanceCreateInfo.enabledExtensionCount = static_cast(instanceExtensions.size()); + vkInstanceCreateInfo.ppEnabledExtensionNames = instanceExtensions.data(); + OH_LOG_INFO(LOG_APP, "配置vkInstanceCreateInfo"); + VkResult ret = vkCreateInstance(&this->vkInstanceCreateInfo, nullptr, &this->vkInstance); + OH_LOG_INFO(LOG_APP, "加载vkInstance：%o", ret); + this->vkSurfaceCreateInfoOHOS.sType = VK_STRUCTURE_TYPE_SURFACE_CREATE_INFO_OHOS; + this->vkSurfaceCreateInfoOHOS.window = this->nativeWindow; + OH_LOG_INFO(LOG_APP, "配置vkSurfaceCreateInfoOHOS"); + ret = vkCreateSurfaceOHOS(this->vkInstance, &this->vkSurfaceCreateInfoOHOS, nullptr, &this->vkSurfaceKHR); + OH_LOG_INFO(LOG_APP, "加载vkSurfaceKHR：%o", ret); +} + +void acgist::VideoCapturer::releaseVulkan() { + if(this->vkSurfaceKHR != VK_NULL_HANDLE) { + vkDestroySurfaceKHR(this->vkInstance, this->vkSurfaceKHR, nullptr); + this->vkSurfaceKHR = VK_NULL_HANDLE; + } + if(this->vkInstance != VK_NULL_HANDLE) { + vkDestroyInstance(this->vkInstance, nullptr); + this->vkInstance = VK_NULL_HANDLE; + } +} + void acgist::VideoCapturer::initOpenGLES() { // IMAGE WINDOW glGenTextures(this->textureSize, &this->textureId); diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoDecoder.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoDecoder.cpp new file mode 100644 index 0000000..cbf63f9 --- /dev/null +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoDecoder.cpp @@ -0,0 +1 @@ +#include "../include/WebRTC.hpp" diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoEncoder.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoEncoder.cpp index dc794df..21192ed 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoEncoder.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoEncoder.cpp @@ -1,4 +1,4 @@ -#include "../include/Capturer.hpp" +#include "../include/WebRTC.hpp" #include "hilog/log.h" @@ -146,7 +146,6 @@ static void OnNeedInputBuffer(OH_AVCodec* codec, uint32_t index, OH_AVBuffer* bu } static void OnNewOutputBuffer(OH_AVCodec* codec, uint32_t index, OH_AVBuffer* buffer, void* userData) { - acgist::VideoCapturer* videoCapturer = (acgist::VideoCapturer*) userData; // TODO: 全局是否性能更好 OH_AVCodecBufferAttr info; OH_AVErrCode ret = OH_AVBuffer_GetBufferAttr(buffer, &info); diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/WebRTC.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/WebRTC.cpp index 0d981ce..b748dba 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/WebRTC.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/WebRTC.cpp @@ -1,14 +1,55 @@ -#include -#include -#include -#include +#include "../include/WebRTC.hpp" -std::unique_ptr webrtc::CreateBuiltinVideoDecoderFactory() { - // TODO: 硬件编解码 - return nullptr; +#include + +static const std::string h264ProfileLevelId = "42e01f"; + +acgist::TaoyaoVideoEncoderFactory::TaoyaoVideoEncoderFactory() { +} + +acgist::TaoyaoVideoEncoderFactory::~TaoyaoVideoEncoderFactory() { +} + +std::vector acgist::TaoyaoVideoEncoderFactory::GetSupportedFormats() const { + std::vector supported_codecs; + std::map params; + params["profile-level-id"] = h264ProfileLevelId; + params["packetization-mode"] = "1"; + params["level-asymmetry-allowed"] = "1"; + webrtc::SdpVideoFormat dst_format("H264", params); + for (const webrtc::SdpVideoFormat& format : webrtc::SupportedH264Codecs()) { + if (format == dst_format) { + supported_codecs.push_back(format); + break; + } + } + supported_codecs.push_back(webrtc::SdpVideoFormat(cricket::kVp8CodecName)); + supported_codecs.push_back(webrtc::SdpVideoFormat(cricket::kVp9CodecName)); + supported_codecs.push_back(webrtc::SdpVideoFormat(cricket::kH264CodecName)); + return supported_codecs; +} + +std::unique_ptr acgist::TaoyaoVideoEncoderFactory::CreateVideoEncoder(const webrtc::SdpVideoFormat& format) { + OH_LOG_DEBUG(LOG_APP, "返回WebRTC编码器：%o", format.name.data()); + // 硬编 + + // 软便 + if (cricket::CodecNamesEq(format.name, cricket::kVp8CodecName)) { + return VP8Encoder::Create(); + } + if (cricket::CodecNamesEq(format.name, cricket::kVp9CodecName)) { + return VP9Encoder::Create(cricket::VideoCodec(format)); + } + if (cricket::CodecNamesEq(format.name, cricket::kH264CodecName)) { + return H264Encoder::Create(cricket::VideoCodec(format)); + } + return nullptr; } std::unique_ptr webrtc::CreateBuiltinVideoEncoderFactory() { - // TODO: 硬件编解码 - return nullptr; -} \ No newline at end of file + return std::unique_ptr(new acgist::TaoyaoVideoEncoderFactory()); +} + +std::unique_ptr webrtc::CreateBuiltinVideoDecoderFactory() { + return std::unique_ptr(new acgist::TaoyaoVideoDecoderFactory()); +} diff --git a/taoyao-client-openharmony/taoyao/media/src/main/ets/pages/Index.ets b/taoyao-client-openharmony/taoyao/media/src/main/ets/pages/Index.ets index e41af5d..64a16cb 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/ets/pages/Index.ets +++ b/taoyao-client-openharmony/taoyao/media/src/main/ets/pages/Index.ets @@ -6,6 +6,15 @@ struct Index { build() { Row() { + Column() { + Button("加载系统") + .fontSize(20) + .fontWeight(FontWeight.Bold) + .onClick(() => { + signal.init(); + }); + } + .width("50%"); Column() { Button("连接信令") .fontSize(20) @@ -24,6 +33,15 @@ struct Index { }); } .width("50%"); + Column() { + Button("卸载系统") + .fontSize(20) + .fontWeight(FontWeight.Bold) + .onClick(() => { + signal.shutdown(); + }); + } + .width("50%"); } .height("100%"); } diff --git a/taoyao-client-openharmony/taoyao/media/src/main/ets/taoyao/TaoyaoSignal.ets b/taoyao-client-openharmony/taoyao/media/src/main/ets/taoyao/TaoyaoSignal.ets index bc0c8c2..849e856 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/ets/taoyao/TaoyaoSignal.ets +++ b/taoyao-client-openharmony/taoyao/media/src/main/ets/taoyao/TaoyaoSignal.ets @@ -35,6 +35,14 @@ class TaoyaoSignal { // 当前终端索引 clientIndex: number = 99999; + init() { + taoyaoModule.init(JSON.stringify(setting.signal), signal.nativePush, signal.nativeRequest); + } + + shutdown() { + taoyaoModule.shutdown("{}"); + } + /** * 连接信令 */