[*] 编解码

taoyao-client-openharmony/taoyao/media/src/main/cpp/CMakeLists.txt

@@ -24,6 +24,8 @@ option(MEDIASOUPCLIENT_BUILD_TESTS OFF)
 set(LIBWEBRTC_BINARY_PATH  "${CMAKE_CURRENT_SOURCE_DIR}/deps/webrtc/lib/")
 set(LIBWEBRTC_INCLUDE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/deps/webrtc/src/")
 
+ADD_DEFINITIONS(-DVK_USE_PLATFORM_OHOS=1)
+
 add_subdirectory("./deps/libmediasoupclient")
 
 add_library(
@@ -38,6 +40,7 @@ add_library(
     media/Player.cpp
     media/AudioPlayer.cpp
     media/VideoPlayer.cpp
+    media/VideoDecoder.cpp
     media/Capturer.cpp
     media/AudioCapturer.cpp
     media/VideoCapturer.cpp
@@ -61,21 +64,26 @@ target_link_libraries(
     libace_napi.z.so
     # LOG
     libhilog_ndk.z.so
+    # EGL
+    libEGL.so
+    # OpenGL ES
+    libGLESv3.so
+    # Vulkan
+    libvulkan.so
     # 音频
     libohaudio.so
     # 相机
     libohcamera.so
+    # 图片
+    libnative_image.so
     # 编码解码
     libnative_media_aenc.so
     libnative_media_venc.so
     libnative_media_core.so
     libnative_media_codecbase.so
-    # EGL
+    # NativeBuffer
-    libEGL.so
-    libGLESv3.so
-    # 图片
-    libnative_image.so
     libnative_buffer.so
+    # NativeWindow
     libnative_window.so
 )
 

taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Capturer.hpp

@@ -19,6 +19,11 @@
 #ifndef TAOYAO_CAPTURER_HPP
 #define TAOYAO_CAPTURER_HPP
 
+#define __VULKAN__ true
+#ifndef __VULKAN__
+#define __OPENGL__ true
+#endif
+
 #include <map>
 
 #include <EGL/egl.h>
@@ -27,6 +32,8 @@
 
 #include "./Signal.hpp"
 
+#include <vulkan/vulkan.h>
+
 #include <native_image/native_image.h>
 #include <native_buffer/native_buffer.h>
 #include <native_window/external_window.h>
@@ -40,8 +47,6 @@
 #include <ohaudio/native_audiocapturer.h>
 #include <ohaudio/native_audiostreambuilder.h>
 
-#include <multimedia/player_framework/native_avcodec_videoencoder.h>
-
 namespace acgist {
 
 /**
@@ -128,63 +133,36 @@ public:
     
 };
 
-/**
- * 视频编码
- */
-class VideoEncoder {
-    
-public:
-    // 视频编码器
-    OH_AVCodec* avCodec = nullptr;
-    // 视频窗口
-    OHNativeWindow* nativeWindow = nullptr;
-    
-public:
-    VideoEncoder();
-    virtual ~VideoEncoder();
-    
-public:
-    // 初始配置
-    void initFormatConfig(OH_AVFormat* format);
-    // 重新开始
-    void restart();
-    // 动态配置
-    void reset(OH_AVFormat* format);
-    // 动态配置
-    void resetIntConfig(const char* key, int32_t value);
-    // 动态配置
-    void resetLongConfig(const char* key, int64_t value);
-    // 动态配置
-    void resetDoubleConfig(const char* key, double value);
-    // 开始编码
-    virtual bool start();
-    // 结束编码
-    virtual bool stop();
-    
-};
-
 /**
  * 视频采集器
  */
 class VideoCapturer: public Capturer<rtc::VideoSinkInterface<webrtc::VideoFrame>> {
     
 public:
+    // ================ Vulkan ================
+    VkInstance vkInstance = VK_NULL_HANDLE;
+    VkSurfaceKHR vkSurfaceKHR = VK_NULL_HANDLE;
+    VkApplicationInfo vkApplicationInfo = {};
+    VkInstanceCreateInfo vkInstanceCreateInfo = {};
+    VkSurfaceCreateInfoOHOS vkSurfaceCreateInfoOHOS = {};
+    // ================ OpenGL ES ================
     // SurfaceId
     uint64_t surfaceId = 0;
     // OpenGL纹理指针
     GLuint textureId = 0;
     // OpenGL纹理数量
     GLsizei textureSize = 1;
-    // NativeImage
-    OH_NativeImage* nativeImage = nullptr;
-    // OHNativeWindow
-    OHNativeWindow* nativeWindow = nullptr;
     // EGL显示设备
     EGLDisplay eglDisplay = EGL_NO_DISPLAY;
     // EGL上下文
     EGLContext eglContext = EGL_NO_CONTEXT;
     // EGL Surface
     EGLSurface eglSurface = EGL_NO_SURFACE;
+    // ================ Camera ================
+    // NativeImage
+    OH_NativeImage* nativeImage = nullptr;
+    // OHNativeWindow
+    OHNativeWindow* nativeWindow = nullptr;
     // 摄像头设备数量
     uint32_t cameraSize = 0;
     // 摄像头索引
@@ -205,6 +183,8 @@ public:
     virtual ~VideoCapturer();
     
 public:
+    void initVulkan();
+    void releaseVulkan();
     void initOpenGLES();
     void releaseOpenGLES();
     virtual bool start() override;

taoyao-client-openharmony/taoyao/media/src/main/cpp/include/WebRTC.hpp

@@ -0,0 +1,105 @@
+#ifndef TAOYAO_WEBRTC_HPP
+#define TAOYAO_WEBRTC_HPP
+
+#include "./Signal.hpp"
+
+#include <ohaudio/native_audiorenderer.h>
+#include <ohaudio/native_audiostreambuilder.h>
+
+#include <multimedia/player_framework/native_avformat.h>
+#include <multimedia/player_framework/native_avbuffer.h>
+#include <multimedia/player_framework/native_avcodec_base.h>
+#include <multimedia/player_framework/native_avcapability.h>
+#include <multimedia/player_framework/native_avcodec_videoencoder.h>
+#include <multimedia/player_framework/native_avcodec_videodecoder.h>
+
+#include "api/rtp_sender_interface.h"
+#include "api/video_codecs/sdp_video_format.h"
+#include "api/audio_codecs/builtin_audio_decoder_factory.h"
+#include "api/audio_codecs/builtin_audio_encoder_factory.h"
+#include "api/video_codecs/builtin_video_decoder_factory.h"
+#include "api/video_codecs/builtin_video_encoder_factory.h"
+
+#include "modules/video_coding/codecs/vp8/include/vp8.h"
+#include "modules/video_coding/codecs/vp9/include/vp9.h"
+#include "modules/video_coding/codecs/h264/include/h264.h"
+
+namespace acgist {
+
+/**
+ * 视频编码
+ */
+class VideoEncoder : public webrtc::VideoEncoder {
+    
+public:
+    // 视频编码器
+    OH_AVCodec* avCodec = nullptr;
+    // 视频窗口
+    OHNativeWindow* nativeWindow = nullptr;
+    
+public:
+    VideoEncoder();
+    virtual ~VideoEncoder();
+    
+public:
+    // 初始配置
+    void initFormatConfig(OH_AVFormat* format);
+    // 重新开始
+    void restart();
+    // 动态配置
+    void reset(OH_AVFormat* format);
+    // 动态配置
+    void resetIntConfig(const char* key, int32_t value);
+    // 动态配置
+    void resetLongConfig(const char* key, int64_t value);
+    // 动态配置
+    void resetDoubleConfig(const char* key, double value);
+    // 开始编码
+    virtual bool start();
+    // 结束编码
+    virtual bool stop();
+    
+};
+
+/**
+ * 视频解码器
+ */
+class VideoDecoder : public webrtc::VideoDecoder {
+    
+public:
+    VideoDecoder();
+    virtual ~VideoDecoder();
+    
+public:
+    virtual bool start();
+    virtual bool stop();
+    
+};
+
+class TaoyaoVideoEncoderFactory : webrtc::VideoEncoderFactory {
+    
+public:
+    TaoyaoVideoEncoderFactory();
+    virtual ~TaoyaoVideoEncoderFactory();
+    
+public:
+    virtual std::vector<webrtc::SdpVideoFormat> GetSupportedFormats() const override;
+    virtual std::unique_ptr<webrtc::VideoEncoder> CreateVideoEncoder(const webrtc::SdpVideoFormat& format) override;
+    
+};
+
+class TaoyaoVideoDecoderFactory : webrtc::VideoDecoderFactory {
+    
+public:
+    TaoyaoVideoDecoderFactory();
+    virtual ~TaoyaoVideoDecoderFactory();
+    
+public:
+    virtual std::vector<webrtc::SdpVideoFormat> GetSupportedFormats() const override;
+    virtual std::unique_ptr<webrtc::VideoDecoder> CreateVideoDecoder(const webrtc::SdpVideoFormat& format) override;
+
+};
+
+}
+
+#endif //TAOYAO_WEBRTC_HPP

taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioPlayer.cpp

@@ -1,10 +1,10 @@
 #include "../include/Player.hpp"
 
+#include <hilog/log.h>
+
 #include <ohaudio/native_audiorenderer.h>
 #include <ohaudio/native_audiostreambuilder.h>
 
-#include <hilog/log.h>
-
 // 播放回调
 static int32_t OnError(OH_AudioRenderer* renderer, void* userData, OH_AudioStream_Result error);
 static int32_t OnWriteData(OH_AudioRenderer* renderer, void* userData, void* buffer, int32_t length);

taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoCapturer.cpp

@@ -42,6 +42,9 @@ acgist::VideoCapturer::VideoCapturer() {
     OH_LOG_INFO(LOG_APP, "获取摄像头输出功能：%o %d %d", ret, this->cameraIndex, this->cameraOutputCapability->videoProfilesSize);
     // 注册相机状态回调
     // OH_CameraManager_RegisterCallback(this->cameraManager, CameraManager_Callbacks* callback);
+//    char surfaceId[OH_XCOMPONENT_ID_LEN_MAX + 1] = {};
+//    uint64_t idSize = OH_XCOMPONENT_ID_LEN_MAX + 1;
+//    OH_NativeXComponent_GetXComponentId((OH_NativeXComponent*) this->nativeWindow, surfaceId, idSize);
     ret = OH_CameraManager_CreateVideoOutput(this->cameraManager, this->cameraOutputCapability->videoProfiles[0], (char*) this->surfaceId, &this->cameraVideoOutput);
     OH_LOG_INFO(LOG_APP, "创建摄像头视频输出：%o", ret);
     ret = OH_CameraManager_CreateCaptureSession(this->cameraManager, &this->cameraCaptureSession);
@@ -111,6 +114,46 @@ bool acgist::VideoCapturer::stop() {
     return ret = Camera_ErrorCode::CAMERA_OK;
 }
 
+void acgist::VideoCapturer::initVulkan() {
+    // vkApplicationInfo
+    this->vkApplicationInfo.sType            = VK_STRUCTURE_TYPE_APPLICATION_INFO;
+    this->vkApplicationInfo.apiVersion       = VK_API_VERSION_1_3;
+    this->vkApplicationInfo.pEngineName      = "vulkan-taoyao";
+    this->vkApplicationInfo.pApplicationName = "vulkan-taoyao";
+    OH_LOG_INFO(LOG_APP, "配置vkApplicationInfo");
+    // vkInstanceCreateInfo
+    this->vkInstanceCreateInfo.sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO;
+    this->vkInstanceCreateInfo.pNext = NULL;
+    this->vkInstanceCreateInfo.pApplicationInfo = &this->vkApplicationInfo;
+    OH_LOG_INFO(LOG_APP, "配置vkInstanceCreateInfo");
+    // vkInstanceCreateInfo
+    std::vector<const char*> instanceExtensions = {
+        VK_KHR_SURFACE_EXTENSION_NAME,
+        VK_OHOS_SURFACE_EXTENSION_NAME
+    };
+    vkInstanceCreateInfo.enabledExtensionCount   = static_cast<uint32_t>(instanceExtensions.size());
+    vkInstanceCreateInfo.ppEnabledExtensionNames = instanceExtensions.data();
+    OH_LOG_INFO(LOG_APP, "配置vkInstanceCreateInfo");
+    VkResult ret = vkCreateInstance(&this->vkInstanceCreateInfo, nullptr, &this->vkInstance);
+    OH_LOG_INFO(LOG_APP, "加载vkInstance：%o", ret);
+    this->vkSurfaceCreateInfoOHOS.sType  = VK_STRUCTURE_TYPE_SURFACE_CREATE_INFO_OHOS;
+    this->vkSurfaceCreateInfoOHOS.window = this->nativeWindow;
+    OH_LOG_INFO(LOG_APP, "配置vkSurfaceCreateInfoOHOS");
+    ret = vkCreateSurfaceOHOS(this->vkInstance, &this->vkSurfaceCreateInfoOHOS, nullptr, &this->vkSurfaceKHR);
+    OH_LOG_INFO(LOG_APP, "加载vkSurfaceKHR：%o", ret);
+}
+
+void acgist::VideoCapturer::releaseVulkan() {
+    if(this->vkSurfaceKHR != VK_NULL_HANDLE) {
+        vkDestroySurfaceKHR(this->vkInstance, this->vkSurfaceKHR, nullptr);
+        this->vkSurfaceKHR = VK_NULL_HANDLE;
+    }
+    if(this->vkInstance != VK_NULL_HANDLE) {
+        vkDestroyInstance(this->vkInstance, nullptr);
+        this->vkInstance = VK_NULL_HANDLE;
+    }
+}
+
 void acgist::VideoCapturer::initOpenGLES() {
     // IMAGE WINDOW
     glGenTextures(this->textureSize, &this->textureId);

taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoDecoder.cpp

@@ -0,0 +1 @@
+#include "../include/WebRTC.hpp"

taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoEncoder.cpp

@@ -1,4 +1,4 @@
-#include "../include/Capturer.hpp"
+#include "../include/WebRTC.hpp"
 
 #include "hilog/log.h"
 
@@ -146,7 +146,6 @@ static void OnNeedInputBuffer(OH_AVCodec* codec, uint32_t index, OH_AVBuffer* bu
 }
 
 static void OnNewOutputBuffer(OH_AVCodec* codec, uint32_t index, OH_AVBuffer* buffer, void* userData) {
-    acgist::VideoCapturer* videoCapturer = (acgist::VideoCapturer*) userData;
     // TODO: 全局是否性能更好
     OH_AVCodecBufferAttr info;
     OH_AVErrCode ret = OH_AVBuffer_GetBufferAttr(buffer, &info);

taoyao-client-openharmony/taoyao/media/src/main/cpp/media/WebRTC.cpp

@@ -1,14 +1,55 @@
-#include <api/audio_codecs/builtin_audio_decoder_factory.h>
+#include "../include/WebRTC.hpp"
-#include <api/audio_codecs/builtin_audio_encoder_factory.h>
-#include <api/video_codecs/builtin_video_decoder_factory.h>
-#include <api/video_codecs/builtin_video_encoder_factory.h>
 
-std::unique_ptr<webrtc::VideoDecoderFactory>  webrtc::CreateBuiltinVideoDecoderFactory() {
+#include <hilog/log.h>
-    // TODO: 硬件编解码
+
+static const std::string h264ProfileLevelId = "42e01f";
+
+acgist::TaoyaoVideoEncoderFactory::TaoyaoVideoEncoderFactory() {
+}
+
+acgist::TaoyaoVideoEncoderFactory::~TaoyaoVideoEncoderFactory() {
+}
+
+std::vector<webrtc::SdpVideoFormat> acgist::TaoyaoVideoEncoderFactory::GetSupportedFormats() const {
+    std::vector<webrtc::SdpVideoFormat> supported_codecs;
+    std::map<std::string, std::string> params;
+    params["profile-level-id"]        = h264ProfileLevelId;
+    params["packetization-mode"]      = "1";
+    params["level-asymmetry-allowed"] = "1";
+    webrtc::SdpVideoFormat dst_format("H264", params);
+    for (const webrtc::SdpVideoFormat& format : webrtc::SupportedH264Codecs()) {
+        if (format == dst_format) {
+            supported_codecs.push_back(format);
+            break;
+        }
+    }
+    supported_codecs.push_back(webrtc::SdpVideoFormat(cricket::kVp8CodecName));
+    supported_codecs.push_back(webrtc::SdpVideoFormat(cricket::kVp9CodecName));
+    supported_codecs.push_back(webrtc::SdpVideoFormat(cricket::kH264CodecName));
+    return supported_codecs;
+}
+
+std::unique_ptr<webrtc::VideoEncoder> acgist::TaoyaoVideoEncoderFactory::CreateVideoEncoder(const webrtc::SdpVideoFormat& format) {
+    OH_LOG_DEBUG(LOG_APP, "返回WebRTC编码器：%o", format.name.data());
+    // 硬编
+    
+    // 软便
+    if (cricket::CodecNamesEq(format.name, cricket::kVp8CodecName)) {
+        return VP8Encoder::Create();
+    }
+    if (cricket::CodecNamesEq(format.name, cricket::kVp9CodecName)) {
+        return VP9Encoder::Create(cricket::VideoCodec(format));
+    }
+    if (cricket::CodecNamesEq(format.name, cricket::kH264CodecName)) {
+        return H264Encoder::Create(cricket::VideoCodec(format));
+    }
     return nullptr;    
 }
 
 std::unique_ptr<webrtc::VideoEncoderFactory>  webrtc::CreateBuiltinVideoEncoderFactory() {
-    // TODO: 硬件编解码
+    return std::unique_ptr(new acgist::TaoyaoVideoEncoderFactory());
-    return nullptr;
+}
+
+std::unique_ptr<webrtc::VideoDecoderFactory> webrtc::CreateBuiltinVideoDecoderFactory() {
+    return std::unique_ptr(new acgist::TaoyaoVideoDecoderFactory());
 }

taoyao-client-openharmony/taoyao/media/src/main/ets/pages/Index.ets

@@ -6,6 +6,15 @@ struct Index {
 
   build() {
     Row() {
+      Column() {
+        Button("加载系统")
+          .fontSize(20)
+          .fontWeight(FontWeight.Bold)
+          .onClick(() => {
+            signal.init();
+          });
+      }
+      .width("50%");
       Column() {
         Button("连接信令")
           .fontSize(20)
@@ -24,6 +33,15 @@ struct Index {
           });
       }
       .width("50%");
+      Column() {
+        Button("卸载系统")
+          .fontSize(20)
+          .fontWeight(FontWeight.Bold)
+          .onClick(() => {
+            signal.shutdown();
+          });
+      }
+      .width("50%");
     }
     .height("100%");
   }

taoyao-client-openharmony/taoyao/media/src/main/ets/taoyao/TaoyaoSignal.ets

@@ -35,6 +35,14 @@ class TaoyaoSignal {
   // 当前终端索引
   clientIndex: number = 99999;
 
+  init() {
+    taoyaoModule.init(JSON.stringify(setting.signal), signal.nativePush, signal.nativeRequest);
+  }
+
+  shutdown() {
+    taoyaoModule.shutdown("{}");
+  }
+
   /**
    * 连接信令
    */