qq27800734/taoyao
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[*] 编解码

Browse Source
This commit is contained in:
acgist
2024-05-09 09:01:44 +08:00
parent b37445398e
commit 4947ef8c4c
10 changed files with 267 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
taoyao-client-openharmony/taoyao/media/src/main/cpp/CMakeLists.txt
View File

@@ -8,14 +8,14 @@ project(taoyao VERSION 1.0.0 LANGUAGES C CXX)
# C编译选项
set(CMAKE_C_STANDARD 17)
#set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c17 -O3")
set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -std=c17 -O0 -g")
#set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c17 -O3")
set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -std=c17 -O0 -g")
set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -std=c17 -O3")
# C++编译选项
set(CMAKE_CXX_STANDARD 17)
#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -O3")
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -std=c++17 -O0 -g")
#set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17 -O3")
set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -std=c++17 -O0 -g")
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -std=c++17 -O3")
option(MEDIASOUPCLIENT_LOG_DEV OFF)
@@ -24,6 +24,8 @@ option(MEDIASOUPCLIENT_BUILD_TESTS OFF)
set(LIBWEBRTC_BINARY_PATH "${CMAKE_CURRENT_SOURCE_DIR}/deps/webrtc/lib/")
set(LIBWEBRTC_INCLUDE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/deps/webrtc/src/")
ADD_DEFINITIONS(-DVK_USE_PLATFORM_OHOS=1)
add_subdirectory("./deps/libmediasoupclient")
add_library(
@@ -38,6 +40,7 @@ add_library(
media/Player.cpp
media/AudioPlayer.cpp
media/VideoPlayer.cpp
media/VideoDecoder.cpp
media/Capturer.cpp
media/AudioCapturer.cpp
media/VideoCapturer.cpp
@@ -61,21 +64,26 @@ target_link_libraries(
libace_napi.z.so
# LOG
libhilog_ndk.z.so
# EGL
libEGL.so
# OpenGL ES
libGLESv3.so
# Vulkan
libvulkan.so
# 音频
libohaudio.so
# 相机
libohcamera.so
# 图片
libnative_image.so
# 编码解码
libnative_media_aenc.so
libnative_media_venc.so
libnative_media_core.so
libnative_media_codecbase.so
# EGL
libEGL.so
libGLESv3.so
# 图片
libnative_image.so
# NativeBuffer
libnative_buffer.so
# NativeWindow
libnative_window.so
)

62
taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Capturer.hpp
View File

@@ -19,6 +19,11 @@
#ifndef TAOYAO_CAPTURER_HPP
#define TAOYAO_CAPTURER_HPP
#define __VULKAN__ true
#ifndef __VULKAN__
#define __OPENGL__ true
#endif
#include <map>
#include <EGL/egl.h>
@@ -27,6 +32,8 @@
#include "./Signal.hpp"
#include <vulkan/vulkan.h>
#include <native_image/native_image.h>
#include <native_buffer/native_buffer.h>
#include <native_window/external_window.h>
@@ -40,8 +47,6 @@
#include <ohaudio/native_audiocapturer.h>
#include <ohaudio/native_audiostreambuilder.h>
#include <multimedia/player_framework/native_avcodec_videoencoder.h>
namespace acgist {
/**
@@ -128,63 +133,36 @@ public:
};
/**
* 视频编码
*/
class VideoEncoder {
public:
// 视频编码器
OH_AVCodec* avCodec = nullptr;
// 视频窗口
OHNativeWindow* nativeWindow = nullptr;
public:
VideoEncoder();
virtual ~VideoEncoder();
public:
// 初始配置
void initFormatConfig(OH_AVFormat* format);
// 重新开始
void restart();
// 动态配置
void reset(OH_AVFormat* format);
// 动态配置
void resetIntConfig(const char* key, int32_t value);
// 动态配置
void resetLongConfig(const char* key, int64_t value);
// 动态配置
void resetDoubleConfig(const char* key, double value);
// 开始编码
virtual bool start();
// 结束编码
virtual bool stop();
};
/**
* 视频采集器
*/
class VideoCapturer: public Capturer<rtc::VideoSinkInterface<webrtc::VideoFrame>> {
public:
// ================ Vulkan ================
VkInstance vkInstance = VK_NULL_HANDLE;
VkSurfaceKHR vkSurfaceKHR = VK_NULL_HANDLE;
VkApplicationInfo vkApplicationInfo = {};
VkInstanceCreateInfo vkInstanceCreateInfo = {};
VkSurfaceCreateInfoOHOS vkSurfaceCreateInfoOHOS = {};
// ================ OpenGL ES ================
// SurfaceId
uint64_t surfaceId = 0;
// OpenGL纹理指针
GLuint textureId = 0;
// OpenGL纹理数量
GLsizei textureSize = 1;
// NativeImage
OH_NativeImage* nativeImage = nullptr;
// OHNativeWindow
OHNativeWindow* nativeWindow = nullptr;
// EGL显示设备
EGLDisplay eglDisplay = EGL_NO_DISPLAY;
// EGL上下文
EGLContext eglContext = EGL_NO_CONTEXT;
// EGL Surface
EGLSurface eglSurface = EGL_NO_SURFACE;
// ================ Camera ================
// NativeImage
OH_NativeImage* nativeImage = nullptr;
// OHNativeWindow
OHNativeWindow* nativeWindow = nullptr;
// 摄像头设备数量
uint32_t cameraSize = 0;
// 摄像头索引
@@ -205,6 +183,8 @@ public:
virtual ~VideoCapturer();
public:
void initVulkan();
void releaseVulkan();
void initOpenGLES();
void releaseOpenGLES();
virtual bool start() override;

105
taoyao-client-openharmony/taoyao/media/src/main/cpp/include/WebRTC.hpp Normal file
View File

@@ -0,0 +1,105 @@
#ifndef TAOYAO_WEBRTC_HPP
#define TAOYAO_WEBRTC_HPP
#include "./Signal.hpp"
#include <ohaudio/native_audiorenderer.h>
#include <ohaudio/native_audiostreambuilder.h>
#include <multimedia/player_framework/native_avformat.h>
#include <multimedia/player_framework/native_avbuffer.h>
#include <multimedia/player_framework/native_avcodec_base.h>
#include <multimedia/player_framework/native_avcapability.h>
#include <multimedia/player_framework/native_avcodec_videoencoder.h>
#include <multimedia/player_framework/native_avcodec_videodecoder.h>
#include "api/rtp_sender_interface.h"
#include "api/video_codecs/sdp_video_format.h"
#include "api/audio_codecs/builtin_audio_decoder_factory.h"
#include "api/audio_codecs/builtin_audio_encoder_factory.h"
#include "api/video_codecs/builtin_video_decoder_factory.h"
#include "api/video_codecs/builtin_video_encoder_factory.h"
#include "modules/video_coding/codecs/vp8/include/vp8.h"
#include "modules/video_coding/codecs/vp9/include/vp9.h"
#include "modules/video_coding/codecs/h264/include/h264.h"
namespace acgist {
/**
* 视频编码
*/
class VideoEncoder : public webrtc::VideoEncoder {
public:
// 视频编码器
OH_AVCodec* avCodec = nullptr;
// 视频窗口
OHNativeWindow* nativeWindow = nullptr;
public:
VideoEncoder();
virtual ~VideoEncoder();
public:
// 初始配置
void initFormatConfig(OH_AVFormat* format);
// 重新开始
void restart();
// 动态配置
void reset(OH_AVFormat* format);
// 动态配置
void resetIntConfig(const char* key, int32_t value);
// 动态配置
void resetLongConfig(const char* key, int64_t value);
// 动态配置
void resetDoubleConfig(const char* key, double value);
// 开始编码
virtual bool start();
// 结束编码
virtual bool stop();
};
/**
* 视频解码器
*/
class VideoDecoder : public webrtc::VideoDecoder {
public:
VideoDecoder();
virtual ~VideoDecoder();
public:
virtual bool start();
virtual bool stop();
};
class TaoyaoVideoEncoderFactory : webrtc::VideoEncoderFactory {
public:
TaoyaoVideoEncoderFactory();
virtual ~TaoyaoVideoEncoderFactory();
public:
virtual std::vector<webrtc::SdpVideoFormat> GetSupportedFormats() const override;
virtual std::unique_ptr<webrtc::VideoEncoder> CreateVideoEncoder(const webrtc::SdpVideoFormat& format) override;
};
class TaoyaoVideoDecoderFactory : webrtc::VideoDecoderFactory {
public:
TaoyaoVideoDecoderFactory();
virtual ~TaoyaoVideoDecoderFactory();
public:
virtual std::vector<webrtc::SdpVideoFormat> GetSupportedFormats() const override;
virtual std::unique_ptr<webrtc::VideoDecoder> CreateVideoDecoder(const webrtc::SdpVideoFormat& format) override;
};
}
#endif //TAOYAO_WEBRTC_HPP

4
taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioPlayer.cpp
View File

@@ -1,10 +1,10 @@
#include "../include/Player.hpp"
#include <hilog/log.h>
#include <ohaudio/native_audiorenderer.h>
#include <ohaudio/native_audiostreambuilder.h>
#include <hilog/log.h>
// 播放回调
static int32_t OnError(OH_AudioRenderer* renderer, void* userData, OH_AudioStream_Result error);
static int32_t OnWriteData(OH_AudioRenderer* renderer, void* userData, void* buffer, int32_t length);

43
taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoCapturer.cpp
View File

@@ -42,6 +42,9 @@ acgist::VideoCapturer::VideoCapturer() {
OH_LOG_INFO(LOG_APP, "获取摄像头输出功能：%o %d %d", ret, this->cameraIndex, this->cameraOutputCapability->videoProfilesSize);
// 注册相机状态回调
// OH_CameraManager_RegisterCallback(this->cameraManager, CameraManager_Callbacks* callback);
// char surfaceId[OH_XCOMPONENT_ID_LEN_MAX + 1] = {};
// uint64_t idSize = OH_XCOMPONENT_ID_LEN_MAX + 1;
// OH_NativeXComponent_GetXComponentId((OH_NativeXComponent*) this->nativeWindow, surfaceId, idSize);
ret = OH_CameraManager_CreateVideoOutput(this->cameraManager, this->cameraOutputCapability->videoProfiles[0], (char*) this->surfaceId, &this->cameraVideoOutput);
OH_LOG_INFO(LOG_APP, "创建摄像头视频输出：%o", ret);
ret = OH_CameraManager_CreateCaptureSession(this->cameraManager, &this->cameraCaptureSession);
@@ -111,6 +114,46 @@ bool acgist::VideoCapturer::stop() {
return ret = Camera_ErrorCode::CAMERA_OK;
}
void acgist::VideoCapturer::initVulkan() {
// vkApplicationInfo
this->vkApplicationInfo.sType = VK_STRUCTURE_TYPE_APPLICATION_INFO;
this->vkApplicationInfo.apiVersion = VK_API_VERSION_1_3;
this->vkApplicationInfo.pEngineName = "vulkan-taoyao";
this->vkApplicationInfo.pApplicationName = "vulkan-taoyao";
OH_LOG_INFO(LOG_APP, "配置vkApplicationInfo");
// vkInstanceCreateInfo
this->vkInstanceCreateInfo.sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO;
this->vkInstanceCreateInfo.pNext = NULL;
this->vkInstanceCreateInfo.pApplicationInfo = &this->vkApplicationInfo;
OH_LOG_INFO(LOG_APP, "配置vkInstanceCreateInfo");
// vkInstanceCreateInfo
std::vector<const char*> instanceExtensions = {
VK_KHR_SURFACE_EXTENSION_NAME,
VK_OHOS_SURFACE_EXTENSION_NAME
};
vkInstanceCreateInfo.enabledExtensionCount = static_cast<uint32_t>(instanceExtensions.size());
vkInstanceCreateInfo.ppEnabledExtensionNames = instanceExtensions.data();
OH_LOG_INFO(LOG_APP, "配置vkInstanceCreateInfo");
VkResult ret = vkCreateInstance(&this->vkInstanceCreateInfo, nullptr, &this->vkInstance);
OH_LOG_INFO(LOG_APP, "加载vkInstance：%o", ret);
this->vkSurfaceCreateInfoOHOS.sType = VK_STRUCTURE_TYPE_SURFACE_CREATE_INFO_OHOS;
this->vkSurfaceCreateInfoOHOS.window = this->nativeWindow;
OH_LOG_INFO(LOG_APP, "配置vkSurfaceCreateInfoOHOS");
ret = vkCreateSurfaceOHOS(this->vkInstance, &this->vkSurfaceCreateInfoOHOS, nullptr, &this->vkSurfaceKHR);
OH_LOG_INFO(LOG_APP, "加载vkSurfaceKHR：%o", ret);
}
void acgist::VideoCapturer::releaseVulkan() {
if(this->vkSurfaceKHR != VK_NULL_HANDLE) {
vkDestroySurfaceKHR(this->vkInstance, this->vkSurfaceKHR, nullptr);
this->vkSurfaceKHR = VK_NULL_HANDLE;
}
if(this->vkInstance != VK_NULL_HANDLE) {
vkDestroyInstance(this->vkInstance, nullptr);
this->vkInstance = VK_NULL_HANDLE;
}
}
void acgist::VideoCapturer::initOpenGLES() {
// IMAGE WINDOW
glGenTextures(this->textureSize, &this->textureId);

1
taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoDecoder.cpp Normal file
View File

@@ -0,0 +1 @@
#include "../include/WebRTC.hpp"

3
taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoEncoder.cpp
View File

@@ -1,4 +1,4 @@
#include "../include/Capturer.hpp"
#include "../include/WebRTC.hpp"
#include "hilog/log.h"
@@ -146,7 +146,6 @@ static void OnNeedInputBuffer(OH_AVCodec* codec, uint32_t index, OH_AVBuffer* bu
}
static void OnNewOutputBuffer(OH_AVCodec* codec, uint32_t index, OH_AVBuffer* buffer, void* userData) {
acgist::VideoCapturer* videoCapturer = (acgist::VideoCapturer*) userData;
// TODO: 全局是否性能更好
OH_AVCodecBufferAttr info;
OH_AVErrCode ret = OH_AVBuffer_GetBufferAttr(buffer, &info);

57
taoyao-client-openharmony/taoyao/media/src/main/cpp/media/WebRTC.cpp
View File

@@ -1,14 +1,55 @@
#include <api/audio_codecs/builtin_audio_decoder_factory.h>
#include <api/audio_codecs/builtin_audio_encoder_factory.h>
#include <api/video_codecs/builtin_video_decoder_factory.h>
#include <api/video_codecs/builtin_video_encoder_factory.h>
#include "../include/WebRTC.hpp"
std::unique_ptr<webrtc::VideoDecoderFactory> webrtc::CreateBuiltinVideoDecoderFactory() {
// TODO: 硬件编解码
#include <hilog/log.h>
static const std::string h264ProfileLevelId = "42e01f";
acgist::TaoyaoVideoEncoderFactory::TaoyaoVideoEncoderFactory() {
}
acgist::TaoyaoVideoEncoderFactory::~TaoyaoVideoEncoderFactory() {
}
std::vector<webrtc::SdpVideoFormat> acgist::TaoyaoVideoEncoderFactory::GetSupportedFormats() const {
std::vector<webrtc::SdpVideoFormat> supported_codecs;
std::map<std::string, std::string> params;
params["profile-level-id"] = h264ProfileLevelId;
params["packetization-mode"] = "1";
params["level-asymmetry-allowed"] = "1";
webrtc::SdpVideoFormat dst_format("H264", params);
for (const webrtc::SdpVideoFormat& format : webrtc::SupportedH264Codecs()) {
if (format == dst_format) {
supported_codecs.push_back(format);
break;
}
}
supported_codecs.push_back(webrtc::SdpVideoFormat(cricket::kVp8CodecName));
supported_codecs.push_back(webrtc::SdpVideoFormat(cricket::kVp9CodecName));
supported_codecs.push_back(webrtc::SdpVideoFormat(cricket::kH264CodecName));
return supported_codecs;
}
std::unique_ptr<webrtc::VideoEncoder> acgist::TaoyaoVideoEncoderFactory::CreateVideoEncoder(const webrtc::SdpVideoFormat& format) {
OH_LOG_DEBUG(LOG_APP, "返回WebRTC编码器：%o", format.name.data());
// 硬编
// 软便
if (cricket::CodecNamesEq(format.name, cricket::kVp8CodecName)) {
return VP8Encoder::Create();
}
if (cricket::CodecNamesEq(format.name, cricket::kVp9CodecName)) {
return VP9Encoder::Create(cricket::VideoCodec(format));
}
if (cricket::CodecNamesEq(format.name, cricket::kH264CodecName)) {
return H264Encoder::Create(cricket::VideoCodec(format));
}
return nullptr;
}
std::unique_ptr<webrtc::VideoEncoderFactory> webrtc::CreateBuiltinVideoEncoderFactory() {
// TODO: 硬件编解码
return nullptr;
return std::unique_ptr(new acgist::TaoyaoVideoEncoderFactory());
}
std::unique_ptr<webrtc::VideoDecoderFactory> webrtc::CreateBuiltinVideoDecoderFactory() {
return std::unique_ptr(new acgist::TaoyaoVideoDecoderFactory());
}

18
taoyao-client-openharmony/taoyao/media/src/main/ets/pages/Index.ets
View File

@@ -6,6 +6,15 @@ struct Index {
build() {
Row() {
Column() {
Button("加载系统")
.fontSize(20)
.fontWeight(FontWeight.Bold)
.onClick(() => {
signal.init();
});
}
.width("50%");
Column() {
Button("连接信令")
.fontSize(20)
@@ -24,6 +33,15 @@ struct Index {
});
}
.width("50%");
Column() {
Button("卸载系统")
.fontSize(20)
.fontWeight(FontWeight.Bold)
.onClick(() => {
signal.shutdown();
});
}
.width("50%");
}
.height("100%");
}

8
taoyao-client-openharmony/taoyao/media/src/main/ets/taoyao/TaoyaoSignal.ets
View File

@@ -35,6 +35,14 @@ class TaoyaoSignal {
// 当前终端索引
clientIndex: number = 99999;
init() {
taoyaoModule.init(JSON.stringify(setting.signal), signal.nativePush, signal.nativeRequest);
}
shutdown() {
taoyaoModule.shutdown("{}");
}
/**
* 连接信令
*/